Skip to content

Instantly share code, notes, and snippets.

@blindman2k

blindman2k/spiflashfs.device.nut

Last active January 18, 2016 14:53
Show Gist options
  • Save blindman2k/206c5251e05b50844fcb to your computer and use it in GitHub Desktop.
Save blindman2k/206c5251e05b50844fcb to your computer and use it in GitHub Desktop.
Download ZIP
Updated SPI Flash File System with 4kb pages and no lookup tables or indices, just data pages with small headers.
Raw
spiflashfs.device.nut
/*
The physical space is divided into blocks (64k) and sectors (16k).
Erase (wiping 0's to 1's) is performed at the sector level.
The file system divides the available space into pages which the size of one or more sectors.
At the start of every spage is an header which contains:
- the file's id (two bytes) for identifying and rejoining parts of a file
- the span id (two bytes) for ordering the parts
- an optional filename (with length byte)
- the span's size (how much of the page is used)
At initialisation the page headers are scanned to bring the data into in-memory FAT which holds
- A blob map of free pages (one bit per page)
- An array of file ids -> ordered page numbers
- An array of file names -> file ids
Limitations:
- Appending to a file means wasting any remainder of the previous page
To do:
- Append (maybe not)
- Make _scan use less memory or write directly to _fat
- Asynch (optional) version of _scan()
- Optional SFFS_PAGE_SIZE (4k or multiples of)
- Clean up the back and forthing between the SFFS, FAT and File classes
- Cache next free pages
- Store the creation date
*/
//==============================================================================
class SPIFlashFileSystem {
_flash = null;
_size = null;
_start = null;
_end = null;
_len = null;
_pages = 0;
_enables = 0;
_fat = null;
_openFiles = null;
_nextFileIdx = 1;
_autoGcThreshold = 4;
_version = [1, 0, 0];
//--------------------------------------------------------------------------
constructor(start = null, end = null, flash = null) {
const SFFS_SECTOR_SIZE = 4096;
const SFFS_PAGE_SIZE = 4096;
const SFFS_MAX_FNAME_SIZE = 64;
const SFFS_MAX_TIMESTAMP_SIZE = 4;
const SFFS_HEADER_SIZE = 6; // id (2) + span (2) + size (2)
const SFFS_STATUS_FREE = 0x00;
const SFFS_STATUS_USED = 0x01;
const SFFS_STATUS_ERASED = 0x02;
const SFFS_STATUS_BAD = 0x03;
const SFFS_LOOKUP_MASK_ID = 0x7FFF;
const SFFS_LOOKUP_MASK_INDEX = 0x8000;
const SFFS_LOOKUP_FREE = 0xFFFF;
const SFFS_LOOKUP_ERASED = 0x0000;
const SFFS_LOOKUP_STAT_ERASED = 0x00;
const SFFS_LOOKUP_STAT_INDEX = 0x01;
const SFFS_LOOKUP_STAT_DATA = 0x02;
const SFFS_LOOKUP_STAT_FREE = 0xFF;
const SFFS_SPIFLASH_VERIFY = 1; // SPIFLASH_DONTVERIFY = 0, SPIFLASH_POSTVERIFY = 1, SPIFLASH_PREVERIFY = 2
_flash = flash ? flash : hardware.spiflash;
_enable();
_size = _flash.size();
_disable();
if (start == null) _start = 0;
else if (start < _size) _start = start;
else throw "Invalid start value";
if (_start % SFFS_SECTOR_SIZE != 0) throw "start must be at a sector boundary";
if (end == null) _end = _size;
else if (end > _start) _end = end;
else throw "Invalid end value";
if (_end % SFFS_SECTOR_SIZE != 0) throw "end must be at a sector boundary";
_len = _end - _start;
_pages = _len / SFFS_PAGE_SIZE;
_openFiles = {};
_fat = SPIFlashFileSystem.FAT(this, _pages);
}
//--------------------------------------------------------------------------
function init() {
if (_openFiles.len() > 0) return server.error("Can't call init() with open files");
// Free up some memory
_fat = null;
// Scan the pages for files
local scan = _scan();
_fat = SPIFlashFileSystem.FAT(this, scan.files, scan.pages);
}
//--------------------------------------------------------------------------
function dimensions() {
return { "size": _size, "len": _len, "start": _start, "end": _end }
}
//--------------------------------------------------------------------------
function isFileOpen(fname) {
// Search through the open file table for matching ids
local id = _fat.get(fname).id;
foreach (ptr,fileId in _openFiles) {
if (id == fileId) return true;
}
return false;
}
//--------------------------------------------------------------------------
function eraseFile(fname) {
if (!_fat.fileExists(fname)) throw "Can't find file '" + fname + "' to erase";
if (isFileOpen(fname)) throw "Can't erase open file '" + fname + "'";
_enable();
// server.log("Erasing " + fname)
local zeros = blob(SFFS_HEADER_SIZE + SFFS_MAX_TIMESTAMP_SIZE + SFFS_MAX_FNAME_SIZE + 1);
local pages = _fat.forEachPage(fname, function(addr) {
// Erase the page headers
local res = _flash.write(addr, zeros, SFFS_SPIFLASH_VERIFY);
assert(res == 0);
// Mark the page map
_fat.markPage(addr, SFFS_STATUS_ERASED);
}.bindenv(this));
_disable();
// Update the fat
_fat.removeFile(fname);
}
//--------------------------------------------------------------------------
function eraseFiles() {
if (_openFiles.len() > 0) return server.error("Can't call eraseAll() with open files");
_enable();
local files_to_erase = _fat.getFileList();
foreach (filename in files_to_erase) {
eraseFile(filename);
}
_disable();
}
//--------------------------------------------------------------------------
function eraseAll() {
if (_openFiles.len() > 0) return server.error("Can't call eraseAll() with open files");
// Empty the FAT
_fat = SPIFlashFileSystem.FAT(this, _pages);
// Format all the sectors
_enable()
for (local p = 0; p < _pages; p++) {
_flash.erasesector(p * SFFS_PAGE_SIZE);
}
_disable()
imp.sleep(0.05);
// server.log("Filesystem erased");
}
//--------------------------------------------------------------------------
function fileExists(fname) {
return _fat.fileExists(fname);
}
//--------------------------------------------------------------------------
function fileSize(fileRef) {
return _fat.get(fileRef).size;
}
//--------------------------------------------------------------------------
function created(fileRef) {
return _fat.get(fileRef).created;
}
//--------------------------------------------------------------------------
function getFileList(orderByDate = false) {
return _fat.getFileList(orderByDate);
}
//--------------------------------------------------------------------------
function getFreeSpace() {
// Smaller files have more overhead than larger files so its impossible to know exactly how much space is free.
// This is a guess that average files are about 20kb or 5 pages.
// local header_size = (SFFS_PAGE_SIZE - SFFS_MAX_FNAME_SIZE - SFFS_MAX_TIMESTAMP_SIZE - SFFS_HEADER_SIZE);
// local data_size = (SFFS_PAGE_SIZE - SFFS_HEADER_SIZE)
// local page_size_guess = ((header_size + (4 * data_size)) / 5).tointeger();
local stats = _fat.getStats();
const page_size_guess = 4076;
local free = stats.free * page_size_guess;
local freeable = (stats.free + stats.erased) * page_size_guess;
return { free = free, freeable = freeable };
}
//--------------------------------------------------------------------------
function open(fname, mode) {
// Check the mode
if ((mode == "r") && !_fat.fileExists(fname)) {
throw format("Can't open '%s' for reading, not found", fname);
} else if ((mode == "w") && _fat.fileExists(fname)) {
throw format("Can't open '%s' for writing, already exists", fname);
} else if (mode != "r" && mode != "w" && mode != "a") {
throw "Unknown mode: " + mode;
}
// Create a new file pointer from the FAT or a new file
local fileId = _fat.getFileId(fname);
local fileIdx = _nextFileIdx++;
_openFiles[fileIdx] <- fileId;
// Return a new file object
return SPIFlashFileSystem.File(this, fileId, fileIdx, fname, mode);
}
//--------------------------------------------------------------------------
function gc() {
_enable();
// Scan the headers of each page, working out what is in each
local collected = 0;
for (local p = 0; p < _pages; p++) {
local page = _start + (p * SFFS_PAGE_SIZE);
local header = _readPage(page, false);
// server.log(page + " = " + header.status.tostring())
if (header.status == SFFS_STATUS_ERASED || header.status == SFFS_STATUS_BAD) {
_flash.erasesector(page);
_fat.markPage(page, SFFS_STATUS_FREE);
collected++;
}
}
_disable();
// if (collected > 0) server.log("Garbage collected " + collected + " pages");
return collected;
}
//--------------------------------------------------------------------------
function setAutoGc(maxPages) {
// Override the default auto garbage collection settings
if (maxPages != null) _autoGcThreshold = maxPages;
}
//--------------------------------------------------------------------------
function _autoGc() {
// Is it worth gc'ing? If so, start it.
local _fatStats = _fat.getStats();
if (_fatStats.free <= _autoGcThreshold && _fatStats.erased > 0) {
// server.log("Automatically starting garbage collection");
gc();
}
}
//--------------------------------------------------------------------------
function _close(fileId, fileIdx, dirty) {
// We have changes to write to disk
if (dirty) {
local file = _fat.get(fileId);
// Write the last span's size to disk
file.pages.seek(-2, 'e');
local page = file.pages.readn('w') * SFFS_PAGE_SIZE;
file.sizes.seek(-2, 'e');
local size = file.sizes.readn('w');
// server.log(format("The last span of file %d has size %d at addr %d", fileId, size, page))
_writeSize(page, size);
}
// Now drop the file pointer;
delete _openFiles[fileIdx];
// Auto garbage collect if required
if (_openFiles.len() == 0 && _autoGcThreshold != 0) _autoGc()
}
//--------------------------------------------------------------------------
function _write(fileId, addr, data) {
// Make sure we have a blob
if (typeof data == "string") {
local data_t = blob(data.len());
data_t.writestring(data);
data = data_t;
data.seek(0);
} else if (typeof data != "blob") {
throw "Can only write blobs and strings";
}
// Work out what we know about this file
local file = _fat.get(fileId);
// server.log(format("Writing %d bytes to '%s' at position %d", data.len() - data.tell(), file.fname, addr));
// Write the data to free pages, one page at a time
local writtenToPage = 0, writtenFromData = 0;
while (!data.eos()) {
// If we need a new page
if (addr % SFFS_PAGE_SIZE == 0) {
// Find and record the next page
try {
addr = _fat.getFreePage();
} catch (e) {
// No free pages, try garbage collection and then die
server.error("Out of space, trying gc()")
gc();
addr = _fat.getFreePage();
}
_fat.addPage(file.id, addr)
file.pageCount++;
file.span++;
}
// Write to the page without the size
local info = _writePage(addr, data, file.id, file.span, file.fname);
// If we are in the middle of a page then add the changes
_fat.addSizeToLastSpan(fileId, info.writtenFromData);
// Shuffle the pointers forward
addr += info.writtenToPage;
data.seek(info.writtenFromData, 'c')
// Keep the counters up to date
writtenFromData += info.writtenFromData;
writtenToPage += info.writtenToPage;
// Go back and write the size of the previous page
if (addr % SFFS_PAGE_SIZE == 0) {
// if we are at the end of the page we can just write 0
_writeSize(addr - SFFS_PAGE_SIZE, 0);
}
}
// Update the FAT
_fat.set(fileId, file);
return { writtenFromData = writtenFromData, writtenToPage = writtenToPage, addr = addr };
}
//--------------------------------------------------------------------------
function _read(fileId, start, len = null) {
local file = _fat.get(fileId);
local result = blob();
// Fix the default length to everything
if (len == null) len = file.sizeTotal - start;
// find the initial address
local next = start, togo = len, pos = 0, page = null;
file.pages.seek(0);
file.sizes.seek(0);
while (!file.pages.eos()) {
page = file.pages.readn('w') * SFFS_PAGE_SIZE;
local size_of_page = file.sizes.readn('w');
// Is any of this page in our target
if (next < pos + size_of_page) {
// Read the data
// logger.log("_readPage(0x%02x, true, %d - %d [%d], %d) size_of_page %d", page, next, pos, next-pos, togo, size_of_page);
local data = _readPage(page, true, next - pos, togo);
data.data.seek(0);
result.writeblob(data.data);
// This is the span we have been looking for
// server.log(format("Found start %d on page %d betweem %d and %d. Read %d of %d bytes", next, page/4096, pos, pos + size_of_page, data.data.len(), len))
// Have we got everything?
togo -= data.data.len();
next += data.data.len();
if (togo == 0) break;
} else {
// server.log(format("Skipping to next span, page = %d, next = %d, pos = %d, size_of_page = %d, pos + size_of_page = %d", page/4096, next, pos, size_of_page, pos + size_of_page))
}
// Move forward in the file we are reading
pos += size_of_page;
}
return result;
}
//--------------------------------------------------------------------------
function _scan() {
local mem = imp.getmemoryfree();
local files = {};
local pages = blob(_pages);
_enable();
// Scan the headers of each page, working out what is in each
for (local p = 0; p < _pages; p++) {
local page = _start + (p * SFFS_PAGE_SIZE);
local header = _readPage(page, false);
// server.log(page + " = " + header.status.tostring())
// Record this page's status
pages.writen(header.status, 'b');
if (header.status == SFFS_STATUS_USED) {
// Make a new file entry, if required
if (!(header.id in files)) {
files[header.id] <- { fn = null, pg = {}, sz = {}, cr = 0 }
}
// Add the span to the files
local file = files[header.id];
if (header.fname != null) file.fn = header.fname;
if (header.created != null) file.cr = header.created;
file.pg[header.span] <- page;
file.sz[header.span] <- header.size;
}
if (files.len() > 0) {
// server.log(format("Interim: %d files, %d ram free, %d ram used", files.len(), imp.getmemoryfree(), (mem - imp.getmemoryfree())))
}
}
_disable();
// server.log("Memory used in scan: " + (mem - imp.getmemoryfree()))
return { files = files, pages = pages };
}
//--------------------------------------------------------------------------
function _writeSize(addr, size) {
// server.log(format(" Updating size to %d at addr %d", size, addr))
assert(addr < _end);
assert(addr % SFFS_PAGE_SIZE == 0);
local headerBlob = blob(SFFS_HEADER_SIZE)
headerBlob.writen(0xFFFF, 'w'); // the id
headerBlob.writen(0xFFFF, 'w'); // The span
headerBlob.writen(size, 'w');
// Write it
_enable();
local res = _flash.write(addr, headerBlob); // Verification will fail
_disable();
assert(res == 0);
}
//--------------------------------------------------------------------------
function _writePage(addr, data, id, span, fname, size = 0xFFFF) {
// server.log(format(" Writing span %d for fileId %d at addr %d", span, id, addr))
assert(addr < _end);
local remInPage = SFFS_PAGE_SIZE - (addr % SFFS_PAGE_SIZE);
local remInData = (data == null) ? 0 : data.len() - data.tell();
local writtenFromData = 0;
local writtenToPage = 0;
// Mark the page as used
_fat.markPage(addr, SFFS_STATUS_USED);
if (remInPage == SFFS_PAGE_SIZE) {
// We are the start of a page, so create the header
local headerBlob = blob(SFFS_HEADER_SIZE)
headerBlob.writen(id, 'w');
headerBlob.writen(span, 'w');
headerBlob.writen(size, 'w');
if (span == 0) {
// Write the time stamp
headerBlob.writen(time(), 'i');
// Truncate and write the filename
if (fname.len() > SFFS_MAX_FNAME_SIZE) {
fname = fname.slice(0, SFFS_MAX_FNAME_SIZE);
}
headerBlob.writen(fname.len(), 'b');
if (fname.len() > 0) {
headerBlob.writestring(fname);
}
}
// Write the header
_enable();
local res = _flash.write(addr, headerBlob, SFFS_SPIFLASH_VERIFY);
_disable();
assert(res == 0);
// Record how much we have written
local dataToWrite = headerBlob.len();
addr += dataToWrite;
remInPage -= dataToWrite;
writtenToPage += dataToWrite;
}
if (remInData > 0) {
// Work out how much to write - the lesser of the remaining in the page and the remaining in the data
local dataToWrite = (remInData < remInPage) ? remInData : remInPage;
// Write the body
_enable();
local res = _flash.write(addr, data, SFFS_SPIFLASH_VERIFY, data.tell(), data.tell() + dataToWrite);
_disable();
assert(res == 0);
addr += dataToWrite;
remInPage -= dataToWrite;
remInData -= dataToWrite;
writtenFromData += dataToWrite;
writtenToPage += dataToWrite;
}
return {
remInPage = remInPage,
remInData = remInData,
writtenFromData = writtenFromData,
writtenToPage = writtenToPage
};
}
//--------------------------------------------------------------------------
function _readPage(addr, readData = false, from = 0, maxlen = null) {
assert(addr < _end);
_enable();
// Read the header
local headerBlob = _flash.read(addr, SFFS_HEADER_SIZE + SFFS_MAX_TIMESTAMP_SIZE + SFFS_MAX_FNAME_SIZE + 1);
// Parse the header
local headerData = {};
headerData.id <- headerBlob.readn('w');
headerData.span <- headerBlob.readn('w');
headerData.size <- headerBlob.readn('w');
headerData.fname <- null;
headerData.created <- null;
if (headerData.span == 0) {
// Read the created timestamp
headerData.created = headerBlob.readn('i');
// Read the file name
local fnameLen = headerBlob.readn('b');
if (fnameLen > 0 && fnameLen <= SFFS_MAX_FNAME_SIZE) {
headerData.fname = headerBlob.readstring(fnameLen);
}
}
// Correct the size
local maxSize = SFFS_PAGE_SIZE - headerBlob.tell();
if ((headerData.span != 0xFFFF) && (headerData.size == 0 || headerData.size > maxSize)) {
headerData.size = maxSize;
}
headerData.eof <- headerBlob.tell() + headerData.size;
// Read the data if required
if (readData) {
local dataOffset = headerBlob.tell();
if (from + maxlen > headerData.size) maxlen = headerData.size - from;
headerData.data <- _flash.read(addr + dataOffset + from, maxlen);
}
_disable();
// Check the results
headerData.status <- SFFS_STATUS_BAD;
if (headerData.id == 0xFFFF && headerData.span == 0xFFFF && headerData.size == 0xFFFF && headerData.fname == null) {
// This is a unwritten page
headerData.status = SFFS_STATUS_FREE;
} else if (headerData.id == 0 && headerData.span == 0 && headerData.size == 0 && headerData.fname == null) {
// This is a erased page
headerData.status = SFFS_STATUS_ERASED;
} else if (headerData.id > 0 && headerData.id < 0xFFFF && headerData.span == 0 && headerData.size < 0xFFFF && headerData.fname != null) {
// This is a header page (span = 0)
headerData.status = SFFS_STATUS_USED;
} else if (headerData.id > 0 && headerData.id < 0xFFFF && headerData.span > 0 && headerData.span < 0xFFFF && headerData.fname == null) {
// This is a normal page
headerData.status = SFFS_STATUS_USED;
} else {
// server.log("Reading at " + addr + " => " + Utils.logObj(headerData))
}
return headerData;
}
//--------------------------------------------------------------------------
function _enable() {
if (_enables++ == 0) {
_flash.enable();
}
}
//--------------------------------------------------------------------------
function _disable() {
if (--_enables <= 0) {
_enables = 0;
_flash.disable();
}
}
//--------------------------------------------------------------------------
function _getSectorFromAddr(page) {
return page - (page % SFFS_SECTOR_SIZE);
}
}
class SPIFlashFileSystem.FAT {
_filesystem = null;
_names = null;
_pages = null;
_sizes = null;
_spans = null;
_creates = null;
_map = null;
_nextId = 1;
//--------------------------------------------------------------------------
constructor(filesystem, files = null, map = null) {
_filesystem = filesystem;
if (typeof files == "integer" && map == null) {
// Make a new, empty page map
_map = blob(files);
for (local i = 0; i < _map.len(); i++) _map[i] = SFFS_STATUS_FREE;
files = null;
} else {
// Store the page map supplied
_map = map;
}
// Mapping of fileId to pages, sizes and spanIds
_pages = {};
_sizes = {};
_spans = {};
_creates = {};
// Mapping of filename to fileId
_names = {};
// Pull the file details out and make a more efficient FAT
if (files != null) {
foreach (fileId,file in files) {
if (file.fn == null) {
file.fn = "recovered_" + math.rand();
logger.error("Recovered bad filename at Idx " + fileId + ": " + file.fn);
}
// Save the filename
_names[file.fn] <- fileId;
// Save the create date
_creates[fileId] <- file.cr;
// Work out the highest spanId
_spans[fileId] <- -1;
foreach (span,page in file.pg) {
if (span > _spans[fileId]) {
_spans[fileId] = span;
}
}
// Save the pages as a single blob
_pages[fileId] <- blob(file.pg.len() * 2);
local pages = pagesOrderedBySpan(file.pg);
foreach (page in pages) {
_pages[fileId].writen(page / SFFS_PAGE_SIZE, 'w');
}
pages = null;
// Save the sizes
_sizes[fileId] <- blob(file.pg.len() * 2);
local sizes = pagesOrderedBySpan(file.sz);
foreach (size in sizes) {
_sizes[fileId].writen(size, 'w');
}
sizes = null;
// Save the file id
if (fileId >= _nextId) {
_nextId = fileId + 1;
}
}
}
}
//--------------------------------------------------------------------------
function describe() {
server.log(format("FAT contained %d files", _names.len()))
foreach (fname,id in _names) {
server.log(format(" File: %s, spans: %d, bytes: %d, created: %d", fname, getPageCount(id), get(id).sizeTotal, get(id).created))
}
}
//--------------------------------------------------------------------------
function getFileList(orderByDate = false) {
local list = [];
foreach (fname,id in _names) {
list.push(fname);
}
// Order the files
if (orderByDate) {
list.sort(function(a, b) { return get(a).created <=> get(b).created }.bindenv(this));
} else {
list.sort();
}
return list;
}
//--------------------------------------------------------------------------
function get(fileRef) {
// Convert the file to an id
local fileId = null, fname = null;
if (typeof fileRef == "string") {
fname = fileRef;
if (fname in _names) {
fileId = _names[fname];
}
} else {
fileId = fileRef;
foreach (filename,id in _names) {
if (fileId == id) {
fname = filename;
break;
}
}
}
// Check the file is valid
if (fileId == null || fname == null) throw "Invalid file reference: " + fileRef;
// Add up the sizes
local sizeTotal = 0, size = 0;
_sizes[fileId].seek(0);
while (!_sizes[fileId].eos()) {
sizeTotal += _sizes[fileId].readn('w');
}
// Return the file entry
return {
id = fileId,
fname = fname,
span = _spans[fileId],
pages = _pages[fileId],
pageCount = _pages[fileId].len() / 2,
sizes = _sizes[fileId],
sizeTotal = sizeTotal,
created = _creates[fileId]
};
}
//--------------------------------------------------------------------------
function set(fileId, file) {
_spans[fileId] = file.span;
}
//--------------------------------------------------------------------------
function getFileId(filename) {
// Check the file is valid
if (!fileExists(filename)) {
// Create a new file
_names[filename] <- _nextId;
_pages[_nextId] <- blob();
_sizes[_nextId] <- blob();
_spans[_nextId] <- -1;
_creates[_nextId] <- 0;
_nextId = (_nextId + 1) % 65535 + 1; // 1 ... 64k-1
}
return (filename in _names) ? _names[filename] : null;
}
//--------------------------------------------------------------------------
function fileExists(fileRef) {
// Check the file is valid
return ((fileRef in _pages) || (fileRef in _names));
}
//--------------------------------------------------------------------------
function getFreePage() {
// Find a random next free page
local map = _map.tostring();
local randStart = math.rand() % _map.len();
local next = map.find(SFFS_STATUS_FREE.tochar(), randStart);
if (next == null) {
// Didn't find one the first time, try from the beginning
next = map.find(SFFS_STATUS_FREE.tochar());
}
if (next == null) throw "No free space available";
// server.log("Searching for: " + SFFS_STATUS_FREE.tostring() + ", in: " + Utils.logBin(_map))
return _filesystem.dimensions().start + (next * SFFS_PAGE_SIZE);
}
//--------------------------------------------------------------------------
function markPage(addr, status) {
// Ammend the page map
local i = (addr - _filesystem.dimensions().start) / SFFS_PAGE_SIZE;
_map[i] = status;
}
//--------------------------------------------------------------------------
function getStats() {
local stats = { free = 0, used = 0, erased = 0, bad = 0 };
local map = _map.tostring();
foreach (ch in map) {
switch (ch) {
case SFFS_STATUS_FREE: stats.free++; break;
case SFFS_STATUS_USED: stats.used++; break;
case SFFS_STATUS_ERASED: stats.erased++; break;
case SFFS_STATUS_BAD: stats.bad++; break;
}
}
return stats;
}
//--------------------------------------------------------------------------
function addPage(fileId, page) {
// Append the page
get(fileId).pages.writen(page / SFFS_PAGE_SIZE, 'w')
get(fileId).sizes.writen(0, 'w');
}
//--------------------------------------------------------------------------
function addSizeToLastSpan(fileId, bytes) {
// Read the last span's size, add the value and rewrite it
local sizes = get(fileId).sizes;
sizes.seek(-2, 'e');
local size = sizes.readn('w') + bytes
sizes.seek(-2, 'e');
sizes.writen(size, 'w');
// server.log(format(" Adding %d bytes to last span = %d", bytes, size))
}
//--------------------------------------------------------------------------
function getPageCount(fileRef) {
return get(fileRef).pages.len() / 2;
}
//--------------------------------------------------------------------------
function forEachPage(fileRef, callback) {
// Find the pages
local pages = get(fileRef).pages;
// Loop through the pages, calling the callback for each one
pages.seek(0);
while (!pages.eos()) {
local page = pages.readn('w') * SFFS_PAGE_SIZE;
callback(page);
}
}
//--------------------------------------------------------------------------
function removeFile(fname) {
// Check the file is valid
if (!fileExists(fname)) throw "Invalid file reference";
// Convert the file to an id
local id = _names[fname];
// Remove them both
delete _names[fname];
delete _pages[id];
delete _sizes[id];
delete _spans[id];
delete _creates[id];
}
//--------------------------------------------------------------------------
function pagesOrderedBySpan(pages) {
// Load the table contents into an array
local interim = [];
foreach (s,p in pages) {
interim.push({ s = s, p = p });
}
// Sort the array by the span
interim.sort(function(first, second) {
return first.s <=> second.s;
});
// Write them to a final array without the key
local result = [];
foreach (i in interim) {
result.push(i.p);
}
return result;
}
}
class SPIFlashFileSystem.File {
_filesystem = null;
_fileIdx = null;
_fileId = null;
_fname = null;
_mode = null;
_pos = 0;
_wpos = 0;
_waddr = 0;
_dirty = false;
//--------------------------------------------------------------------------
constructor(filesystem, fileId, fileIdx, fname, mode) {
_filesystem = filesystem;
_fileIdx = fileIdx;
_fileId = fileId;
_fname = fname;
_mode = mode;
}
//--------------------------------------------------------------------------
function close() {
return _filesystem._close(_fileId, _fileIdx, _dirty);
}
//--------------------------------------------------------------------------
function seek(pos) {
// Set the new pointer position
_pos = pos;
return this;
}
//--------------------------------------------------------------------------
function tell() {
return _pos;
}
//--------------------------------------------------------------------------
function eof() {
return _pos == _filesystem._fat.get(_fileId).sizeTotal;
}
//--------------------------------------------------------------------------
function size() {
return _filesystem._fat.get(_fileId).sizeTotal;
}
//--------------------------------------------------------------------------
function created() {
return _filesystem._fat.get(_fileId).created;
}
//--------------------------------------------------------------------------
function read(len = null) {
local data = _filesystem._read(_fileId, _pos, len);
_pos += data.len();
return data;
}
//--------------------------------------------------------------------------
function write(data) {
if (_mode == "r") throw "Can't write - file mode is 'r'";
local info = _filesystem._write(_fileId, _waddr, data);
_wpos += info.writtenFromData;
_waddr = info.addr;
_dirty = true;
return info.writtenToPage;
}
}
Raw
tests.nut
//==============================================================================
// return;
// Initialise the hardware
if (true) {
local SFFS_START_ADDR = 0;
local SFFS_END_ADDR = 0x100 * SFFS_SECTOR_SIZE;
imp.setpowersave(true);
if ("pinF" in hardware) {
hardware.pinE.configure(DIGITAL_OUT, 1); // Red
hardware.pinF.configure(DIGITAL_OUT, 1); // Green
hardware.pinK.configure(DIGITAL_OUT, 1); // Blue
hardware.pinM.configure(DIGITAL_OUT, 0); // White
squiffy <- SPIFlashFileSystem(SFFS_START_ADDR, SFFS_END_ADDR);
} else {
flash <- SPIFlash(hardware.spi189, hardware.pin6, 128);
flash.configure();
squiffy <- SPIFlashFileSystem(SFFS_START_ADDR, SFFS_END_ADDR, flash);
}
}
// Power down after 5 minutes to save batteries
if (true) {
imp.wakeup(60*5, function() {
server.sleepfor(1000000);
})
}
// Turn off the blinkup LED
if (true) {
imp.wakeup(5, function() {
imp.enableblinkup(false);
})
}
// ** Erase the entire file system **
if (false) {
server.log("========[ Erasing ]=========")
try {
squiffy.eraseAll();
} catch (e) {
server.error("Failed: " + e);
}
server.log("========[ Done ]=========\n\n")
}
// Initialise the file system
if (true) {
server.log("========[ Initialising ]=========")
try {
local start = hardware.millis();
squiffy.init();
squiffy._fat.describe();
server.log("Init time: " + (hardware.millis() - start))
} catch (e) {
server.error("Failed: " + e);
}
server.log("========[ Done ]=========\n\n")
}
// New code test rig
if (false) {
server.log("========[ Testing stuff ]=========")
local start = hardware.millis();
try {
// Write
if (!squiffy.fileExists("giant.bin")) {
local bigblob = blob(1000);
for (local i = 0; i < bigblob.len(); i++) bigblob[i] = i % 100;
local fileobj = squiffy.open("giant.bin", "w");
for (local i = 0; i < 6; i++) {
bigblob.seek(0);
fileobj.write(bigblob);
}
fileobj.close();
}
// Read
local fileobj = squiffy.open("giant.bin", "r");
fileobj.seek(5000);
local data = fileobj.read(10);
if (data.len() > 0) {
server.log("READ: " + data.len() + " bytes: " + Utils.logBin(data));
} else {
server.log("READ: Failed");
}
fileobj.close();
} catch (e) {
server.error("Failed: " + e);
}
server.log("Test time: " + (hardware.millis() - start))
server.log("========[ Done ]=========\n\n")
}
// Write lots of files
if (true) {
server.log("========[ Writing, reading and erasing lots of files ]=========")
try {
local t_open = 0, t_write = 0, t_close = 0, t_read = 0, t_erase = 0, t_start = 0;
for (local i = 0; i < 150; i++) {
local fname = format("test_%02x.txt", i);
if (squiffy.fileExists(fname)) squiffy.eraseFile(fname);
// Write
t_start = hardware.millis();
local fileobj = squiffy.open(fname, "w");
t_open += (hardware.millis() - t_start);
t_start = hardware.millis();
fileobj.write(fname)
t_write += (hardware.millis() - t_start);
t_start = hardware.millis();
fileobj.close();
t_close += (hardware.millis() - t_start);
// Read
local fileobj = squiffy.open(fname, "r");
t_start = hardware.millis();
local data = fileobj.read();
t_read += (hardware.millis() - t_start);
if (data.tostring() != fname) server.error(format("Data was wrong in '%s': %s (%d)", fname, data.tostring(), data.len()));
fileobj.close();
// Erase
t_start = hardware.millis();
squiffy.eraseFile(fname)
t_erase += (hardware.millis() - t_start);
}
server.log("= Time spent opening: " + t_open + " ms")
server.log("= Time spent writing: " + t_write + " ms")
server.log("= Time spent closing: " + t_close + " ms")
server.log("= Time spent reading: " + t_read + " ms")
server.log("= Time spent erasing: " + t_erase + " ms")
} catch (e) {
server.error("Failed: " + e);
}
server.log("========[ Done ]=========\n\n")
}
// Write a 64k file in 1k writes
if (false) {
server.log("========[ Writing 64k file in 1k chunks ]=========")
try {
local bigblob = blob(1024);
local start = hardware.millis();
local fileobj = squiffy.open("giant.bin", "w");
for (local i = 0; i < 64; i++) {
bigblob.seek(0);
if (i == 63) {
bigblob[1023] = 0x42;
}
fileobj.write(bigblob);
}
fileobj.close();
server.log("Create time: " + (hardware.millis() - start))
local fileobj = squiffy.open("giant.bin", "r");
fileobj.seek(64*1024-2);
server.log("Seeking to: " + fileobj.tell());
local data = fileobj.read()
local ok = (data[0] == 0x0 && data[1] == 0x42);
server.log(format("Giant file (%d bytes), content is: %s", fileobj.size(), ok ? "ok": "NOT OK"));
fileobj.close();
local start = hardware.millis();
squiffy.eraseFile("giant.bin");
server.log("Erase time: " + (hardware.millis() - start))
} catch (e) {
server.error("Failed: " + e);
}
server.log("========[ Done ]=========\n\n")
}
// Opening a read pointer multiple times
if (false) {
server.log("========[ Opening multiple read files ]=========")
try {
local fname = "multiple.txt";
if (squiffy.fileExists(fname)) squiffy.eraseFile(fname);
local fileobj0 = squiffy.open(fname, "a");
fileobj0.write("Hello, world. ");
local fileobj1 = squiffy.open(fname, "r");
local fileobj2 = squiffy.open(fname, "r");
local string1 = fileobj1.seek(7).read(5).tostring();
local string2 = fileobj2.read(5).tostring();
fileobj0.write("Goodbye. ");
local string3 = fileobj1.seek(14).read(7).tostring();
server.log(string1 == "world");
server.log(string2 == "Hello");
server.log(string3 == "Goodbye");
fileobj0.close();
fileobj1.close();
fileobj2.close();
} catch (e) {
server.error("Failed: " + e);
}
server.log("========[ Done ]=========\n\n")
}
// Appending to an existing file
if (false) {
// NOTE: I am not 100% sure this correctectly handles the pgsIdx blob. Worth checking a bit deeper than this example.
server.log("========[ Appending to an existing file ]=========")
try {
local fileobj = squiffy.open("test3.txt", "a");
fileobj.write("Append 1. ");
fileobj.close();
local fileobj = squiffy.open("test3.txt", "a");
fileobj.write("Append 2. ");
fileobj.close();
local fileobj = squiffy.open("test3.txt", "a");
fileobj.write("Append 3. ");
fileobj.close();
} catch (e) {
server.error("Failed: " + e);
}
try {
local fileobj = squiffy.open("test3.txt", "r");
local data = fileobj.read();
server.log("Length: " + data.len() + " == " + fileobj.size())
server.log("Data: " + data.tostring());
fileobj.close();
} catch (e) {
server.error("Failed: " + e);
}
server.log("========[ Done ]=========\n\n")
}
// Try to erase an open file
if (false) {
server.log("========[ Erasing an open file ]=========")
try {
if (!squiffy.fileExists("eraseMe.txt")) {
local fileobj = squiffy.open("eraseMe.txt", "w");
fileobj.write("Some data");
fileobj.close();
}
local fileobj = squiffy.open("eraseMe.txt", "r");
try {
squiffy.eraseFile("eraseMe.txt");
} catch (e) {
server.log("Success: " + e);
}
fileobj.close();
} catch (e) {
server.error("Failed: " + e);
}
server.log("========[ Done ]=========\n\n")
}
// Begin a garbage collection
if (false) {
server.log("========[ Garbage Collection ]=========")
try {
// Keep one file around
server.log("Creating keepMe.bin");
if (squiffy.fileExists("keepMe.bin")) {
// Check the file is intact
local fileobj = squiffy.open("keepMe.bin", "r");
server.log("Are the contents good? " + (fileobj.read().tostring() == "Hello, world."));
fileobj.close();
} else {
// Create the file
local fileobj = squiffy.open("keepMe.bin", "w");
fileobj.write("Hello, world.");
fileobj.close();
}
// Repeatedly write and erase some files
for (local i = 0; i < 10; i++) {
// Make sure we have a file
local fname = format("eraseMe_%02d.bin", i);
if (!squiffy.fileExists(fname)) {
local fileobj = squiffy.open(fname, "w");
fileobj.write("Hello, world.");
fileobj.close();
}
// Make sure the contents are correct
local fileobj = squiffy.open(fname, "r");
if (fileobj.read().tostring() != "Hello, world.") {
server.error("Check of " + fname + " FAILED");
}
fileobj.close();
// Delete every second one
if (i % 2 == 0) {
server.log("Erasing " + fname);
squiffy.eraseFile(fname);
} else {
server.log("Keeping " + fname);
}
}
// Now start the collecting of garbage
squiffy.gc();
} catch (e) {
server.error("Failed: " + e);
}
server.log("========[ Done ]=========\n\n")
}
// Repair the lookup tables
if (false) {
server.log("========[ Repair ]=========")
try {
squiffy.repair(function(file) {
server.log(format("Found file '%s' (%d) with size %d bytes", file.fname, file.id, file.size));
});
} catch (e) {
server.error("Failed: " + e);
}
server.log("========[ Done ]=========\n\n")
}
// Reinitialise the fat
if (true) {
server.log("========[ Reinitialising ]=========")
local start = hardware.millis();
try {
squiffy.init();
// squiffy._fat.describe();
} catch (e) {
server.error("Failed: " + e);
}
server.log("Reinit time: " + (hardware.millis() - start))
server.log("========[ Done ]=========\n\n")
}
server.log("Memory: " + imp.getmemoryfree() + " bytes free")
Raw
utils.nut
//==============================================================================
class Utils {
function logObj(data, tabLevel = 1) {
local dbg = "";
local tabs = "";
for (local t = 0; t < tabLevel; t++) tabs += "\t";
if (typeof data == "table" || typeof data == "array") {
dbg += format("<%s>\n", typeof data);
foreach (k,v in data) {
dbg += format("%s%s => %s", tabs, k.tostring(), logObj(v, tabLevel+1));
}
dbg += format("</%s>\n", typeof data);
} else if (typeof data == "blob") {
dbg += format("<blob> %d bytes </blob>\n", data.len());
} else if (typeof data == "string") {
dbg += format("<string> %d characters </string>\n", data.len());
} else if (data == null) {
dbg += format("<null />\n");
} else {
dbg += format("<%s> %s </%s>\n", typeof data, data.tostring(), typeof data);
}
return dbg;
}
function logBin(data, start = null, end = null) {
if (start == null) start = 0;
if (end == null) end = data.len();
local dbg = "";
if (data == null) {
dbg = "<null>";
} else {
for (local j = start; j < end; j++) {
dbg += format("%02x ", data[j]);
}
}
return dbg;
}
function join(data, separator = ", ") {
local dbg = "";
foreach (datum in data) {
dbg += datum.tostring() + separator;
}
return dbg.slice(0, -separator.len());
}
function joinKey(data, separator = ", ") {
local dbg = "";
foreach (key,datum in data) {
dbg += key.tostring() + separator;
}
return dbg.slice(0, -separator.len());
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment