From 355b28838689be3d87800eefe91055cff9e177b9 Mon Sep 17 00:00:00 2001 From: Hongzhen Luo Date: Tue, 19 Nov 2024 11:49:25 +0800 Subject: [PATCH 1/3] [EROFS]: refactor: reorganize code Reorganize the code to make the structure clearer. Specifically: - erofs_common.*: contains some common content used internally by EROFS, such as I/O controllers and constants; - liberofs.*: contains the content for creating EROFS filesystem images; - erofs_fs.*: contains the content for building the EROFS filesystem from EROFS filesystem images and performing operations on the filesystem. Additional changes: - rename `min` to `erofs_min` to avoid conflicts with `std::min`; - declare erofs_check_fs() and erofs_create_fs() as extern in comm_func.cpp, as including erofs_fs.h at this point leads to conflicts. Subsequent patches will clean up erofs_fs.h to allow it to be included by third parties (hide content related to liberofs to avoid conflicts). No other changes were made. Signed-off-by: Hongzhen Luo --- src/overlaybd/tar/erofs/erofs_common.cpp | 390 ++++++++++ src/overlaybd/tar/erofs/erofs_common.h | 98 +++ src/overlaybd/tar/erofs/erofs_fs.cpp | 428 +++++++++++ src/overlaybd/tar/erofs/erofs_fs.h | 88 +++ src/overlaybd/tar/erofs/liberofs.cpp | 875 +---------------------- src/overlaybd/tar/erofs/liberofs.h | 3 - src/tools/comm_func.cpp | 4 +- 7 files changed, 1008 insertions(+), 878 deletions(-) create mode 100644 src/overlaybd/tar/erofs/erofs_common.cpp create mode 100644 src/overlaybd/tar/erofs/erofs_common.h create mode 100644 src/overlaybd/tar/erofs/erofs_fs.cpp create mode 100644 src/overlaybd/tar/erofs/erofs_fs.h diff --git a/src/overlaybd/tar/erofs/erofs_common.cpp b/src/overlaybd/tar/erofs/erofs_common.cpp new file mode 100644 index 00000000..74ba8c8e --- /dev/null +++ b/src/overlaybd/tar/erofs/erofs_common.cpp @@ -0,0 +1,390 @@ +/* + Copyright The Overlaybd Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "erofs_common.h" +#include + +ssize_t ErofsCache::write_sector(u64 addr, char *buf) +{ + struct liberofs_inmem_sector *sector; + + if (addr & (SECTOR_SIZE-1)) { + LOG_ERROR("Invalid addr, should be aligned to SECTOR_SIZE."); + return -1; + } + if (caches.find(addr) != caches.end()) { + sector = caches[addr]; + memcpy(sector->data, buf, SECTOR_SIZE); + dirty.insert(addr); + } else { + if (caches.size() == capacity) { + auto it = caches.begin(); + if (dirty.find(it->first) != dirty.end()) { + if (file->pwrite(it->second->data, SECTOR_SIZE, it->first) + != SECTOR_SIZE) + { + LOG_ERROR("Fail to write sector %lld.", it->first); + return -EIO; + } + dirty.erase(it->first); + } + sector = it->second; + caches.erase(it); + } else { + sector = (struct liberofs_inmem_sector*)malloc( + sizeof(struct liberofs_inmem_sector)); + if (!sector) + return -ENOMEM; + } + memcpy(sector->data, buf, SECTOR_SIZE); + caches[addr] = sector; + dirty.insert(addr); + } + return SECTOR_SIZE; +} + +ssize_t ErofsCache::read_sector(u64 addr, char *buf) +{ + struct liberofs_inmem_sector *sector; + + if (addr & (SECTOR_SIZE - 1)) { + LOG_ERROR("Invalid addr, should be aligned to SECTOR_SIZE."); + return -1; + } + + if (caches.find(addr) != caches.end()) { + sector = caches[addr]; + memcpy(buf, sector->data, SECTOR_SIZE); + } else { + if (caches.size() == capacity) { + auto it = caches.begin(); + if (dirty.find(it->first) != dirty.end()) { + sector = it->second; + if (file->pwrite(sector->data, SECTOR_SIZE, it->first) + != SECTOR_SIZE) + { + LOG_ERROR("Fail to write sector %lld.", it->first); + return -EIO; + } + dirty.erase(it->first); + } + sector = it->second; + caches.erase(it); + } else { + sector = (struct liberofs_inmem_sector*)malloc( + sizeof(struct liberofs_inmem_sector)); + if (!sector) + return -ENOMEM; + } + if (file->pread(sector->data, SECTOR_SIZE, addr) != SECTOR_SIZE) { + LOG_ERROR("Fail to read sector %lld", addr); + return -EIO; + } + caches[addr] = sector; + memcpy(buf, sector->data, SECTOR_SIZE); + } + return SECTOR_SIZE; +} + +int ErofsCache::flush() +{ + for (auto it = caches.begin(); it != caches.end();) { + if (dirty.find(it->first) != dirty.end()) { + if (file->pwrite(it->second->data, SECTOR_SIZE, it->first) + != SECTOR_SIZE) + { + LOG_ERROR("Fail to flush sector %lld.\n", it->first); + return -1; + } + dirty.erase(it->first); + } + free(it->second); + it = caches.erase(it); + } + return dirty.size() != 0 || caches.size() != 0; +} + +/* + * Helper function for reading from the photon file, since + * the photon file requires reads to be 512-byte aligned. + */ +ssize_t erofs_read_photon_file(void *buf, u64 offset, size_t len, + ErofsCache *cache) +{ + + ssize_t read; + u64 start, end; + char extra_buf[SECTOR_SIZE]; + u64 i, j; + + start = round_down_blk(offset); + end = round_up_blk(offset + len); + read = 0; + + /* we use the extra_buf to store the first sector or last sector when: + * - start != offset + * or + * - end != offset + len + */ + if (start != offset || end != offset + len) { + i = start == offset ? start : start + SECTOR_SIZE; + j = end == offset + len ? end : end - SECTOR_SIZE; + + /* read the first sector */ + if (i != start) { + if (cache->read_sector(start, extra_buf) != SECTOR_SIZE) { + LOG_ERROR("Fail to read start sector."); + return -1; + } + memcpy((char*)buf, extra_buf + offset - start, + erofs_min(start + SECTOR_SIZE - offset,len)); + read += erofs_min(start + SECTOR_SIZE - offset,len); + } + + /* read the last sector and avoid re-reading the same sector as above */ + if (j != end && (i == start || end - start > SECTOR_SIZE)) { + if (cache->read_sector(end - SECTOR_SIZE, + extra_buf) != SECTOR_SIZE) + { + LOG_ERROR("Fail to read start sector."); + return -1; + } + memcpy((char*)buf + end - SECTOR_SIZE - offset, extra_buf, + len + offset + SECTOR_SIZE - end); + read += len + offset + SECTOR_SIZE - end; + } + + for (u64 addr = i; addr < j; addr += SECTOR_SIZE) { + if (cache->read_sector(addr, + addr - offset + (char*)buf) != SECTOR_SIZE) + { + LOG_ERROR("Fail to read sector %lld in read_photo_file.\n", i); + return -1; + } + read += SECTOR_SIZE; + } + } else { + /* if read request is sector-aligned, we use the original buffer */ + for (u64 i = start; i < end; i += SECTOR_SIZE) { + if (cache->read_sector(i, (char*)buf + i - start) != SECTOR_SIZE) { + LOG_ERROR("Fail to read start sector."); + return -1; + } + read += SECTOR_SIZE; + } + } + + return read; +} + +/* + * Helper function for writing to a photon file. + */ +ssize_t erofs_write_photon_file(const void *buf, u64 offset, + size_t len, ErofsCache *cache) + { + ssize_t write; + u64 start, end; + char extra_buf[SECTOR_SIZE]; + u64 i, j; + + start = round_down_blk(offset); + end = round_up_blk(offset + len); + write = 0; + + if (start != offset || end != offset + len) { + i = start == offset ? start : start + SECTOR_SIZE; + j = end == offset + len ? end : end - SECTOR_SIZE; + + if (i != start) { + if (cache->read_sector(start, extra_buf) != SECTOR_SIZE) { + LOG_ERROR("Fail to read sector %lld.\n", start); + return -1; + } + memcpy(extra_buf + offset - start, buf, + erofs_min(start + SECTOR_SIZE - offset, len)); + if (cache->write_sector(start, extra_buf) != SECTOR_SIZE) { + LOG_ERROR("Fail to write sector %lld\n", start); + return -1; + } + write += erofs_min(start + SECTOR_SIZE - offset, len); + } + + if (j != end && (i == start || end - start > SECTOR_SIZE)) { + if (cache->read_sector(end - SECTOR_SIZE, extra_buf) + != SECTOR_SIZE) + { + LOG_ERROR("Fail to read sector %lld.", end - SECTOR_SIZE); + return -1; + } + memcpy(extra_buf, (char*)buf + end - SECTOR_SIZE - offset, + offset + len + SECTOR_SIZE - end); + if (cache->write_sector(end - SECTOR_SIZE, extra_buf) + != SECTOR_SIZE) + { + LOG_ERROR("Fail to write sector %lld.", end - SECTOR_SIZE); + return -1; + } + write += offset + len + SECTOR_SIZE - end; + } + + for (u64 addr = i; addr < j; addr += SECTOR_SIZE) { + if (cache->write_sector(addr, (char*)buf + addr - offset) + != SECTOR_SIZE) + { + LOG_ERROR("Fail to write sector %lld.", addr); + return -1; + } + write += SECTOR_SIZE; + } + } else { + for (u64 addr = start; addr < end; addr += SECTOR_SIZE) { + if (cache->write_sector(addr, (char *)buf + addr - start) + != SECTOR_SIZE) + { + LOG_ERROR("Fail to write sector %lld.", addr); + return -1; + } + write += SECTOR_SIZE; + } + } + + return write; +} + +/* I/O control for target */ +ssize_t erofs_target_pread(struct erofs_vfile *vf, void *buf, u64 offset, + size_t len) +{ + struct liberofs_file *target_file = + reinterpret_cast(vf->ops); + + if (!target_file) + return -EINVAL; + if (erofs_read_photon_file(buf, offset, len, target_file->cache) + != (ssize_t)len) + return -1; + + return len; +} + +ssize_t erofs_target_pwrite(struct erofs_vfile *vf, const void *buf, + u64 offset, size_t len) +{ + struct liberofs_file *target_file = + reinterpret_cast(vf->ops); + + if (!target_file) + return -EINVAL; + if (!buf) + return -EINVAL; + + return erofs_write_photon_file(buf, offset, len, target_file->cache); +} + +int erofs_target_fsync(struct erofs_vfile *vf) +{ + struct liberofs_file *target_file = + reinterpret_cast(vf->ops); + + if (!target_file) + return -EINVAL; + return target_file->cache->flush(); +} + +int erofs_target_fallocate(struct erofs_vfile *vf, u64 offset, + size_t len, bool pad) +{ + static const char zero[4096] = {0}; + ssize_t ret; + + while (len > 4096) { + ret = erofs_target_pwrite(vf, zero, offset, 4096); + if (ret) + return ret; + len -= 4096; + offset += 4096; + } + ret = erofs_target_pwrite(vf, zero, offset, len); + if (ret != (ssize_t)len) { + return -1; + } + return 0; +} + +int erofs_target_ftruncate(struct erofs_vfile *vf, u64 length) +{ + return 0; +} + +ssize_t erofs_target_read(struct erofs_vfile *vf, void *buf, size_t len) +{ + return -EROFS_UNIMPLEMENTED; +} + +off_t erofs_target_lseek(struct erofs_vfile *vf, u64 offset, int whence) +{ + return -EROFS_UNIMPLEMENTED; +} + +/* I/O control for source */ +ssize_t erofs_source_pread(struct erofs_vfile *vf, void *buf, u64 offset, + size_t len) +{ + return -EROFS_UNIMPLEMENTED; +} + +ssize_t erofs_source_pwrite(struct erofs_vfile *vf, const void *buf, + u64 offset, size_t len) +{ + return -EROFS_UNIMPLEMENTED; +} + +int erofs_source_fsync(struct erofs_vfile *vf) +{ + return -EROFS_UNIMPLEMENTED; +} + +int erofs_source_fallocate(struct erofs_vfile *vf, + u64 offset, size_t len, bool pad) +{ + return -EROFS_UNIMPLEMENTED; +} + +int erofs_source_ftruncate(struct erofs_vfile *vf, u64 length) +{ + return -EROFS_UNIMPLEMENTED; +} + +ssize_t erofs_source_read(struct erofs_vfile *vf, void *buf, + size_t bytes) +{ + struct liberofs_file *source_file = + reinterpret_cast(vf->ops); + + if (!source_file) + return -EINVAL; + return source_file->file->read(buf, bytes); +} + +off_t erofs_source_lseek(struct erofs_vfile *vf, u64 offset, int whence) +{ + struct liberofs_file *source_file = + reinterpret_cast(vf->ops); + if (!source_file) + return -EINVAL; + return source_file->file->lseek(offset, whence); +} diff --git a/src/overlaybd/tar/erofs/erofs_common.h b/src/overlaybd/tar/erofs/erofs_common.h new file mode 100644 index 00000000..581add6d --- /dev/null +++ b/src/overlaybd/tar/erofs/erofs_common.h @@ -0,0 +1,98 @@ +/* + Copyright The Overlaybd Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// common contents used internally by erofs + +#pragma once + +#include "erofs/io.h" +#include +#include +#include + +#define SECTOR_SIZE 512ULL +#define SECTOR_BITS 9 + +#define round_down_blk(addr) ((addr) & (~(SECTOR_SIZE - 1))) +#define round_up_blk(addr) (round_down_blk((addr) + SECTOR_SIZE - 1)) +#define erofs_min(a, b) (a) < (b) ? (a) : (b) + +#define EROFS_ROOT_XATTR_SZ (16 * 1024) + +#define EROFS_UNIMPLEMENTED 1 + +#define EROFS_UNIMPLEMENTED_FUNC(ret_type, cls, func, ret) \ +ret_type cls::func { \ + return ret; \ +} + +struct liberofs_inmem_sector { + char data[SECTOR_SIZE]; +}; + +class ErofsCache { +public: + ErofsCache(photon::fs::IFile *file, unsigned long int capacity): + file(file), capacity(capacity) + {} + ~ErofsCache() {} + ssize_t write_sector(u64 addr, char *buf); + ssize_t read_sector(u64 addr, char *buf); + int flush(); +public: + photon::fs::IFile *file; + long unsigned int capacity; + std::mapcaches; + std::set dirty; +}; + +struct liberofs_file { + struct erofs_vfops ops; + photon::fs::IFile *file; + ErofsCache *cache; +}; + +/* helper functions for reading and writing photon files */ +extern ssize_t erofs_read_photon_file(void *buf, u64 offset, size_t len, + ErofsCache *cache); +extern ssize_t erofs_write_photon_file(const void *buf, u64 offset, + size_t len, ErofsCache *cache); + +/* I/O controllers for target */ +extern ssize_t erofs_target_pread(struct erofs_vfile *vf, void *buf, + u64 offset, size_t len); +extern ssize_t erofs_target_pwrite(struct erofs_vfile *vf, const void *buf, + u64 offset, size_t len); +extern int erofs_target_fsync(struct erofs_vfile *vf); +extern int erofs_target_fallocate(struct erofs_vfile *vf, u64 offset, + size_t len, bool pad); +extern int erofs_target_ftruncate(struct erofs_vfile *vf, u64 length); +extern ssize_t erofs_target_read(struct erofs_vfile *vf, void *buf, size_t len); +extern off_t erofs_target_lseek(struct erofs_vfile *vf, u64 offset, int whence); + +/* I/O controllers for source */ +extern ssize_t erofs_source_pread(struct erofs_vfile *vf, void *buf, u64 offset, + size_t len); +extern ssize_t erofs_source_pwrite(struct erofs_vfile *vf, const void *buf, + u64 offset, size_t len); +extern int erofs_source_fsync(struct erofs_vfile *vf); +extern int erofs_source_fallocate(struct erofs_vfile *vf, + u64 offset, size_t len, bool pad); +extern int erofs_source_ftruncate(struct erofs_vfile *vf, u64 length); +extern ssize_t erofs_source_read(struct erofs_vfile *vf, void *buf, + size_t bytes); +extern off_t erofs_source_lseek(struct erofs_vfile *vf, u64 offset, + int whence); diff --git a/src/overlaybd/tar/erofs/erofs_fs.cpp b/src/overlaybd/tar/erofs/erofs_fs.cpp new file mode 100644 index 00000000..0102a6fb --- /dev/null +++ b/src/overlaybd/tar/erofs/erofs_fs.cpp @@ -0,0 +1,428 @@ +/* + Copyright The Overlaybd Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "erofs_common.h" +#include "erofs_fs.h" +#include "erofs/inode.h" +#include "erofs/dir.h" +#include +#include + +// ErofsFile +ErofsFile::ErofsFile(ErofsFileSystem *fs): fs(fs) +{ + memset(&inode, 0, sizeof(struct erofs_inode)); +} +photon::fs::IFileSystem *ErofsFile::filesystem() { return fs; } + +struct liberofs_nameidata { + struct erofs_sb_info *sbi; + erofs_nid_t nid; +}; + +static int liberofs_link_path_walk(const char *name, + struct liberofs_nameidata *nd); + +static struct erofs_dirent *liberofs_find_dirent(void *data, const char *name, + unsigned int len, + unsigned int nameoff, + unsigned int maxsize) +{ + struct erofs_dirent *de = (struct erofs_dirent *)data; + const struct erofs_dirent *end = (struct erofs_dirent *)((char *)data + nameoff); + + while (de < end) { + const char *de_name; + unsigned int de_namelen; + + nameoff = le16_to_cpu(de->nameoff); + de_name = (char *)((char *)data + nameoff); + if (de + 1 >= end) + de_namelen = strnlen(de_name, maxsize - nameoff); + else + de_namelen = le16_to_cpu(de[1].nameoff - nameoff); + + if (nameoff + de_namelen > maxsize) { + LOG_ERROR("[erofs] bogus dirent"); + return (struct erofs_dirent *)ERR_PTR(-EINVAL); + } + + if (len == de_namelen && !memcmp(de_name, name, de_namelen)) + return de; + ++de; + } + return NULL; +} + +static int liberofs_namei(struct liberofs_nameidata *nd, const char *name, + unsigned int len) +{ + erofs_nid_t nid = nd->nid; + int ret; + char buf[EROFS_MAX_BLOCK_SIZE]; + struct erofs_sb_info *sbi = nd->sbi; + struct erofs_inode vi = {}; + erofs_off_t offset; + + vi.sbi = sbi; + vi.nid = nid; + ret = erofs_read_inode_from_disk(&vi); + if (ret) + return ret; + + offset = 0; + while (offset < vi.i_size) { + erofs_off_t maxsize = min_t(erofs_off_t, + vi.i_size - offset, erofs_blksiz(sbi)); + struct erofs_dirent *de = (struct erofs_dirent *)buf; + unsigned int nameoff; + + ret = erofs_pread(&vi, buf, maxsize, offset); + if (ret) + return ret; + + nameoff = le16_to_cpu(de->nameoff); + if (nameoff < sizeof(struct erofs_dirent) || + nameoff >= erofs_blksiz(sbi)) + LOG_ERRNO_RETURN(-EINVAL, -EINVAL, "[erofs] invalid nameoff"); + + de = liberofs_find_dirent(buf, name, len, nameoff, maxsize); + if (IS_ERR(de)) + return PTR_ERR(de); + + if (de) { + nd->nid = le64_to_cpu(de->nid); + return 0; + } + offset += maxsize; + } + return -ENOENT; +} + + +static int liberofs_step_into_link(struct liberofs_nameidata *nd, + struct erofs_inode *vi) +{ + char buf[PATH_MAX]; + int err; + + if (vi->i_size > PATH_MAX) + return -EINVAL; + memset(buf, 0, sizeof(buf)); + err = erofs_pread(vi, buf, vi->i_size, 0); + if (err) + return err; + return liberofs_link_path_walk(buf, nd); +} + +static int liberofs_link_path_walk(const char *name, + struct liberofs_nameidata *nd) +{ + struct erofs_inode vi; + erofs_nid_t nid; + const char *p; + int ret; + + if (*name == '/') + nd->nid = nd->sbi->root_nid; + + while (*name == '/') + name ++; + + while (*name != '\0') { + p = name; + do { + ++p; + } while (*p != '\0' && *p != '/'); + + nid = nd->nid; + ret = liberofs_namei(nd, name, p - name); + if (ret) + return ret; + vi.sbi = nd->sbi; + vi.nid = nd->nid; + ret = erofs_read_inode_from_disk(&vi); + if (ret) + return ret; + if (S_ISLNK(vi.i_mode)) { + nd->nid = nid; + ret = liberofs_step_into_link(nd, &vi); + if (ret) + return ret; + } + for (name = p; *name == '/'; ++name) + ; + } + return 0; +} + + +static int do_erofs_ilookup(const char *path, struct erofs_inode *vi) +{ + int ret; + struct liberofs_nameidata nd = {.sbi = vi->sbi}; + + nd.nid = vi->sbi->root_nid; + ret = liberofs_link_path_walk(path, &nd); + if (ret) + return ret; + vi->nid = nd.nid; + return erofs_read_inode_from_disk(vi); +} + +int ErofsFile::fstat(struct stat *buf) +{ + buf->st_mode = inode.i_mode; + buf->st_nlink = inode.i_nlink; + buf->st_size = inode.i_size; + buf->st_blocks = roundup(inode.i_size, erofs_blksiz(inode.sbi)) >> 9; + buf->st_uid = inode.i_uid; + buf->st_gid = inode.i_gid; + buf->st_ctime = inode.i_mtime; + buf->st_mtime = inode.i_mtime; + buf->st_atime = inode.i_mtime; + return 0; +} + +int ErofsFile::fiemap(struct photon::fs::fiemap *map) +{ + photon::fs::fiemap_extent *ext_buf = &map->fm_extents[0]; + struct erofs_map_blocks erofs_map; + int err; + + map->fm_mapped_extents = 0; + erofs_map.index = UINT_MAX; + erofs_map.m_la = 0; + + while (erofs_map.m_la < inode.i_size) { + err = erofs_map_blocks(&inode, &erofs_map, 0); + if (err) + LOG_ERROR_RETURN(err, err, "[erofs] Fail to map erofs blocks"); + ext_buf[map->fm_mapped_extents].fe_physical = erofs_map.m_pa; + ext_buf[map->fm_mapped_extents].fe_length = erofs_map.m_plen; + map->fm_mapped_extents += 1; + erofs_map.m_la += erofs_map.m_llen; + } + return 0; +} + +// ErofsFileSystem +EROFS_UNIMPLEMENTED_FUNC(photon::fs::IFile*, ErofsFileSystem, open(const char *pathname, int flags, mode_t mode), NULL) +EROFS_UNIMPLEMENTED_FUNC(photon::fs::IFile*, ErofsFileSystem, creat(const char *pathname, mode_t mode), NULL) +EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, mkdir(const char *pathname, mode_t mode), -EROFS_UNIMPLEMENTED) +EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, rmdir(const char *pathname), -EROFS_UNIMPLEMENTED) +EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, symlink(const char *oldname, const char *newname), -EROFS_UNIMPLEMENTED) +EROFS_UNIMPLEMENTED_FUNC(ssize_t, ErofsFileSystem, readlink(const char *path, char *buf, size_t bufsiz), -EROFS_UNIMPLEMENTED) +EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, link(const char *oldname, const char *newname), -EROFS_UNIMPLEMENTED) +EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, rename(const char *oldname, const char *newname), -EROFS_UNIMPLEMENTED) +EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, unlink(const char *filename), -EROFS_UNIMPLEMENTED) +EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, chmod(const char *pathname, mode_t mode), -EROFS_UNIMPLEMENTED) +EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, chown(const char *pathname, uid_t owner, gid_t group), -EROFS_UNIMPLEMENTED) +EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, lchown(const char *pathname, uid_t owner, gid_t group), -EROFS_UNIMPLEMENTED) +EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, statfs(const char *path, struct statfs *buf), -EROFS_UNIMPLEMENTED) +EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, statvfs(const char *path, struct statvfs *buf), -EROFS_UNIMPLEMENTED) +EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, lstat(const char *path, struct stat *buf), -EROFS_UNIMPLEMENTED) +EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, access(const char *pathname, int mode), -EROFS_UNIMPLEMENTED) +EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, truncate(const char *path, off_t length), -EROFS_UNIMPLEMENTED) +EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, utime(const char *path, const struct utimbuf *file_times), -EROFS_UNIMPLEMENTED) +EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, utimes(const char *path, const struct timeval times[2]), -EROFS_UNIMPLEMENTED) +EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, lutimes(const char *path, const struct timeval times[2]), -EROFS_UNIMPLEMENTED) +EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, mknod(const char *path, mode_t mode, dev_t dev), -EROFS_UNIMPLEMENTED) +EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, syncfs(), -EROFS_UNIMPLEMENTED) + +ErofsFileSystem::ErofsFileSystem(photon::fs::IFile *imgfile, uint64_t blksize) +{ + target_file.ops.pread = erofs_target_pread; + target_file.ops.pwrite = erofs_target_pwrite; + target_file.ops.pread = erofs_target_pread; + target_file.ops.pwrite = erofs_target_pwrite; + target_file.ops.fsync = erofs_target_fsync; + target_file.ops.fallocate = erofs_target_fallocate; + target_file.ops.ftruncate = erofs_target_ftruncate; + target_file.ops.read = erofs_target_read; + target_file.ops.lseek = erofs_target_lseek; + target_file.file = imgfile; + target_file.cache = new ErofsCache(target_file.file, 128); + + memset(&sbi, 0, sizeof(struct erofs_sb_info)); + sbi.blkszbits = ilog2(blksize); + sbi.bdev.ops = &target_file.ops; + target_file.file->lseek(0,0); + sbi.devsz = INT64_MAX; + if (erofs_read_superblock(&sbi)) + LOG_ERROR("[erofs] Fail to read_super_block"); +} + +ErofsFileSystem::~ErofsFileSystem() +{ + delete target_file.cache; +} + +int ErofsFileSystem::stat(const char *path, struct stat *buf) +{ + struct erofs_inode vi; + int err; + + vi.sbi = &sbi; + err = do_erofs_ilookup(path, &vi); + if (err) + LOG_ERRNO_RETURN(err, err, "[erofs] Fail to lookup inode"); + buf->st_mode = vi.i_mode; + buf->st_nlink = vi.i_nlink; + buf->st_size = vi.i_size; + buf->st_blocks = roundup(vi.i_size, erofs_blksiz(vi.sbi)) >> 9; + buf->st_uid = vi.i_uid; + buf->st_gid = vi.i_gid; + buf->st_ctime = vi.i_mtime; + buf->st_mtime = vi.i_mtime; + buf->st_atime = vi.i_mtime; + return 0; +} + +photon::fs::IFile* ErofsFileSystem::open(const char *pathname, int flags) +{ + ErofsFile *file = new ErofsFile(this); + int err; + + file->inode.sbi = &sbi; + err = do_erofs_ilookup(pathname, &file->inode); + if (err) { + delete file; + LOG_ERROR_RETURN(-err, nullptr, "[erofs] Fail to lookup inode by path"); + } + return file; +} + +struct liberofs_dir_context { + struct erofs_dir_context ctx; + std::vector<::dirent> *dirs; +}; + +static int liberofs_readdir(struct erofs_dir_context *ctx) +{ + struct liberofs_dir_context *libctx = + reinterpret_cast(ctx); + std::vector<::dirent> *dirs = libctx->dirs; + struct dirent tmpdir; + + if (ctx->dot_dotdot) + return 0; + + tmpdir.d_ino = (ino_t) ctx->de_nid; + tmpdir.d_off = 0; + tmpdir.d_reclen = sizeof(struct erofs_dirent); + if (ctx->de_namelen > sizeof(tmpdir.d_name)) + LOG_ERROR_RETURN(-EINVAL, -EINVAL, "[erofs] Invalid name length"); + memset(tmpdir.d_name, 0, sizeof(tmpdir.d_name)); + memcpy(tmpdir.d_name, ctx->dname, ctx->de_namelen); + dirs->emplace_back(tmpdir); + return 0; +} + +static int do_erofs_readdir(struct erofs_sb_info *sbi, const char *path, + std::vector<::dirent> *dirs) +{ + struct liberofs_dir_context ctx; + struct erofs_inode vi; + int err; + + vi.sbi = sbi; + err = do_erofs_ilookup(path, &vi); + if (err) + LOG_ERRNO_RETURN(err, err, "[erofs] Fail to lookup inode"); + ctx.ctx.dir = &vi; + ctx.ctx.cb = liberofs_readdir; + ctx.dirs = dirs; + + return erofs_iterate_dir(&ctx.ctx, false); +} + +photon::fs::DIR* ErofsFileSystem::opendir(const char *name) +{ + std::vector<::dirent> dirs; + + auto ret = do_erofs_readdir(&sbi, name, &dirs); + if (ret) { + errno = -ret; + return nullptr; + } + return new ErofsDir(dirs); +} + +// ErofsDir +ErofsDir::ErofsDir(std::vector<::dirent> &dirs) : loc(0) { + m_dirs = std::move(dirs); + next(); +} + +ErofsDir::~ErofsDir() { + closedir(); +} + +int ErofsDir::closedir() { + if (!m_dirs.empty()) { + m_dirs.clear(); + } + return 0; +} + +dirent *ErofsDir::get() { + return direntp; +} + +int ErofsDir::next() { + if (!m_dirs.empty()) { + if (loc < (long) m_dirs.size()) { + direntp = &m_dirs[loc++]; + } else { + direntp = nullptr; + } + } + return direntp != nullptr ? 1 : 0; +} + +void ErofsDir::rewinddir() { + loc = 0; + next(); +} + +void ErofsDir::seekdir(long loc){ + this->loc = loc; + next(); +} + +long ErofsDir::telldir() { + return loc; +} + +bool erofs_check_fs(const photon::fs::IFile *imgfile) +{ + u8 data[EROFS_MAX_BLOCK_SIZE]; + struct erofs_super_block *dsb; + photon::fs::IFile *file = const_cast(imgfile); + int ret; + + ret = file->pread(data, EROFS_MAX_BLOCK_SIZE, 0); + if (ret != EROFS_MAX_BLOCK_SIZE) + LOG_ERROR_RETURN(-EIO, false, "[erofs] Fail to read superblock"); + dsb = reinterpret_cast(data + EROFS_SUPER_OFFSET); + return le32_to_cpu(dsb->magic) == EROFS_SUPER_MAGIC_V1; +} + +photon::fs::IFileSystem *erofs_create_fs(photon::fs::IFile *imgfile, uint64_t blksz) +{ + return new ErofsFileSystem(imgfile, blksz); +} diff --git a/src/overlaybd/tar/erofs/erofs_fs.h b/src/overlaybd/tar/erofs/erofs_fs.h new file mode 100644 index 00000000..377aea22 --- /dev/null +++ b/src/overlaybd/tar/erofs/erofs_fs.h @@ -0,0 +1,88 @@ +/* + Copyright The Overlaybd Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#pragma once + +#include +#include +#include +#include +#include +#include "erofs_common.h" +#include "erofs/internal.h" + +class ErofsFileSystem: public photon::fs::IFileSystem { +public: + struct erofs_sb_info sbi; + struct liberofs_file target_file; + + ErofsFileSystem(photon::fs::IFile *imgfile, uint64_t blksize); + ~ErofsFileSystem(); + photon::fs::IFile* open(const char *pathname, int flags); + photon::fs::IFile* open(const char *pathname, int flags, mode_t mode); + photon::fs::IFile* creat(const char *pathname, mode_t mode); + int mkdir(const char *pathname, mode_t mode); + int rmdir(const char *pathname); + int symlink(const char *oldname, const char *newname); + ssize_t readlink(const char *path, char *buf, size_t bufsiz); + int link(const char *oldname, const char *newname); + int rename(const char *oldname, const char *newname); + int unlink(const char *filename); + int chmod(const char *pathname, mode_t mode); + int chown(const char *pathname, uid_t owner, gid_t group); + int lchown(const char *pathname, uid_t owner, gid_t group); + int statfs(const char *path, struct statfs *buf); + int statvfs(const char *path, struct statvfs *buf); + int stat(const char *path, struct stat *buf); + int lstat(const char *path, struct stat *buf); + int access(const char *pathname, int mode); + int truncate(const char *path, off_t length); + int utime(const char *path, const struct utimbuf *file_times); + int utimes(const char *path, const struct timeval times[2]); + int lutimes(const char *path, const struct timeval times[2]); + int mknod(const char *path, mode_t mode, dev_t dev); + int syncfs(); + photon::fs::DIR* opendir(const char *name); +}; + +class ErofsFile: public photon::fs::VirtualReadOnlyFile { +public: + ErofsFileSystem *fs; + struct erofs_inode inode; + + ErofsFile(ErofsFileSystem *fs); + photon::fs::IFileSystem *filesystem(); + int fstat(struct stat *buf); + int fiemap(struct photon::fs::fiemap *map); +}; + +class ErofsDir: public photon::fs::DIR { +public: + std::vector<::dirent> m_dirs; + ::dirent *direntp = nullptr; + long loc; + ErofsDir(std::vector<::dirent> &dirs); + ~ErofsDir(); + int closedir(); + dirent *get(); + int next(); + void rewinddir(); + void seekdir(long loc); + long telldir(); +}; + +bool erofs_check_fs(const photon::fs::IFile *imgfile); +photon::fs::IFileSystem *erofs_create_fs(photon::fs::IFile *imgfile, uint64_t blksz); diff --git a/src/overlaybd/tar/erofs/liberofs.cpp b/src/overlaybd/tar/erofs/liberofs.cpp index 7a205df4..24fa1dab 100644 --- a/src/overlaybd/tar/erofs/liberofs.cpp +++ b/src/overlaybd/tar/erofs/liberofs.cpp @@ -1,3 +1,4 @@ +#include "erofs_common.h" #include "liberofs.h" #include "erofs/tar.h" #include "erofs/io.h" @@ -14,413 +15,6 @@ #include #include -#define SECTOR_SIZE 512ULL -#define SECTOR_BITS 9 -#define round_down_blk(addr) ((addr) & (~(SECTOR_SIZE - 1))) -#define round_up_blk(addr) (round_down_blk((addr) + SECTOR_SIZE - 1)) -#define min(a, b) (a) < (b) ? (a) : (b) -#define EROFS_ROOT_XATTR_SZ (16 * 1024) - -#define EROFS_UNIMPLEMENTED 1 - -struct liberofs_inmem_sector { - char data[SECTOR_SIZE]; -}; - -class ErofsCache { -public: - ErofsCache(photon::fs::IFile *file, unsigned long int capacity): - file(file), capacity(capacity) - {} - ~ErofsCache() {} - ssize_t write_sector(u64 addr, char *buf); - ssize_t read_sector(u64 addr, char *buf); - int flush(); -public: - photon::fs::IFile *file; - long unsigned int capacity; - std::mapcaches; - std::set dirty; -}; - -struct liberofs_file { - struct erofs_vfops ops; - photon::fs::IFile *file; - ErofsCache *cache; -}; - -ssize_t ErofsCache::write_sector(u64 addr, char *buf) -{ - struct liberofs_inmem_sector *sector; - - if (addr & (SECTOR_SIZE-1)) { - LOG_ERROR("Invalid addr, should be aligned to SECTOR_SIZE."); - return -1; - } - if (caches.find(addr) != caches.end()) { - sector = caches[addr]; - memcpy(sector->data, buf, SECTOR_SIZE); - dirty.insert(addr); - } else { - if (caches.size() == capacity) { - auto it = caches.begin(); - if (dirty.find(it->first) != dirty.end()) { - if (file->pwrite(it->second->data, SECTOR_SIZE, it->first) - != SECTOR_SIZE) - { - LOG_ERROR("Fail to write sector %lld.", it->first); - return -EIO; - } - dirty.erase(it->first); - } - sector = it->second; - caches.erase(it); - } else { - sector = (struct liberofs_inmem_sector*)malloc( - sizeof(struct liberofs_inmem_sector)); - if (!sector) - return -ENOMEM; - } - memcpy(sector->data, buf, SECTOR_SIZE); - caches[addr] = sector; - dirty.insert(addr); - } - return SECTOR_SIZE; -} - -ssize_t ErofsCache::read_sector(u64 addr, char *buf) -{ - struct liberofs_inmem_sector *sector; - - if (addr & (SECTOR_SIZE - 1)) { - LOG_ERROR("Invalid addr, should be aligned to SECTOR_SIZE."); - return -1; - } - - if (caches.find(addr) != caches.end()) { - sector = caches[addr]; - memcpy(buf, sector->data, SECTOR_SIZE); - } else { - if (caches.size() == capacity) { - auto it = caches.begin(); - if (dirty.find(it->first) != dirty.end()) { - sector = it->second; - if (file->pwrite(sector->data, SECTOR_SIZE, it->first) - != SECTOR_SIZE) - { - LOG_ERROR("Fail to write sector %lld.", it->first); - return -EIO; - } - dirty.erase(it->first); - } - sector = it->second; - caches.erase(it); - } else { - sector = (struct liberofs_inmem_sector*)malloc( - sizeof(struct liberofs_inmem_sector)); - if (!sector) - return -ENOMEM; - } - if (file->pread(sector->data, SECTOR_SIZE, addr) != SECTOR_SIZE) { - LOG_ERROR("Fail to read sector %lld", addr); - return -EIO; - } - caches[addr] = sector; - memcpy(buf, sector->data, SECTOR_SIZE); - } - return SECTOR_SIZE; -} - -int ErofsCache::flush() -{ - for (auto it = caches.begin(); it != caches.end();) { - if (dirty.find(it->first) != dirty.end()) { - if (file->pwrite(it->second->data, SECTOR_SIZE, it->first) - != SECTOR_SIZE) - { - LOG_ERROR("Fail to flush sector %lld.\n", it->first); - return -1; - } - dirty.erase(it->first); - } - free(it->second); - it = caches.erase(it); - } - return dirty.size() != 0 || caches.size() != 0; -} - -/* - * Helper function for reading from the photon file, since - * the photon file requires reads to be 512-byte aligned. - */ -static ssize_t erofs_read_photon_file(void *buf, u64 offset, size_t len, - ErofsCache *cache) -{ - - ssize_t read; - u64 start, end; - char extra_buf[SECTOR_SIZE]; - u64 i, j; - - start = round_down_blk(offset); - end = round_up_blk(offset + len); - read = 0; - - /* we use the extra_buf to store the first sector or last sector when: - * - start != offset - * or - * - end != offset + len - */ - if (start != offset || end != offset + len) { - i = start == offset ? start : start + SECTOR_SIZE; - j = end == offset + len ? end : end - SECTOR_SIZE; - - /* read the first sector */ - if (i != start) { - if (cache->read_sector(start, extra_buf) != SECTOR_SIZE) { - LOG_ERROR("Fail to read start sector."); - return -1; - } - memcpy((char*)buf, extra_buf + offset - start, - min(start + SECTOR_SIZE - offset,len)); - read += min(start + SECTOR_SIZE - offset,len); - } - - /* read the last sector and avoid re-reading the same sector as above */ - if (j != end && (i == start || end - start > SECTOR_SIZE)) { - if (cache->read_sector(end - SECTOR_SIZE, - extra_buf) != SECTOR_SIZE) - { - LOG_ERROR("Fail to read start sector."); - return -1; - } - memcpy((char*)buf + end - SECTOR_SIZE - offset, extra_buf, - len + offset + SECTOR_SIZE - end); - read += len + offset + SECTOR_SIZE - end; - } - - for (u64 addr = i; addr < j; addr += SECTOR_SIZE) { - if (cache->read_sector(addr, - addr - offset + (char*)buf) != SECTOR_SIZE) - { - LOG_ERROR("Fail to read sector %lld in read_photo_file.\n", i); - return -1; - } - read += SECTOR_SIZE; - } - } else { - /* if read request is sector-aligned, we use the original buffer */ - for (u64 i = start; i < end; i += SECTOR_SIZE) { - if (cache->read_sector(i, (char*)buf + i - start) != SECTOR_SIZE) { - LOG_ERROR("Fail to read start sector."); - return -1; - } - read += SECTOR_SIZE; - } - } - - return read; -} - -/* - * Helper function for writing to a photon file. - */ - static ssize_t erofs_write_photon_file(const void *buf, u64 offset, - size_t len, ErofsCache *cache) - { - ssize_t write; - u64 start, end; - char extra_buf[SECTOR_SIZE]; - u64 i, j; - - start = round_down_blk(offset); - end = round_up_blk(offset + len); - write = 0; - - if (start != offset || end != offset + len) { - i = start == offset ? start : start + SECTOR_SIZE; - j = end == offset + len ? end : end - SECTOR_SIZE; - - if (i != start) { - if (cache->read_sector(start, extra_buf) != SECTOR_SIZE) { - LOG_ERROR("Fail to read sector %lld.\n", start); - return -1; - } - memcpy(extra_buf + offset - start, buf, - min(start + SECTOR_SIZE - offset, len)); - if (cache->write_sector(start, extra_buf) != SECTOR_SIZE) { - LOG_ERROR("Fail to write sector %lld\n", start); - return -1; - } - write += min(start + SECTOR_SIZE - offset, len); - } - - if (j != end && (i == start || end - start > SECTOR_SIZE)) { - if (cache->read_sector(end - SECTOR_SIZE, extra_buf) - != SECTOR_SIZE) - { - LOG_ERROR("Fail to read sector %lld.", end - SECTOR_SIZE); - return -1; - } - memcpy(extra_buf, (char*)buf + end - SECTOR_SIZE - offset, - offset + len + SECTOR_SIZE - end); - if (cache->write_sector(end - SECTOR_SIZE, extra_buf) - != SECTOR_SIZE) - { - LOG_ERROR("Fail to write sector %lld.", end - SECTOR_SIZE); - return -1; - } - write += offset + len + SECTOR_SIZE - end; - } - - for (u64 addr = i; addr < j; addr += SECTOR_SIZE) { - if (cache->write_sector(addr, (char*)buf + addr - offset) - != SECTOR_SIZE) - { - LOG_ERROR("Fail to write sector %lld.", addr); - return -1; - } - write += SECTOR_SIZE; - } - } else { - for (u64 addr = start; addr < end; addr += SECTOR_SIZE) { - if (cache->write_sector(addr, (char *)buf + addr - start) - != SECTOR_SIZE) - { - LOG_ERROR("Fail to write sector %lld.", addr); - return -1; - } - write += SECTOR_SIZE; - } - } - - return write; -} - -/* I/O control for target */ -static ssize_t erofs_target_pread(struct erofs_vfile *vf, void *buf, u64 offset, - size_t len) -{ - struct liberofs_file *target_file = - reinterpret_cast(vf->ops); - - if (!target_file) - return -EINVAL; - if (erofs_read_photon_file(buf, offset, len, target_file->cache) - != (ssize_t)len) - return -1; - - return len; -} - -static ssize_t erofs_target_pwrite(struct erofs_vfile *vf, const void *buf, - u64 offset, size_t len) -{ - struct liberofs_file *target_file = - reinterpret_cast(vf->ops); - - if (!target_file) - return -EINVAL; - if (!buf) - return -EINVAL; - - return erofs_write_photon_file(buf, offset, len, target_file->cache); -} - -static int erofs_target_fsync(struct erofs_vfile *vf) -{ - struct liberofs_file *target_file = - reinterpret_cast(vf->ops); - - if (!target_file) - return -EINVAL; - return target_file->cache->flush(); -} - -static int erofs_target_fallocate(struct erofs_vfile *vf, u64 offset, - size_t len, bool pad) -{ - static const char zero[4096] = {0}; - ssize_t ret; - - while (len > 4096) { - ret = erofs_target_pwrite(vf, zero, offset, 4096); - if (ret) - return ret; - len -= 4096; - offset += 4096; - } - ret = erofs_target_pwrite(vf, zero, offset, len); - if (ret != (ssize_t)len) { - return -1; - } - return 0; -} - -static int erofs_target_ftruncate(struct erofs_vfile *vf, u64 length) -{ - return 0; -} - -static ssize_t erofs_target_read(struct erofs_vfile *vf, void *buf, size_t len) -{ - return -EROFS_UNIMPLEMENTED; -} - -static off_t erofs_target_lseek(struct erofs_vfile *vf, u64 offset, int whence) -{ - return -EROFS_UNIMPLEMENTED; -} - -/* I/O control for source */ -static ssize_t erofs_source_pread(struct erofs_vfile *vf, void *buf, u64 offset, - size_t len) -{ - return -EROFS_UNIMPLEMENTED; -} - -static ssize_t erofs_source_pwrite(struct erofs_vfile *vf, const void *buf, - u64 offset, size_t len) -{ - return -EROFS_UNIMPLEMENTED; -} - -static int erofs_source_fsync(struct erofs_vfile *vf) -{ - return -EROFS_UNIMPLEMENTED; -} - -static int erofs_source_fallocate(struct erofs_vfile *vf, - u64 offset, size_t len, bool pad) -{ - return -EROFS_UNIMPLEMENTED; -} - -static int erofs_source_ftruncate(struct erofs_vfile *vf, u64 length) -{ - return -EROFS_UNIMPLEMENTED; -} - -static ssize_t erofs_source_read(struct erofs_vfile *vf, void *buf, - size_t bytes) -{ - struct liberofs_file *source_file = - reinterpret_cast(vf->ops); - - if (!source_file) - return -EINVAL; - return source_file->file->read(buf, bytes); -} - -static off_t erofs_source_lseek(struct erofs_vfile *vf, u64 offset, int whence) -{ - struct liberofs_file *source_file = - reinterpret_cast(vf->ops); - if (!source_file) - return -EINVAL; - return source_file->file->lseek(offset, whence); -} - struct erofs_mkfs_cfg { struct erofs_sb_info *sbi; struct erofs_tarfile *erofstar; @@ -722,470 +316,3 @@ LibErofs::LibErofs(photon::fs::IFile *target, uint64_t blksize, bool import_tar_ LibErofs::~LibErofs() { } - -class ErofsFileSystem: public photon::fs::IFileSystem { -public: - struct erofs_sb_info sbi; - struct liberofs_file target_file; - - ErofsFileSystem(photon::fs::IFile *imgfile, uint64_t blksize); - ~ErofsFileSystem(); - photon::fs::IFile* open(const char *pathname, int flags); - photon::fs::IFile* open(const char *pathname, int flags, mode_t mode); - photon::fs::IFile* creat(const char *pathname, mode_t mode); - int mkdir(const char *pathname, mode_t mode); - int rmdir(const char *pathname); - int symlink(const char *oldname, const char *newname); - ssize_t readlink(const char *path, char *buf, size_t bufsiz); - int link(const char *oldname, const char *newname); - int rename(const char *oldname, const char *newname); - int unlink(const char *filename); - int chmod(const char *pathname, mode_t mode); - int chown(const char *pathname, uid_t owner, gid_t group); - int lchown(const char *pathname, uid_t owner, gid_t group); - int statfs(const char *path, struct statfs *buf); - int statvfs(const char *path, struct statvfs *buf); - int stat(const char *path, struct stat *buf); - int lstat(const char *path, struct stat *buf); - int access(const char *pathname, int mode); - int truncate(const char *path, off_t length); - int utime(const char *path, const struct utimbuf *file_times); - int utimes(const char *path, const struct timeval times[2]); - int lutimes(const char *path, const struct timeval times[2]); - int mknod(const char *path, mode_t mode, dev_t dev); - int syncfs(); - photon::fs::DIR* opendir(const char *name); -}; - -class ErofsFile: public photon::fs::VirtualReadOnlyFile { -public: - ErofsFileSystem *fs; - struct erofs_inode inode; - - ErofsFile(ErofsFileSystem *fs); - photon::fs::IFileSystem *filesystem(); - int fstat(struct stat *buf); - int fiemap(struct photon::fs::fiemap *map); -}; - -class ErofsDir: public photon::fs::DIR { -public: - std::vector<::dirent> m_dirs; - ::dirent *direntp = nullptr; - long loc; - ErofsDir(std::vector<::dirent> &dirs); - ~ErofsDir(); - int closedir(); - dirent *get(); - int next(); - void rewinddir(); - void seekdir(long loc); - long telldir(); -}; - -#define EROFS_UNIMPLEMENTED_FUNC(ret_type, cls, func, ret) \ -ret_type cls::func { \ - return ret; \ -} - -// ErofsFile -ErofsFile::ErofsFile(ErofsFileSystem *fs): fs(fs) -{ - memset(&inode, 0, sizeof(struct erofs_inode)); -} -photon::fs::IFileSystem *ErofsFile::filesystem() { return fs; } - -struct liberofs_nameidata { - struct erofs_sb_info *sbi; - erofs_nid_t nid; -}; - -static int liberofs_link_path_walk(const char *name, - struct liberofs_nameidata *nd); - -static struct erofs_dirent *liberofs_find_dirent(void *data, const char *name, - unsigned int len, - unsigned int nameoff, - unsigned int maxsize) -{ - struct erofs_dirent *de = (struct erofs_dirent *)data; - const struct erofs_dirent *end = (struct erofs_dirent *)((char *)data + nameoff); - - while (de < end) { - const char *de_name; - unsigned int de_namelen; - - nameoff = le16_to_cpu(de->nameoff); - de_name = (char *)((char *)data + nameoff); - if (de + 1 >= end) - de_namelen = strnlen(de_name, maxsize - nameoff); - else - de_namelen = le16_to_cpu(de[1].nameoff - nameoff); - - if (nameoff + de_namelen > maxsize) { - LOG_ERROR("[erofs] bogus dirent"); - return (struct erofs_dirent *)ERR_PTR(-EINVAL); - } - - if (len == de_namelen && !memcmp(de_name, name, de_namelen)) - return de; - ++de; - } - return NULL; -} - -static int liberofs_namei(struct liberofs_nameidata *nd, const char *name, - unsigned int len) -{ - erofs_nid_t nid = nd->nid; - int ret; - char buf[EROFS_MAX_BLOCK_SIZE]; - struct erofs_sb_info *sbi = nd->sbi; - struct erofs_inode vi = {}; - erofs_off_t offset; - - vi.sbi = sbi; - vi.nid = nid; - ret = erofs_read_inode_from_disk(&vi); - if (ret) - return ret; - - offset = 0; - while (offset < vi.i_size) { - erofs_off_t maxsize = min_t(erofs_off_t, - vi.i_size - offset, erofs_blksiz(sbi)); - struct erofs_dirent *de = (struct erofs_dirent *)buf; - unsigned int nameoff; - - ret = erofs_pread(&vi, buf, maxsize, offset); - if (ret) - return ret; - - nameoff = le16_to_cpu(de->nameoff); - if (nameoff < sizeof(struct erofs_dirent) || - nameoff >= erofs_blksiz(sbi)) - LOG_ERRNO_RETURN(-EINVAL, -EINVAL, "[erofs] invalid nameoff"); - - de = liberofs_find_dirent(buf, name, len, nameoff, maxsize); - if (IS_ERR(de)) - return PTR_ERR(de); - - if (de) { - nd->nid = le64_to_cpu(de->nid); - return 0; - } - offset += maxsize; - } - return -ENOENT; -} - - -static int liberofs_step_into_link(struct liberofs_nameidata *nd, - struct erofs_inode *vi) -{ - char buf[PATH_MAX]; - int err; - - if (vi->i_size > PATH_MAX) - return -EINVAL; - memset(buf, 0, sizeof(buf)); - err = erofs_pread(vi, buf, vi->i_size, 0); - if (err) - return err; - return liberofs_link_path_walk(buf, nd); -} - -static int liberofs_link_path_walk(const char *name, - struct liberofs_nameidata *nd) -{ - struct erofs_inode vi; - erofs_nid_t nid; - const char *p; - int ret; - - if (*name == '/') - nd->nid = nd->sbi->root_nid; - - while (*name == '/') - name ++; - - while (*name != '\0') { - p = name; - do { - ++p; - } while (*p != '\0' && *p != '/'); - - nid = nd->nid; - ret = liberofs_namei(nd, name, p - name); - if (ret) - return ret; - vi.sbi = nd->sbi; - vi.nid = nd->nid; - ret = erofs_read_inode_from_disk(&vi); - if (ret) - return ret; - if (S_ISLNK(vi.i_mode)) { - nd->nid = nid; - ret = liberofs_step_into_link(nd, &vi); - if (ret) - return ret; - } - for (name = p; *name == '/'; ++name) - ; - } - return 0; -} - - -static int do_erofs_ilookup(const char *path, struct erofs_inode *vi) -{ - int ret; - struct liberofs_nameidata nd = {.sbi = vi->sbi}; - - nd.nid = vi->sbi->root_nid; - ret = liberofs_link_path_walk(path, &nd); - if (ret) - return ret; - vi->nid = nd.nid; - return erofs_read_inode_from_disk(vi); -} - -int ErofsFile::fstat(struct stat *buf) -{ - buf->st_mode = inode.i_mode; - buf->st_nlink = inode.i_nlink; - buf->st_size = inode.i_size; - buf->st_blocks = roundup(inode.i_size, erofs_blksiz(inode.sbi)) >> 9; - buf->st_uid = inode.i_uid; - buf->st_gid = inode.i_gid; - buf->st_ctime = inode.i_mtime; - buf->st_mtime = inode.i_mtime; - buf->st_atime = inode.i_mtime; - return 0; -} - -int ErofsFile::fiemap(struct photon::fs::fiemap *map) -{ - photon::fs::fiemap_extent *ext_buf = &map->fm_extents[0]; - struct erofs_map_blocks erofs_map; - int err; - - map->fm_mapped_extents = 0; - erofs_map.index = UINT_MAX; - erofs_map.m_la = 0; - - while (erofs_map.m_la < inode.i_size) { - err = erofs_map_blocks(&inode, &erofs_map, 0); - if (err) - LOG_ERROR_RETURN(err, err, "[erofs] Fail to map erofs blocks"); - ext_buf[map->fm_mapped_extents].fe_physical = erofs_map.m_pa; - ext_buf[map->fm_mapped_extents].fe_length = erofs_map.m_plen; - map->fm_mapped_extents += 1; - erofs_map.m_la += erofs_map.m_llen; - } - return 0; -} - -// ErofsFileSystem -EROFS_UNIMPLEMENTED_FUNC(photon::fs::IFile*, ErofsFileSystem, open(const char *pathname, int flags, mode_t mode), NULL) -EROFS_UNIMPLEMENTED_FUNC(photon::fs::IFile*, ErofsFileSystem, creat(const char *pathname, mode_t mode), NULL) -EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, mkdir(const char *pathname, mode_t mode), -EROFS_UNIMPLEMENTED) -EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, rmdir(const char *pathname), -EROFS_UNIMPLEMENTED) -EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, symlink(const char *oldname, const char *newname), -EROFS_UNIMPLEMENTED) -EROFS_UNIMPLEMENTED_FUNC(ssize_t, ErofsFileSystem, readlink(const char *path, char *buf, size_t bufsiz), -EROFS_UNIMPLEMENTED) -EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, link(const char *oldname, const char *newname), -EROFS_UNIMPLEMENTED) -EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, rename(const char *oldname, const char *newname), -EROFS_UNIMPLEMENTED) -EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, unlink(const char *filename), -EROFS_UNIMPLEMENTED) -EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, chmod(const char *pathname, mode_t mode), -EROFS_UNIMPLEMENTED) -EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, chown(const char *pathname, uid_t owner, gid_t group), -EROFS_UNIMPLEMENTED) -EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, lchown(const char *pathname, uid_t owner, gid_t group), -EROFS_UNIMPLEMENTED) -EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, statfs(const char *path, struct statfs *buf), -EROFS_UNIMPLEMENTED) -EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, statvfs(const char *path, struct statvfs *buf), -EROFS_UNIMPLEMENTED) -EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, lstat(const char *path, struct stat *buf), -EROFS_UNIMPLEMENTED) -EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, access(const char *pathname, int mode), -EROFS_UNIMPLEMENTED) -EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, truncate(const char *path, off_t length), -EROFS_UNIMPLEMENTED) -EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, utime(const char *path, const struct utimbuf *file_times), -EROFS_UNIMPLEMENTED) -EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, utimes(const char *path, const struct timeval times[2]), -EROFS_UNIMPLEMENTED) -EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, lutimes(const char *path, const struct timeval times[2]), -EROFS_UNIMPLEMENTED) -EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, mknod(const char *path, mode_t mode, dev_t dev), -EROFS_UNIMPLEMENTED) -EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, syncfs(), -EROFS_UNIMPLEMENTED) - -ErofsFileSystem::ErofsFileSystem(photon::fs::IFile *imgfile, uint64_t blksize) -{ - target_file.ops.pread = erofs_target_pread; - target_file.ops.pwrite = erofs_target_pwrite; - target_file.ops.pread = erofs_target_pread; - target_file.ops.pwrite = erofs_target_pwrite; - target_file.ops.fsync = erofs_target_fsync; - target_file.ops.fallocate = erofs_target_fallocate; - target_file.ops.ftruncate = erofs_target_ftruncate; - target_file.ops.read = erofs_target_read; - target_file.ops.lseek = erofs_target_lseek; - target_file.file = imgfile; - target_file.cache = new ErofsCache(target_file.file, 128); - - memset(&sbi, 0, sizeof(struct erofs_sb_info)); - (void)erofs_init_sbi(&sbi, target_file.file, &target_file.ops, ilog2(blksize)); - if (erofs_read_superblock(&sbi)) - LOG_ERROR("[erofs] Fail to read_super_block"); -} - -ErofsFileSystem::~ErofsFileSystem() -{ - delete target_file.cache; -} - -int ErofsFileSystem::stat(const char *path, struct stat *buf) -{ - struct erofs_inode vi; - int err; - - vi.sbi = &sbi; - err = do_erofs_ilookup(path, &vi); - if (err) - LOG_ERRNO_RETURN(err, err, "[erofs] Fail to lookup inode"); - buf->st_mode = vi.i_mode; - buf->st_nlink = vi.i_nlink; - buf->st_size = vi.i_size; - buf->st_blocks = roundup(vi.i_size, erofs_blksiz(vi.sbi)) >> 9; - buf->st_uid = vi.i_uid; - buf->st_gid = vi.i_gid; - buf->st_ctime = vi.i_mtime; - buf->st_mtime = vi.i_mtime; - buf->st_atime = vi.i_mtime; - return 0; -} - -photon::fs::IFile* ErofsFileSystem::open(const char *pathname, int flags) -{ - ErofsFile *file = new ErofsFile(this); - int err; - - file->inode.sbi = &sbi; - err = do_erofs_ilookup(pathname, &file->inode); - if (err) { - delete file; - LOG_ERROR_RETURN(-err, nullptr, "[erofs] Fail to lookup inode by path"); - } - return file; -} - -struct liberofs_dir_context { - struct erofs_dir_context ctx; - std::vector<::dirent> *dirs; -}; - -static int liberofs_readdir(struct erofs_dir_context *ctx) -{ - struct liberofs_dir_context *libctx = reinterpret_cast(ctx); - std::vector<::dirent> *dirs = libctx->dirs; - struct dirent tmpdir; - - if (ctx->dot_dotdot) - return 0; - - tmpdir.d_ino = (ino_t) ctx->de_nid; - tmpdir.d_off = 0; - tmpdir.d_reclen = sizeof(struct erofs_dirent); - if (ctx->de_namelen > sizeof(tmpdir.d_name)) - LOG_ERROR_RETURN(-EINVAL, -EINVAL, "[erofs] Invalid name length"); - memset(tmpdir.d_name, 0, sizeof(tmpdir.d_name)); - memcpy(tmpdir.d_name, ctx->dname, ctx->de_namelen); - dirs->emplace_back(tmpdir); - return 0; -} - -static int do_erofs_readdir(struct erofs_sb_info *sbi, const char *path, std::vector<::dirent> *dirs) -{ - struct liberofs_dir_context ctx; - struct erofs_inode vi; - int err; - - vi.sbi = sbi; - err = do_erofs_ilookup(path, &vi); - if (err) - LOG_ERRNO_RETURN(err, err, "[erofs] Fail to lookup inode"); - ctx.ctx.dir = &vi; - ctx.ctx.cb = liberofs_readdir; - ctx.dirs = dirs; - - return erofs_iterate_dir(&ctx.ctx, false); -} - -photon::fs::DIR* ErofsFileSystem::opendir(const char *name) -{ - std::vector<::dirent> dirs; - - auto ret = do_erofs_readdir(&sbi, name, &dirs); - if (ret) { - errno = -ret; - return nullptr; - } - return new ErofsDir(dirs); -} - - -// ErofsDir -ErofsDir::ErofsDir(std::vector<::dirent> &dirs) : loc(0) { - m_dirs = std::move(dirs); - next(); -} - -ErofsDir::~ErofsDir() { - closedir(); -} - -int ErofsDir::closedir() { - if (!m_dirs.empty()) { - m_dirs.clear(); - } - return 0; -} - -dirent *ErofsDir::get() { - return direntp; -} - -int ErofsDir::next() { - if (!m_dirs.empty()) { - if (loc < (long) m_dirs.size()) { - direntp = &m_dirs[loc++]; - } else { - direntp = nullptr; - } - } - return direntp != nullptr ? 1 : 0; -} - -void ErofsDir::rewinddir() { - loc = 0; - next(); -} - -void ErofsDir::seekdir(long loc){ - this->loc = loc; - next(); -} - -long ErofsDir::telldir() { - return loc; -} - -bool erofs_check_fs(const photon::fs::IFile *imgfile) -{ - u8 data[EROFS_MAX_BLOCK_SIZE]; - struct erofs_super_block *dsb; - photon::fs::IFile *file = const_cast(imgfile); - int ret; - - ret = file->pread(data, EROFS_MAX_BLOCK_SIZE, 0); - if (ret != EROFS_MAX_BLOCK_SIZE) - LOG_ERROR_RETURN(-EIO, false, "[erofs] Fail to read superblock"); - dsb = reinterpret_cast(data + EROFS_SUPER_OFFSET); - return le32_to_cpu(dsb->magic) == EROFS_SUPER_MAGIC_V1; -} - -photon::fs::IFileSystem *erofs_create_fs(photon::fs::IFile *imgfile, uint64_t blksz) -{ - return new ErofsFileSystem(imgfile, blksz); -} diff --git a/src/overlaybd/tar/erofs/liberofs.h b/src/overlaybd/tar/erofs/liberofs.h index 4805ca6f..c072e5dd 100644 --- a/src/overlaybd/tar/erofs/liberofs.h +++ b/src/overlaybd/tar/erofs/liberofs.h @@ -16,7 +16,4 @@ class LibErofs { bool ddtaridx; }; -bool erofs_check_fs(const photon::fs::IFile *imgfile); -photon::fs::IFileSystem *erofs_create_fs(photon::fs::IFile *imgfile, uint64_t blksz); - #endif diff --git a/src/tools/comm_func.cpp b/src/tools/comm_func.cpp index ffa42873..875d141e 100644 --- a/src/tools/comm_func.cpp +++ b/src/tools/comm_func.cpp @@ -23,7 +23,6 @@ #include "../overlaybd/registryfs/registryfs.h" #include "../image_service.h" #include "../image_file.h" -#include "../overlaybd/tar/erofs/liberofs.h" using namespace std; @@ -81,6 +80,9 @@ photon::fs::IFileSystem *create_ext4fs(photon::fs::IFile *imgfile, bool mkfs, return target; } +extern bool erofs_check_fs(const photon::fs::IFile *imgfile); +extern photon::fs::IFileSystem *erofs_create_fs(photon::fs::IFile *imgfile, uint64_t blksz); + bool is_erofs_fs(const photon::fs::IFile *imgfile) { if (imgfile == nullptr) From 104f77962344d37530c69dcebf960c552cffd531 Mon Sep 17 00:00:00 2001 From: Hongzhen Luo Date: Tue, 19 Nov 2024 14:37:38 +0800 Subject: [PATCH 2/3] [EROFS]: clean up erofs_fs Hide the parts related to liberofs in erofs_fs.h so that erofs_fs.h can be included. Signed-off-by: Hongzhen Luo --- src/overlaybd/tar/erofs/erofs_common.h | 12 +++ src/overlaybd/tar/erofs/erofs_fs.cpp | 128 ++++++++++++++++++------- src/overlaybd/tar/erofs/erofs_fs.h | 7 +- src/tools/comm_func.cpp | 4 +- 4 files changed, 108 insertions(+), 43 deletions(-) diff --git a/src/overlaybd/tar/erofs/erofs_common.h b/src/overlaybd/tar/erofs/erofs_common.h index 581add6d..df536145 100644 --- a/src/overlaybd/tar/erofs/erofs_common.h +++ b/src/overlaybd/tar/erofs/erofs_common.h @@ -23,15 +23,19 @@ #include #include +/* block-related definitions */ #define SECTOR_SIZE 512ULL #define SECTOR_BITS 9 +/* address alignment operation */ #define round_down_blk(addr) ((addr) & (~(SECTOR_SIZE - 1))) #define round_up_blk(addr) (round_down_blk((addr) + SECTOR_SIZE - 1)) #define erofs_min(a, b) (a) < (b) ? (a) : (b) #define EROFS_ROOT_XATTR_SZ (16 * 1024) +#define LIBEROFS_MAGIC 0x64616469 // dadi + #define EROFS_UNIMPLEMENTED 1 #define EROFS_UNIMPLEMENTED_FUNC(ret_type, cls, func, ret) \ @@ -43,6 +47,10 @@ struct liberofs_inmem_sector { char data[SECTOR_SIZE]; }; +/* + * Internal cache of EROFS, used to accelerate + * the read and write operations of an IFile. + */ class ErofsCache { public: ErofsCache(photon::fs::IFile *file, unsigned long int capacity): @@ -59,6 +67,10 @@ class ErofsCache { std::set dirty; }; +/* + * Encapsulation of IFile by liberofs, + * including I/O operations and ErofsCache. + */ struct liberofs_file { struct erofs_vfops ops; photon::fs::IFile *file; diff --git a/src/overlaybd/tar/erofs/erofs_fs.cpp b/src/overlaybd/tar/erofs/erofs_fs.cpp index 0102a6fb..b88f916b 100644 --- a/src/overlaybd/tar/erofs/erofs_fs.cpp +++ b/src/overlaybd/tar/erofs/erofs_fs.cpp @@ -21,11 +21,30 @@ #include #include +struct erofs_fs_private { + struct erofs_sb_info sbi; + struct liberofs_file target_file; + unsigned long magic = LIBEROFS_MAGIC; +}; + +struct erofs_file_private { + struct erofs_inode inode; + unsigned long magic = LIBEROFS_MAGIC; +}; + // ErofsFile ErofsFile::ErofsFile(ErofsFileSystem *fs): fs(fs) { - memset(&inode, 0, sizeof(struct erofs_inode)); + struct erofs_fs_private *_fs_private = + reinterpret_cast(fs->fs_private); + struct erofs_file_private *_file_private = new struct erofs_file_private; + + memset(&_file_private->inode, 0, sizeof(struct erofs_inode)); + _file_private->inode.sbi = &_fs_private->sbi; + _file_private->magic = LIBEROFS_MAGIC; + file_private = _file_private; } + photon::fs::IFileSystem *ErofsFile::filesystem() { return fs; } struct liberofs_nameidata { @@ -185,30 +204,42 @@ static int do_erofs_ilookup(const char *path, struct erofs_inode *vi) int ErofsFile::fstat(struct stat *buf) { - buf->st_mode = inode.i_mode; - buf->st_nlink = inode.i_nlink; - buf->st_size = inode.i_size; - buf->st_blocks = roundup(inode.i_size, erofs_blksiz(inode.sbi)) >> 9; - buf->st_uid = inode.i_uid; - buf->st_gid = inode.i_gid; - buf->st_ctime = inode.i_mtime; - buf->st_mtime = inode.i_mtime; - buf->st_atime = inode.i_mtime; + struct erofs_file_private *_file_private = + reinterpret_cast(file_private); + + if (!_file_private || _file_private->magic != LIBEROFS_MAGIC) + LOG_ERRNO_RETURN(0, -EINVAL, "[erofs_fs] invalid file_private"); + + buf->st_mode = _file_private->inode.i_mode; + buf->st_nlink = _file_private->inode.i_nlink; + buf->st_size = _file_private->inode.i_size; + buf->st_blocks = roundup(_file_private->inode.i_size, + erofs_blksiz(_file_private->inode.sbi)) >> 9; + buf->st_uid = _file_private->inode.i_uid; + buf->st_gid = _file_private->inode.i_gid; + buf->st_ctime = _file_private->inode.i_mtime; + buf->st_mtime = _file_private->inode.i_mtime; + buf->st_atime = _file_private->inode.i_mtime; return 0; } int ErofsFile::fiemap(struct photon::fs::fiemap *map) { + struct erofs_file_private *_file_private = + reinterpret_cast(file_private); photon::fs::fiemap_extent *ext_buf = &map->fm_extents[0]; struct erofs_map_blocks erofs_map; int err; + if (!_file_private || _file_private->magic != LIBEROFS_MAGIC) + LOG_ERRNO_RETURN(0, -EINVAL, "[erofs_fs] invalid file_private"); + map->fm_mapped_extents = 0; erofs_map.index = UINT_MAX; erofs_map.m_la = 0; - while (erofs_map.m_la < inode.i_size) { - err = erofs_map_blocks(&inode, &erofs_map, 0); + while (erofs_map.m_la < _file_private->inode.i_size) { + err = erofs_map_blocks(&_file_private->inode, &erofs_map, 0); if (err) LOG_ERROR_RETURN(err, err, "[erofs] Fail to map erofs blocks"); ext_buf[map->fm_mapped_extents].fe_physical = erofs_map.m_pa; @@ -245,38 +276,56 @@ EROFS_UNIMPLEMENTED_FUNC(int, ErofsFileSystem, syncfs(), -EROFS_UNIMPLEMENTED) ErofsFileSystem::ErofsFileSystem(photon::fs::IFile *imgfile, uint64_t blksize) { - target_file.ops.pread = erofs_target_pread; - target_file.ops.pwrite = erofs_target_pwrite; - target_file.ops.pread = erofs_target_pread; - target_file.ops.pwrite = erofs_target_pwrite; - target_file.ops.fsync = erofs_target_fsync; - target_file.ops.fallocate = erofs_target_fallocate; - target_file.ops.ftruncate = erofs_target_ftruncate; - target_file.ops.read = erofs_target_read; - target_file.ops.lseek = erofs_target_lseek; - target_file.file = imgfile; - target_file.cache = new ErofsCache(target_file.file, 128); - - memset(&sbi, 0, sizeof(struct erofs_sb_info)); - sbi.blkszbits = ilog2(blksize); - sbi.bdev.ops = &target_file.ops; - target_file.file->lseek(0,0); - sbi.devsz = INT64_MAX; - if (erofs_read_superblock(&sbi)) + struct erofs_fs_private *_fs_private = new erofs_fs_private; + + if (!_fs_private) + LOG_ERROR("[erofs] Fail to prepare fs_private: no mem."); + + _fs_private->magic = LIBEROFS_MAGIC; + // init target_file + _fs_private->target_file.ops.pread = erofs_target_pread; + _fs_private->target_file.ops.pwrite = erofs_target_pwrite; + _fs_private->target_file.ops.pread = erofs_target_pread; + _fs_private->target_file.ops.pwrite = erofs_target_pwrite; + _fs_private->target_file.ops.fsync = erofs_target_fsync; + _fs_private->target_file.ops.fallocate = erofs_target_fallocate; + _fs_private->target_file.ops.ftruncate = erofs_target_ftruncate; + _fs_private->target_file.ops.read = erofs_target_read; + _fs_private->target_file.ops.lseek = erofs_target_lseek; + _fs_private->target_file.file = imgfile; + _fs_private->target_file.cache = new ErofsCache(imgfile, 128); + + // init sbi + memset(&_fs_private->sbi, 0, sizeof(struct erofs_sb_info)); + _fs_private->sbi.blkszbits = ilog2(blksize); + _fs_private->sbi.bdev.ops = &_fs_private->target_file.ops; + _fs_private->target_file.file->lseek(0,0); + _fs_private->sbi.devsz = INT64_MAX; + if (erofs_read_superblock(&_fs_private->sbi)) LOG_ERROR("[erofs] Fail to read_super_block"); + fs_private = _fs_private; } ErofsFileSystem::~ErofsFileSystem() { - delete target_file.cache; + struct erofs_fs_private *_fs_private = + reinterpret_cast(fs_private); + delete _fs_private->target_file.cache; + delete _fs_private; + fs_private = NULL; } int ErofsFileSystem::stat(const char *path, struct stat *buf) { + struct erofs_fs_private *_fs_private = + reinterpret_cast(fs_private); struct erofs_inode vi; int err; - vi.sbi = &sbi; + if (!_fs_private || _fs_private->magic != LIBEROFS_MAGIC) + LOG_ERROR_RETURN(0, -EINVAL, "[erofs_fs] invalid fs_private"); + + vi.sbi = &_fs_private->sbi; err = do_erofs_ilookup(path, &vi); if (err) LOG_ERRNO_RETURN(err, err, "[erofs] Fail to lookup inode"); @@ -295,10 +344,14 @@ int ErofsFileSystem::stat(const char *path, struct stat *buf) photon::fs::IFile* ErofsFileSystem::open(const char *pathname, int flags) { ErofsFile *file = new ErofsFile(this); + struct erofs_file_private *_file_private = + reinterpret_cast(file->file_private); int err; - file->inode.sbi = &sbi; - err = do_erofs_ilookup(pathname, &file->inode); + if (!_file_private || _file_private->magic != LIBEROFS_MAGIC) + LOG_ERRNO_RETURN(0, nullptr, "[erofs_fs] invalid file_private."); + + err = do_erofs_ilookup(pathname, &_file_private->inode); if (err) { delete file; LOG_ERROR_RETURN(-err, nullptr, "[erofs] Fail to lookup inode by path"); @@ -352,9 +405,14 @@ static int do_erofs_readdir(struct erofs_sb_info *sbi, const char *path, photon::fs::DIR* ErofsFileSystem::opendir(const char *name) { + struct erofs_fs_private *_fs_private = + reinterpret_cast(fs_private); std::vector<::dirent> dirs; - auto ret = do_erofs_readdir(&sbi, name, &dirs); + if (!_fs_private || _fs_private->magic != LIBEROFS_MAGIC) + LOG_ERROR_RETURN(0, nullptr, "[erofs_fs] invalid fs_private."); + + auto ret = do_erofs_readdir(&_fs_private->sbi, name, &dirs); if (ret) { errno = -ret; return nullptr; diff --git a/src/overlaybd/tar/erofs/erofs_fs.h b/src/overlaybd/tar/erofs/erofs_fs.h index 377aea22..3e972784 100644 --- a/src/overlaybd/tar/erofs/erofs_fs.h +++ b/src/overlaybd/tar/erofs/erofs_fs.h @@ -21,13 +21,10 @@ #include #include #include -#include "erofs_common.h" -#include "erofs/internal.h" class ErofsFileSystem: public photon::fs::IFileSystem { public: - struct erofs_sb_info sbi; - struct liberofs_file target_file; + void *fs_private; ErofsFileSystem(photon::fs::IFile *imgfile, uint64_t blksize); ~ErofsFileSystem(); @@ -61,7 +58,7 @@ class ErofsFileSystem: public photon::fs::IFileSystem { class ErofsFile: public photon::fs::VirtualReadOnlyFile { public: ErofsFileSystem *fs; - struct erofs_inode inode; + void *file_private; ErofsFile(ErofsFileSystem *fs); photon::fs::IFileSystem *filesystem(); diff --git a/src/tools/comm_func.cpp b/src/tools/comm_func.cpp index 875d141e..6e362661 100644 --- a/src/tools/comm_func.cpp +++ b/src/tools/comm_func.cpp @@ -23,6 +23,7 @@ #include "../overlaybd/registryfs/registryfs.h" #include "../image_service.h" #include "../image_file.h" +#include "../overlaybd/tar/erofs/erofs_fs.h" using namespace std; @@ -80,9 +81,6 @@ photon::fs::IFileSystem *create_ext4fs(photon::fs::IFile *imgfile, bool mkfs, return target; } -extern bool erofs_check_fs(const photon::fs::IFile *imgfile); -extern photon::fs::IFileSystem *erofs_create_fs(photon::fs::IFile *imgfile, uint64_t blksz); - bool is_erofs_fs(const photon::fs::IFile *imgfile) { if (imgfile == nullptr) From b73ed2ce8c9ab5ceb661d7145d208ad3359d88af Mon Sep 17 00:00:00 2001 From: Hongzhen Luo Date: Sat, 16 Nov 2024 16:59:46 +0800 Subject: [PATCH 3/3] [EROFS]: test: add test for ErofsCache Add an unit test for the internal ErofsCache. Signed-off-by: Hongzhen Luo --- src/overlaybd/tar/erofs/test/test.cpp | 241 +++++++++++++++++++++++++- 1 file changed, 236 insertions(+), 5 deletions(-) diff --git a/src/overlaybd/tar/erofs/test/test.cpp b/src/overlaybd/tar/erofs/test/test.cpp index 6b334d1a..a7f29496 100644 --- a/src/overlaybd/tar/erofs/test/test.cpp +++ b/src/overlaybd/tar/erofs/test/test.cpp @@ -269,14 +269,18 @@ class ErofsPax : public ::testing::Test { virtual void TearDown() override{ ASSERT_NE(nullptr, host_fs); - if (host_fs->access(src_path.c_str(), 0) == 0) + if (host_fs->access(src_path.c_str(), 0) == 0) { ASSERT_EQ(0, host_fs->unlink(src_path.c_str())); - if (host_fs->access(fn_idx.c_str(), 0) == 0) + } + if (host_fs->access(fn_idx.c_str(), 0) == 0) { ASSERT_EQ(0, host_fs->unlink(fn_idx.c_str())); - if (host_fs->access(fn_meta.c_str(), 0) == 0) + } + if (host_fs->access(fn_meta.c_str(), 0) == 0) { ASSERT_EQ(0, host_fs->unlink(fn_meta.c_str())); - if (host_fs->access(sha256_path.c_str(), 0) == 0) + } + if (host_fs->access(sha256_path.c_str(), 0) == 0) { ASSERT_EQ(0, host_fs->unlink(sha256_path.c_str())); + } delete host_fs; } @@ -534,7 +538,7 @@ TEST_F(ErofsPax, pax_test) { auto dir = erofs_fs->opendir(tmp.c_str()); while (dir->next()) { dirent *dent = dir->get(); - items.emplace_back(tmp + "/" + dent->d_name); + items.emplace_back(tmp + "/" + std::string(dent->d_name)); } dir->closedir(); delete dir; @@ -551,6 +555,233 @@ TEST_F(ErofsPax, pax_test) { delete sha256file; } +/* test for the internal ErofsCache */ +typedef uint64_t u64; +class ErofsCache { +public: + ErofsCache(photon::fs::IFile *file, unsigned long int capacity): + file(file), capacity(capacity) + {} + ~ErofsCache() {} + ssize_t write_sector(u64 addr, char *buf); + ssize_t read_sector(u64 addr, char *buf); + int flush(); +public: + photon::fs::IFile *file; + long unsigned int capacity; + std::mapcaches; + std::set dirty; +}; + +/* helper functions for reading and writing photon files */ +extern ssize_t erofs_read_photon_file(void *buf, u64 offset, size_t len, + ErofsCache *cache); +extern ssize_t erofs_write_photon_file(const void *buf, u64 offset, + size_t len, ErofsCache *cache); +class ErofsCacheTest: public ::testing::Test { +protected: + std::string workdir = "/tmp/erofs_cache_test"; + std::string file_path = workdir + "/img_file"; + photon::fs::IFileSystem *host_fs; + photon::fs::IFile *img_file; + ErofsCache *cache; + + virtual void SetUp() override{ + /* prepare for workdir */ + host_fs = photon::fs::new_localfs_adaptor(); + ASSERT_NE(nullptr, host_fs); + if (host_fs->access(workdir.c_str(), 0)) { + ASSERT_EQ(host_fs->mkdir(workdir.c_str(), 0755), 0); + } + + /* prepare for img_file */ + img_file = host_fs->open(file_path.c_str(), O_RDWR | O_CREAT | O_TRUNC, + 0666); + ASSERT_NE(nullptr, img_file); + + /* + * prepare for cache. here, we use a cache with a size of only + * one sector to simulate a memory-constrained situation. + */ + cache = new ErofsCache(img_file, 1); + ASSERT_NE(nullptr, cache); + } + + virtual void TearDown() override{ + ASSERT_NE(nullptr, host_fs); + if (host_fs->access(file_path.c_str(), 0) == 0) { + ASSERT_EQ(host_fs->unlink(file_path.c_str()), 0); + } + delete host_fs; + delete img_file; + delete cache; + } +}; + +TEST_F(ErofsCacheTest, erofs_cache) { + +#define SECTOR_SIZE 512ULL +#define HALF_SECTOR (SECTOR_SIZE / 2) +#define BIG_OFFSET ((1ULL << 32) - 1) +#define round_down_blk(addr) ((addr) & (~(SECTOR_SIZE - 1))) +#define round_up_blk(addr) (round_down_blk((addr) + SECTOR_SIZE - 1)) + + char buffer[SECTOR_SIZE]; + char buffer_cmp[SECTOR_SIZE]; + + /* + * TC001 + * 1. write 0xff to [0, 512] + * 2. write 0x00 to [256, 512] + * 3. compare + */ + memset(buffer, 0xff, SECTOR_SIZE); + ASSERT_EQ(SECTOR_SIZE, erofs_write_photon_file(buffer, 0, SECTOR_SIZE, + cache)); + memset(buffer, 0x00, SECTOR_SIZE); + ASSERT_EQ(HALF_SECTOR, erofs_write_photon_file(buffer, HALF_SECTOR, + HALF_SECTOR, cache)); + memset(buffer_cmp, 0xff, HALF_SECTOR); + memset(buffer_cmp + HALF_SECTOR, 0x00, HALF_SECTOR); + ASSERT_EQ(SECTOR_SIZE, erofs_read_photon_file(buffer, 0, SECTOR_SIZE, + cache)); + ASSERT_EQ(0, memcmp(buffer, buffer_cmp, SECTOR_SIZE)); + + /* + * TC002 + * 1. write 0xff to [0, 1024] + * 2. write 0x00 to [0, 512] + * 3. flush, then read and compare + */ + memset(buffer, 0xff, SECTOR_SIZE); + ASSERT_EQ(SECTOR_SIZE, erofs_write_photon_file(buffer, 0, SECTOR_SIZE, + cache)); + ASSERT_EQ(SECTOR_SIZE, erofs_write_photon_file(buffer, SECTOR_SIZE, + SECTOR_SIZE, cache)); + memset(buffer, 0x00, SECTOR_SIZE); + ASSERT_EQ(SECTOR_SIZE, erofs_write_photon_file(buffer, 0, SECTOR_SIZE, + cache)); + cache->flush(); + memset(buffer, 0x00, SECTOR_SIZE); + ASSERT_EQ(SECTOR_SIZE, img_file->pread(buffer_cmp, SECTOR_SIZE, 0)); + ASSERT_EQ(0, memcmp(buffer, buffer_cmp, SECTOR_SIZE)); + memset(buffer, 0xff, SECTOR_SIZE); + ASSERT_EQ(SECTOR_SIZE, img_file->pread(buffer_cmp, SECTOR_SIZE, + SECTOR_SIZE)); + ASSERT_EQ(0, memcmp(buffer, buffer_cmp, SECTOR_SIZE)); + + /* + * TC003 + * write 11 blocks, odd blocks contain + * 0xff and even blocks contain 0x00 + */ + for (int i = 0; i < 11; i ++) { + if (i & 1) + memset(buffer, 0xff, SECTOR_SIZE); + else + memset(buffer, 0x00, SECTOR_SIZE); + ASSERT_EQ(SECTOR_SIZE, erofs_write_photon_file(buffer, SECTOR_SIZE * i, + SECTOR_SIZE, cache)); + } + cache->flush(); + for (int i = 0; i < 11; i ++) { + if (i & 1) + memset(buffer_cmp, 0xff, SECTOR_SIZE); + else + memset(buffer_cmp, 0x00, SECTOR_SIZE); + ASSERT_EQ(SECTOR_SIZE, erofs_read_photon_file(buffer, SECTOR_SIZE * i, + SECTOR_SIZE, cache)); + ASSERT_EQ(0, memcmp(buffer, buffer_cmp, SECTOR_SIZE)); + ASSERT_EQ(SECTOR_SIZE, img_file->pread(buffer, SECTOR_SIZE, + SECTOR_SIZE * i)); + ASSERT_EQ(0, memcmp(buffer, buffer_cmp, SECTOR_SIZE)); + } + + /* + * TC 004 + * test for non-aligned blocks + */ + for (int i = 0; i < 10; i ++) { + if (i & 1) + memset(buffer, 0xff, SECTOR_SIZE); + else + memset(buffer, 0x00, SECTOR_SIZE); + ASSERT_EQ(SECTOR_SIZE, erofs_write_photon_file(buffer, + HALF_SECTOR + SECTOR_SIZE * i, SECTOR_SIZE, cache)); + } + cache->flush(); + for (int i = 0; i < 10; i ++) { + if (i & 1) + memset(buffer_cmp, 0xff, SECTOR_SIZE); + else + memset(buffer_cmp, 0x00, SECTOR_SIZE); + ASSERT_EQ(SECTOR_SIZE, erofs_read_photon_file(buffer, + HALF_SECTOR + SECTOR_SIZE * i, SECTOR_SIZE, cache)); + ASSERT_EQ(0, memcmp(buffer, buffer_cmp, SECTOR_SIZE)); + ASSERT_EQ(SECTOR_SIZE, img_file->pread(buffer, SECTOR_SIZE, + HALF_SECTOR + SECTOR_SIZE * i)); + ASSERT_EQ(0, memcmp(buffer, buffer_cmp, SECTOR_SIZE)); + } + + /* + * TC 005 + * test for offset bigger than 2^32 (block-aligned) + */ + for (int i = 0; i < 11; i ++) { + if (i & 1) + memset(buffer, 0xff, SECTOR_SIZE); + else + memset(buffer, 0x00, SECTOR_SIZE); + ASSERT_EQ(SECTOR_SIZE, erofs_write_photon_file(buffer, + round_up_blk(BIG_OFFSET) + SECTOR_SIZE * i, SECTOR_SIZE, cache)); + } + cache->flush(); + for (int i = 0; i < 11; i ++) { + if (i & 1) + memset(buffer_cmp, 0xff, SECTOR_SIZE); + else + memset(buffer_cmp, 0x00, SECTOR_SIZE); + ASSERT_EQ(SECTOR_SIZE, erofs_read_photon_file(buffer, + round_up_blk(BIG_OFFSET) + SECTOR_SIZE * i, SECTOR_SIZE, cache)); + ASSERT_EQ(0, memcmp(buffer, buffer_cmp, SECTOR_SIZE)); + ASSERT_EQ(SECTOR_SIZE, img_file->pread(buffer, SECTOR_SIZE, + round_up_blk(BIG_OFFSET) + SECTOR_SIZE * i)); + ASSERT_EQ(0, memcmp(buffer, buffer_cmp, SECTOR_SIZE)); + } + + /* + * TC 006 + * test for offset bigger than 2^32 (non-aligned) + */ + for (int i = 0; i < 10; i ++) { + if (i & 1) + memset(buffer, 0xff, SECTOR_SIZE); + else + memset(buffer, 0x00, SECTOR_SIZE); + ASSERT_EQ(SECTOR_SIZE, erofs_write_photon_file(buffer, + BIG_OFFSET + SECTOR_SIZE * i, SECTOR_SIZE, cache)); + } + cache->flush(); + for (int i = 0; i < 10; i ++) { + if (i & 1) + memset(buffer_cmp, 0xff, SECTOR_SIZE); + else + memset(buffer_cmp, 0x00, SECTOR_SIZE); + ASSERT_EQ(SECTOR_SIZE, erofs_read_photon_file(buffer, + BIG_OFFSET + SECTOR_SIZE * i, SECTOR_SIZE, cache)); + ASSERT_EQ(0, memcmp(buffer, buffer_cmp, SECTOR_SIZE)); + ASSERT_EQ(SECTOR_SIZE, img_file->pread(buffer, SECTOR_SIZE, + BIG_OFFSET + SECTOR_SIZE * i)); + ASSERT_EQ(0, memcmp(buffer, buffer_cmp, SECTOR_SIZE)); + } + +#undef SECTOR_SIZE +#undef HALF_SECTOR +#undef BIG_OFFSET +#undef round_down_blk +#undef round_up_blk +} + int main(int argc, char **argv) { ::testing::InitGoogleTest(&argc, argv);