Merge pull request #320 from salvete2022/dev_erofs

add support for tarerofs for TurboOCI

src/overlaybd/tar/tarerofs.cpp

@@ -0,0 +1,135 @@
+#include "tarerofs.h"
+#include <stdio.h>
+#include <string.h>
+#include <sys/param.h>
+#include <sys/types.h>
+#include <sys/sysmacros.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <cstdio>
+#include <utime.h>
+#include <set>
+#include <string>
+#include <photon/fs/localfs.h>
+#include <photon/fs/path.h>
+#include <photon/common/string_view.h>
+#include <photon/fs/filesystem.h>
+#include <photon/common/alog.h>
+#include <photon/fs/fiemap.h>
+#include "../lsmt/file.h"
+#include "../lsmt/index.h"
+
+#define TAREROFS_BLOCK_SIZE 4096
+#define __stringify_1(x...)	#x
+#define __stringify(x...)	__stringify_1(x)
+
+int TarErofs::extract_all() {
+    ssize_t read;
+    struct stat st;
+    char buf[128*1024];
+    char base_path[64] = "/tmp/tarerofs_base_XXXXXX";
+    char command_line[256] = "mkfs.erofs --tar=0,upper.map,1073741824 -b" __stringify(TAREROFS_BLOCK_SIZE) " --aufs";
+    const char command_line2[] = " upper.erofs";
+
+    FILE *fp;
+    photon::fs::IFile *rawfs = nullptr;
+    int status;
+    uint64_t blkaddr, toff, metasize;
+    uint32_t nblocks;
+
+    if (!meta_only) {
+         LOG_ERROR("currently EROFS supports fastoci mode only", strerror(errno));
+         return -1;
+    }
+
+    if (fs_base_file->fstat(&st)>= 0 && st.st_blocks > 0) {
+        int fd = mkstemp(base_path);
+        if (fd < 0) {
+            LOG_ERROR("cannot generate a temporary file to dump overlaybd disk");
+            return -1;
+        }
+        std::strcat(command_line, " --base ");
+        std::strcat(command_line, base_path);
+
+        if (fs_base_file->pread(&metasize, sizeof(metasize), 0) !=
+            sizeof(metasize)) {
+            LOG_ERROR("failed to read EROFS metadata size");
+            return -1;
+        }
+
+        while (metasize) {
+            int count = std::min(sizeof(buf), metasize);
+            read = fs_base_file->read(buf, count);
+            if (read != count ||
+                write(fd, buf, read) != read) {
+                read = -1;
+                break;
+            }
+            metasize -= read;
+        }
+        close(fd);
+        if (read < 0) {
+            return -1;
+        }
+    }
+    std::strcat(command_line, command_line2);
+    fp = popen(command_line, "w");
+    if (fp == NULL) {
+         LOG_ERROR("failed to execute mkfs.erofs", strerror(errno));
+         return -1;
+    }
+
+    while ((read = file->read(buf, sizeof(buf))) > 0) {
+        if (fwrite(buf, read, 1, fp) != 1) {
+            read = -1;
+            break;
+        }
+    }
+    status = pclose(fp);
+
+    if (read < 0 || status) {
+        return -1;
+    }
+
+    rawfs = photon::fs::open_localfile_adaptor("upper.erofs", O_RDONLY, 0644);
+    DEFER({ delete rawfs; });
+
+    /* write to LSMT */
+    metasize = rawfs->lseek(0, SEEK_END);
+    rawfs->lseek(0, 0);
+    fout->lseek(0, 0);
+    while ((read = rawfs->read(buf, sizeof(buf))) > 0) {
+        if (metasize) {                // since pwrite < 512 is unsupported.
+            *(uint64_t *)buf = metasize;
+            metasize = 0;
+        }
+
+        if (fout->write(buf, read) != read) {
+            read = -1;
+            break;
+        }
+        metasize += read;
+    }
+
+
+    /* write mapfile */
+    fp = fopen("upper.map", "r");
+    if (fp == NULL) {
+       LOG_ERROR("unable to get upper.map, ignored");
+       return -1;
+    }
+    while (fscanf(fp, "%" PRIx64" %x %" PRIx64 "\n", &blkaddr, &nblocks, &toff) >= 3) {
+        LSMT::RemoteMapping lba;
+        lba.offset = blkaddr * TAREROFS_BLOCK_SIZE;
+        lba.count = nblocks * TAREROFS_BLOCK_SIZE;
+        lba.roffset = toff;
+        int nwrite = fout->ioctl(LSMT::IFileRW::RemoteData, lba);
+        if ((unsigned) nwrite != lba.count) {
+            LOG_ERRNO_RETURN(0, -1, "failed to write lba");
+        }
+    }
+    fclose(fp);
+    return 0;
+}

src/overlaybd/tar/tarerofs.h

@@ -0,0 +1,36 @@
+#pragma once
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <tar.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/param.h>
+#include <pwd.h>
+#include <grp.h>
+#include <string.h>
+#include <photon/fs/filesystem.h>
+#include <photon/fs/fiemap.h>
+#include <photon/common/string_view.h>
+#include <set>
+#include <vector>
+#include <string>
+#include <list>
+#include <map>
+
+class TarErofs {
+public:
+    TarErofs(photon::fs::IFile *file, photon::fs::IFile *target, uint64_t fs_blocksize = 4096,
+          photon::fs::IFile *bf = nullptr, bool meta_only = true)
+        : file(file), fout(target), fs_base_file(bf), meta_only(meta_only) {}
+
+    int extract_all();
+
+private:
+    photon::fs::IFile *file = nullptr;     // source
+    photon::fs::IFile *fout = nullptr; // target
+    photon::fs::IFile *fs_base_file = nullptr;
+    bool meta_only;
+    std::set<std::string> unpackedPaths;
+    std::list<std::pair<std::string, int>> dirs; // <path, utime>
+};

src/tools/turboOCI-apply.cpp

@@ -23,6 +23,7 @@
 #include "../overlaybd/lsmt/file.h"
 #include "../overlaybd/zfile/zfile.h"
 #include "../overlaybd/tar/libtar.h"
+#include "../overlaybd/tar/tarerofs.h"
 #include "../overlaybd/gzindex/gzfile.h"
 #include "../overlaybd/gzip/gz.h"
 #include <errno.h>
@@ -60,12 +61,13 @@ int dump_tar_headers(IFile *src_file, const string &out) {
 }
 
 int main(int argc, char **argv) {
-    std::string image_config_path, input_path, gz_index_path, config_path;
+    std::string image_config_path, input_path, gz_index_path, config_path, fstype;
     bool raw = false, mkfs = false, verbose = false;
     bool export_tar_headers = false, import_tar_headers = false;
 
     CLI::App app{"this is turboOCI-apply, apply OCIv1 tar layer to 'Overlaybd-TurboOCI v1' format"};
     app.add_flag("--mkfs", mkfs, "mkfs before apply")->default_val(false);
+    app.add_option("--fstype", fstype, "filesystem type")->default_val("ext4");
     app.add_flag("--verbose", verbose, "output debug info")->default_val(false);
     app.add_option("--service_config_path", config_path, "overlaybd image service config path")
         ->type_name("FILEPATH")
@@ -124,18 +126,29 @@ int main(int argc, char **argv) {
     });
 
     // for now, buffer_file can't be used with turboOCI
-    auto target = create_ext4fs(imgfile, mkfs, false, "/");
-    DEFER({ delete target; });
-
-    photon::fs::IFile *base_file = raw ? nullptr : ((ImageFile *)imgfile)->get_base();
-    bool gen_turboOCI = true;
-    int option = (import_tar_headers ? TAR_IGNORE_CRC : 0);
-    auto tar =
-        new UnTar(src_file, target, option, 4096, base_file, gen_turboOCI, import_tar_headers);
-
-    if (tar->extract_all() < 0) {
-        fprintf(stderr, "failed to extract\n");
-        exit(-1);
+    if (fstype == "erofs") {
+        photon::fs::IFile* base_file = raw ? nullptr : ((ImageFile *)imgfile)->get_base();
+
+        auto tar =
+           new TarErofs(src_file, imgfile, 4096, base_file, true);
+
+        if (tar->extract_all() < 0) {
+            fprintf(stderr, "failed to extract\n");
+            exit(-1);
+        }
+    } else {
+        auto target = create_ext4fs(imgfile, mkfs, false, "/");
+        DEFER({ delete target; });
+
+        photon::fs::IFile *base_file = raw ? nullptr : ((ImageFile *)imgfile)->get_base();
+        bool gen_turboOCI = true;
+        int option = (import_tar_headers ? TAR_IGNORE_CRC : 0);
+        auto tar =
+            new UnTar(src_file, target, option, 4096, base_file, gen_turboOCI, import_tar_headers);
+        if (tar->extract_all() < 0) {
+            fprintf(stderr, "failed to extract\n");
+            exit(-1);
+        }
     }
     fprintf(stdout, "turboOCI-apply done\n");
     return 0;