[feat.] Support uploading 'output' to registry while merging layers

Signed-off-by: Yifan Yuan <[email protected]>

src/overlaybd/lsmt/test/test.cpp

@@ -246,6 +246,12 @@ void test_combo(const IMemoryIndex *indexes[], size_t ni, const SegmentMapping s
         auto ret = memcmp(pm, stdrst, nrst * sizeof(stdrst[0]));
         EXPECT_EQ(ret, 0);
     }
+    LOG_INFO("make RO index of ci");
+    auto ro_idx = ci.make_read_only_index();
+    if (ro_idx->size() == nrst) {
+        auto ret = memcmp(ro_idx->buffer(), stdrst, nrst * sizeof(stdrst[0]));
+        EXPECT_EQ(ret, 0);
+    }
 
     ci.backing_index();
     // delete mi;
@@ -599,20 +605,27 @@ TEST_F(FileTest3, stack_files) {
     LOG_INFO("RO valid data: `", stat.valid_data_size);
     merged = lfs->open(fn_merged, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU);
     EXPECT_EQ(lower->flatten(merged), 0);
-    cout << "verifying flattened layer of lowers" << endl;
+    // auto localfile = lfs->open(fn_merged, O_RDONLY);
+    struct stat st;
+    merged->fstat(&st);
+    cout << "verifying flattened layer of lowers, st_size: "<<st.st_size << endl;
     verify_file(fn_merged);
     delete merged;
     cout << "generating a RW layer by randwrite()" << endl;
     auto upper = create_file_rw();
     auto file = stack_files(upper, lower, 0, true);
     randwrite(file, FLAGS_nwrites);
     verify_file(file);
-    cout << "verifying flattened layer of stacked layers" << endl;
 
     merged = lfs->open(fn_merged, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU);
     file->flatten(merged);
+    merged->fstat(&st);
+    cout << "verifying flattened layer of stacked layers, st_size: "<<st.st_size << endl;
+    merged->close();
+
     verify_file(fn_merged);
     delete file;
+    delete merged;
 }
 
 TEST_F(FileTest3, seek_data) {

src/overlaybd/registryfs/registryfs.h

@@ -43,10 +43,14 @@ photon::fs::IFileSystem *new_registryfs_v2(PasswordCB callback,
                                            const char *customized_ua = nullptr);
 
 photon::fs::IFile* new_registry_uploader(photon::fs::IFile *lfile,
-                                         std::string &upload_url,
-                                         std::string &username, std::string &password,
+                                         const std::string &upload_url,
+                                         const std::string &username,
+                                         const std::string &password,
                                          uint64_t timeout,
                                          ssize_t upload_bs = -1,
                                          const char *cert_file = nullptr,
                                          const char *key_file = nullptr);
+
+int registry_uploader_fini(photon::fs::IFile *uploader, std::string &digest);
+
 }

src/overlaybd/registryfs/registryfs_v2.cpp

@@ -17,6 +17,7 @@
 #include "registryfs.h"
 
 
+#include <cerrno>
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <unistd.h>
@@ -561,8 +562,8 @@ class RegistryUploader : public VirtualFile {
     uint64_t m_timeout = -1;
     estring m_token;
 
-    RegistryUploader(IFile *lfile, std::string &upload_url, std::string &username,
-                     std::string &password, uint64_t timeout, ssize_t upload_bs,
+    RegistryUploader(IFile *lfile, const std::string &upload_url, const std::string &username,
+                     const std::string &password, uint64_t timeout, ssize_t upload_bs,
                      photon::net::TLSContext *ctx)
         : m_local_file(lfile), m_origin_upload_url(upload_url), m_username(username), m_password(password),
           m_timeout(timeout), m_tls_ctx(ctx) {
@@ -829,15 +830,15 @@ class RegistryUploader : public VirtualFile {
             LOG_INFO(VALUE(m_upload_url));
             return 0;
         }
-        LOG_ERROR_RETURN(0, -1, "failed to get upload url, code=`", op.status_code);
+        LOG_ERROR_RETURN(0, -1, "failed to get upload url [`], code=`", m_upload_url, op.status_code);
     }
 
 protected:
     photon::net::TLSContext *m_tls_ctx;
 };
 
-IFile *new_registry_uploader(IFile *lfile, std::string &upload_url, std::string &username,
-                             std::string &password, uint64_t timeout, ssize_t upload_bs,
+IFile *new_registry_uploader(IFile *lfile, const std::string &upload_url, const std::string &username,
+                             const std::string &password, uint64_t timeout, ssize_t upload_bs,
                              const char *cert_file, const char *key_file) {
     auto ctx = new_tls_context_from_file(cert_file, key_file);
     if (!ctx) {
@@ -893,3 +894,14 @@ photon::net::TLSContext *new_tls_context_from_file(const char *cert_file, const
     }
     return photon::net::new_tls_context(cert_str.c_str(), key_str.c_str(), nullptr);
 }
+
+int registry_uploader_fini(photon::fs::IFile *uploader, std::string &digest) {
+    if (uploader == nullptr) {
+        LOG_ERRNO_RETURN(0, EINVAL, "invalid pointer");
+    }
+    if (uploader->fsync() < 0) {
+        LOG_ERRNO_RETURN(0, -1, "failed to upload blob");
+    }
+    digest = ((RegistryUploader *) uploader)->m_sha256sum;
+    return 0;
+}

src/tools/comm_func.cpp

@@ -20,6 +20,7 @@
 #include <photon/fs/subfs.h>
 #include <photon/fs/localfs.h>
 #include <photon/fs/extfs/extfs.h>
+#include "../overlaybd/registryfs/registryfs.h"
 #include "../image_service.h"
 #include "../image_file.h"
 #include "../overlaybd/tar/erofs/liberofs.h"
@@ -91,3 +92,21 @@ photon::fs::IFileSystem *create_erofs_fs(photon::fs::IFile *imgfile, uint64_t bl
 {
     return erofs_create_fs(imgfile, blksz);
 }
+IFile *create_uploader(ZFile::CompressArgs *zfile_args,  IFile *src,
+    const string &upload_url, const string &cred_file_path, uint64_t timeout, uint64_t upload_bs_KB){
+
+    zfile_args->overwrite_header = false;
+    LOG_INFO("upload to `", upload_url);
+    std::string username, password;
+    if (load_cred_from_file(cred_file_path, upload_url, username, password) <  0) {
+        fprintf(stderr, "failed to read upload cred file\n");
+        exit(-1);
+    }
+    auto upload_builder = new_registry_uploader(src, upload_url, username, password,
+        timeout*60*1000*1000, upload_bs_KB*1024);
+    if (upload_builder == nullptr) {
+        fprintf(stderr, "failed to init upload\n");
+        exit(-1);
+    }
+    return upload_builder;
+}

src/tools/comm_func.h

@@ -41,6 +41,9 @@ photon::fs::IFile *open_file(const char *fn, int flags, mode_t mode = 0, photon:
 int create_overlaybd(const std::string &srv_config, const std::string &dev_config,
     ImageService *&, photon::fs::IFile *&);
 
+photon::fs::IFile *create_uploader(ZFile::CompressArgs *zfile_args,  IFile *src,
+    const std::string &upload_url, const std::string &cred_file_path, uint64_t timeout_minute, uint64_t upload_bs_KB);
+
 photon::fs::IFileSystem *create_ext4fs(photon::fs::IFile *imgfile, bool mkfs,
     bool enable_buffer, const char* root);
 

src/tools/overlaybd-commit.cpp

@@ -21,19 +21,19 @@
 #include "../overlaybd/lsmt/file.h"
 #include "../overlaybd/zfile/zfile.h"
 #include "../overlaybd/tar/tar_file.h"
-#include "../overlaybd/registryfs/registryfs.h"
-#include "../image_service.h"
 #include <errno.h>
 #include <fcntl.h>
 #include <inttypes.h>
-#include <memory>
 #include <string>
 #include <stdio.h>
 #include <stdlib.h>
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <unistd.h>
 #include "CLI11.hpp"
+#include "comm_func.h"
+#include "../overlaybd/registryfs/registryfs.h"
+
 
 using namespace std;
 using namespace LSMT;
@@ -85,7 +85,7 @@ int main(int argc, char **argv) {
     app.add_option("--compress_threads", compress_threads, "compress threads")->default_val(1);
     app.add_flag("--verbose", verbose, "output debug info")->default_val(false);
     app.add_option("--upload", upload_url, "registry upload url");
-    app.add_option("--upload_bs", upload_bs, "block size for upload, in KB");
+    app.add_option("--upload_bs", upload_bs, "block size for upload, in KB")->default_val(1024);
     app.add_option("--cred_file_path", cred_file_path, "cred file path for registryfs")->type_name("FILEPATH")->check(CLI::ExistingFile);
 
     CLI11_PARSE(app, argc, argv);
@@ -164,21 +164,9 @@ int main(int argc, char **argv) {
         zfile_args->overwrite_header = true;
 
         if (!upload_url.empty()) {
-            zfile_args->overwrite_header = false;
-            LOG_INFO("upload to `", upload_url);
-            std::string username, password;
-            if (load_cred_from_file(cred_file_path, upload_url, username, password) <  0) {
-                fprintf(stderr, "failed to read upload cred file\n");
-                exit(-1);
-            }
-            upload_builder = new_registry_uploader(out, upload_url, username, password, 2UL*60*1000*1000, upload_bs*1024);
-            if (upload_builder == nullptr) {
-                fprintf(stderr, "failed to init upload\n");
-                exit(-1);
-            }
+            upload_builder = create_uploader(zfile_args, out, upload_url, cred_file_path, 2, upload_bs);
             out = upload_builder;
         }
-
         zfile_builder = ZFile::new_zfile_builder(out, zfile_args, false);
         out = zfile_builder;
     } else {
@@ -213,12 +201,12 @@ int main(int argc, char **argv) {
     if (zfile_args) {
         delete zfile_args;
     }
-
-    if (upload_builder != nullptr && upload_builder->fsync() < 0) {
-        fprintf(stderr, "failed to commit or upload");
-        return -1;
-    }
-
+    string digest = "";
+    if (upload_builder != nullptr && registry_uploader_fini(upload_builder, digest) != 0){
+        fprintf(stderr, "failed to commit or upload\n");
+        exit(-1);
+    };
+    fprintf(stderr, "%s\n", digest.c_str());
     delete upload_builder;
     delete fout;
     delete fin;

src/tools/overlaybd-merge.cpp

@@ -20,15 +20,9 @@
 #include <photon/fs/virtual-file.h>
 #include <photon/fs/extfs/extfs.h>
 #include <photon/photon.h>
-#include "../overlaybd/lsmt/file.h"
 #include "../overlaybd/zfile/zfile.h"
-#include "../overlaybd/tar/libtar.h"
-#include "../overlaybd/gzindex/gzfile.h"
-#include "../overlaybd/gzip/gz.h"
-#include <errno.h>
 #include <fcntl.h>
 #include <inttypes.h>
-#include <memory>
 #include <string>
 #include <stdio.h>
 #include <stdlib.h>
@@ -37,6 +31,7 @@
 #include <unistd.h>
 #include "../image_service.h"
 #include "../image_file.h"
+#include "../overlaybd/registryfs/registryfs.h"
 #include "CLI11.hpp"
 #include "comm_func.h"
 #include "sha256file.h"
@@ -79,15 +74,19 @@ class FIFOFile : public VirtualReadOnlyFile {
 };
 
 int main(int argc, char **argv) {
-    std::string image_config_path, input_path, output, config_path, sha256_checksum;
-    string tarheader;
-    bool zfile = false, mkfs = false, verbose = false, raw = false;
+    std::string image_config_path, input_path, output, config_path, sha256_checksum, upload_url, cred_file_path;
+    int upload_bs = 0;
+    bool zfile = false, verbose = false;
+    IFile *upload_builder = nullptr;
 
     CLI::App app{"this is overlaybd-merge, merge multiple overlaybd layers into a single."};
 
     app.add_flag("--verbose", verbose, "output debug info")->default_val(false);
     app.add_option("--service_config_path", config_path, "overlaybd image service config path")->type_name("FILEPATH")->check(CLI::ExistingFile)->default_val("/etc/overlaybd/overlaybd.json");
     app.add_flag("--compress", zfile, "do zfile compression for the output layer")->default_val(true);
+    app.add_option("--upload", upload_url, "upload to remote registry URL while generating merged layer.");
+    app.add_option("--upload_bs", upload_bs, "block size for upload, in KB")->default_val(65536);
+    app.add_option("--cred_file_path", cred_file_path, "cred file path for registryfs")->type_name("FILEPATH")->check(CLI::ExistingFile);
 
     app.add_option("image_config_path", image_config_path, "overlaybd image config path")->type_name("FILEPATH")->check(CLI::ExistingFile)->required();
     app.add_option("output", output, "compacted layer path")->type_name("FILEPATH");
@@ -117,7 +116,15 @@ int main(int argc, char **argv) {
     }
     DEFER(delete rst);
     if (zfile) {
-        rst = ZFile::new_zfile_builder(rst);
+        ZFile::CompressOptions opt;
+        opt.verify = 1;
+        ZFile::CompressArgs zfile_args(opt);
+         if (!upload_url.empty()) {
+            LOG_INFO("upload url detected, create uploader");
+            upload_builder = create_uploader(&zfile_args, rst, upload_url, cred_file_path, 2 /* 2min timeout foreach block */, upload_bs);
+            rst = upload_builder;
+        }
+        rst = ZFile::new_zfile_builder(rst, &zfile_args, true);
         if (rst == nullptr) {
             fprintf(stderr, "failed to create zfile\n");
             exit(-1);
@@ -127,6 +134,12 @@ int main(int argc, char **argv) {
         fprintf(stderr, "failed to compact\n");
         exit(-1);
     }
-
+    string digest = "";
+    if (upload_builder != nullptr && registry_uploader_fini(upload_builder, digest) != 0){
+        fprintf(stderr, "failed to upload\n");
+        exit(-1);
+    };
+    fprintf(stderr, "%s\n", digest.c_str());
+    delete upload_builder;
     return 0;
 }