diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5340215a..95e7afe4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -53,6 +53,7 @@ jobs: RELEASE_VERSION="${new_version}rc" #1.1.4rc fi RELEASE_VERSION=${RELEASE_VERSION#v} + COMMIT_ID="${RELEASE_VERSION}-$(git rev-parse --short HEAD)" RELEASE_NO="$(date +%Y%m%d).$(git rev-parse --short HEAD)" echo "RELEASE_VERSION=${RELEASE_VERSION} RELEASE_NO=${RELEASE_NO}" @@ -67,7 +68,7 @@ jobs: echo "OS=${OS}" - docker buildx build --build-arg BUILD_IMAGE=${BUILD_IMAGE} --build-arg OS=${OS} --build-arg RELEASE_VERSION=${RELEASE_VERSION} --build-arg RELEASE_NO=${RELEASE_NO} -f .github/workflows/release/Dockerfile --platform=${{ matrix.platforms }} -o releases/ . + docker buildx build --build-arg COMMIT_ID=${COMMIT_ID} --build-arg BUILD_IMAGE=${BUILD_IMAGE} --build-arg OS=${OS} --build-arg RELEASE_VERSION=${RELEASE_VERSION} --build-arg RELEASE_NO=${RELEASE_NO} -f .github/workflows/release/Dockerfile --platform=${{ matrix.platforms }} -o releases/ . # remove unused package if [[ "${OS}" =~ "ubuntu" ]]; then diff --git a/.github/workflows/release/Dockerfile b/.github/workflows/release/Dockerfile index 1f507600..ae03f826 100644 --- a/.github/workflows/release/Dockerfile +++ b/.github/workflows/release/Dockerfile @@ -19,7 +19,8 @@ COPY . . ARG OS ARG RELEASE_VERSION ARG RELEASE_NO -RUN ls -l /src && chmod 755 .github/workflows/release/build.sh && .github/workflows/release/build.sh ${OS} ${RELEASE_VERSION} ${RELEASE_NO} +ARG COMMIT_ID +RUN ls -l /src && chmod 755 .github/workflows/release/build.sh && .github/workflows/release/build.sh ${OS} ${RELEASE_VERSION} ${RELEASE_NO} ${COMMIT_ID} FROM scratch AS release COPY --from=builder /src/build/overlaybd-*.* / diff --git a/.github/workflows/release/build.sh b/.github/workflows/release/build.sh index c5b8a71a..b7be4901 100644 --- a/.github/workflows/release/build.sh +++ b/.github/workflows/release/build.sh @@ -18,6 +18,7 @@ OS=${1} PACKAGE_VERSION=${2} RELEASE_NO=${3} +COMMIT_ID=${4} ARCH=`uname -m` BUILD_TYPE="Release" COMPILER="" @@ -75,6 +76,6 @@ fi # Build mkdir build cd build -${CMAKE} .. -DPACKAGE_VERSION=${PACKAGE_VERSION} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DBUILD_TESTING=0 -DENABLE_DSA=0 -DENABLE_ISAL=0 ${PACKAGE_RELEASE} ${COMPILER} +${CMAKE} .. -DOBD_VER="overlaybd/${COMMIT_ID}" -DPACKAGE_VERSION=${PACKAGE_VERSION} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DBUILD_TESTING=0 -DENABLE_DSA=0 -DENABLE_ISAL=0 ${PACKAGE_RELEASE} ${COMPILER} make -j8 ${CPACK} --verbose diff --git a/CMakeLists.txt b/CMakeLists.txt index 10fa7e20..271ea536 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,13 +12,15 @@ if (NOT (${ARCH} STREQUAL x86_64) AND NOT (${ARCH} STREQUAL aarch64) AND NOT (${ message(FATAL_ERROR "Unknown CPU architecture ${ARCH}") endif () +option(OBD_VER "Overlaybd version" "overlaybd/0.0.0-undefined") + set(LIBRARY_OUTPUT_PATH "${CMAKE_SOURCE_DIR}/build/output") set(EXECUTABLE_OUTPUT_PATH "${CMAKE_SOURCE_DIR}/build/output") set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/CMake") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fpic -Wall -Werror=sign-compare") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fpic -Wall -Werror=sign-compare") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fpic -Wall -Werror=sign-compare -DOVERLAYBD_VER=${OBD_VER}") if (${ARCH} STREQUAL aarch64) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsigned-char") @@ -39,6 +41,7 @@ endif() set(CMAKE_CXX_STANDARD 14) set(CMAKE_CXX_STANDARD_REQUIRED on) set(ENABLE_MIMIC_VDSO off) + option(BUILD_CURL_FROM_SOURCE "Compile static libcurl" off) option(ORIGIN_EXT2FS "Use original libext2fs" off) find_package(photon REQUIRED) diff --git a/src/config.h b/src/config.h index acbfc148..9af34d90 100644 --- a/src/config.h +++ b/src/config.h @@ -19,6 +19,10 @@ #include #include "overlaybd/config_util.h" +#define MACROTOSTR(x) #x +#define PRINTMACRO(x) MACROTOSTR(x) +static const char OVERLAYBD_VERSION[] = PRINTMACRO(OVERLAYBD_VER); + namespace ImageConfigNS { const int MAX_LAYER_CNT = 256; diff --git a/src/image_file.cpp b/src/image_file.cpp index e22da6d2..ad4475ee 100644 --- a/src/image_file.cpp +++ b/src/image_file.cpp @@ -102,7 +102,7 @@ IFile *ImageFile::__open_ro_target_file(const std::string &path) { } IFile *ImageFile::__open_ro_target_remote(const std::string &dir, const std::string &data_digest, - const uint64_t size, int layer_index) { + const uint64_t size, int layer_index) { std::string url; if (conf.repoBlobUrl() == "") { @@ -153,8 +153,7 @@ IFile *ImageFile::__open_ro_remote(const std::string &dir, const std::string &di LOG_ERROR_RETURN(0, nullptr, "empty repoBlobUrl for remote layer"); } estring url = estring().appends("/", conf.repoBlobUrl(), - (conf.repoBlobUrl().back() != '/') ? "/" : "", - digest); + (conf.repoBlobUrl().back() != '/') ? "/" : "", digest); LOG_INFO("open file from remotefs: `, size: `", url, size); IFile *remote_file = image_service.global_fs.remote_fs->open(url.c_str(), O_RDONLY); @@ -164,8 +163,14 @@ IFile *ImageFile::__open_ro_remote(const std::string &dir, const std::string &di set_failed("failed to open remote file ", url, ": ", err_msg); LOG_ERRNO_RETURN(0, nullptr, "failed to open remote file `: `", url, err_msg); } - remote_file->ioctl(SET_SIZE, size); - remote_file->ioctl(SET_LOCAL_DIR, dir); + if (!dir.empty()) { + remote_file->ioctl(SET_SIZE, size); + remote_file->ioctl(SET_LOCAL_DIR, dir); + } else { + LOG_WARN( + "local dir of layer %d (%s) didn't set, skip background anyway", + layer_index, digest.c_str()); + } IFile *tar_file = new_tar_file_adaptor(remote_file); if (!tar_file) { @@ -183,15 +188,15 @@ IFile *ImageFile::__open_ro_remote(const std::string &dir, const std::string &di LOG_ERRNO_RETURN(0, nullptr, "failed to open switch file `", url); } - if (conf.HasMember("download") && conf.download().enable() == 1) { + if (conf.HasMember("download") && conf.download().enable() == 1 && !dir.empty()) { // download from registry, verify sha256 after downloaded. IFile *srcfile = image_service.global_fs.srcfs->open(url.c_str(), O_RDONLY); if (srcfile == nullptr) { LOG_WARN("failed to open source file, ignore download"); } else { - BKDL::BkDownload *obj = - new BKDL::BkDownload(switch_file, srcfile, size, dir, digest, url, m_status, - conf.download().maxMBps(), conf.download().tryCnt(), conf.download().blockSize()); + BKDL::BkDownload *obj = new BKDL::BkDownload( + switch_file, srcfile, size, dir, digest, url, m_status, conf.download().maxMBps(), + conf.download().tryCnt(), conf.download().blockSize()); LOG_DEBUG("add to download list for `", dir); dl_list.push_back(obj); } @@ -209,8 +214,8 @@ void ImageFile::start_bk_dl_thread() { uint64_t extra_range = conf.download().delayExtra(); extra_range = (extra_range <= 0) ? 30 : extra_range; uint64_t delay_sec = (rand() % extra_range) + conf.download().delay(); - LOG_INFO("background download is enabled, delay `, maxMBps `, tryCnt `, blockSize `", - delay_sec, conf.download().maxMBps(), conf.download().tryCnt(), conf.download().blockSize()); + LOG_INFO("background download is enabled, delay `, maxMBps `, tryCnt `, blockSize `", delay_sec, + conf.download().maxMBps(), conf.download().tryCnt(), conf.download().blockSize()); dl_thread_jh = photon::thread_enable_join( photon::thread_create11(&BKDL::bk_download_proc, dl_list, delay_sec, m_status)); } @@ -256,30 +261,30 @@ void *do_parallel_open_files(ImageFile *imgfile, ParallelOpenTask &tm) { return nullptr; } -int ImageFile::open_lower_layer(IFile *&file, ImageConfigNS::LayerConfig &layer, - int index) { - std::string opened; +IFile *ImageFile::open_localfile(ImageConfigNS::LayerConfig &layer, std::string &opened) { if (layer.file() != "") { opened = layer.file(); - file = __open_ro_file(opened); - } else { - // open downloaded blob or remote blob - if (BKDL::check_downloaded(layer.dir())) { - opened = layer.dir() + "/" + COMMIT_FILE_NAME; - file = __open_ro_file(opened); - } else { - auto sealed = layer.dir() + "/" + SEALED_FILE_NAME; - if (::access(sealed.c_str(), 0) == 0) { - // open sealed blob - opened = sealed; - file = __open_ro_file(opened); - } else { - opened = layer.digest(); - file = __open_ro_remote(layer.dir(), layer.digest(), layer.size(), index); - } - } + return __open_ro_file(opened); + } + if (BKDL::check_downloaded(layer.dir())) { + opened = layer.dir() + "/" + COMMIT_FILE_NAME; + return __open_ro_file(opened); + } + auto sealed = layer.dir() + "/" + SEALED_FILE_NAME; + if (::access(sealed.c_str(), 0) == 0) { + // open sealed blob + opened = sealed; + return __open_ro_file(opened); + } + return nullptr; +} +int ImageFile::open_lower_layer(IFile *&file, ImageConfigNS::LayerConfig &layer, int index) { + std::string opened; + file = open_localfile(layer, opened); // try to open localfile if downloaded + if (file == nullptr) { + opened = layer.digest(); + file = __open_ro_remote(layer.dir(), layer.digest(), layer.size(), index); } - if (file == nullptr) { return -1; } @@ -305,8 +310,8 @@ int ImageFile::open_lower_layer(IFile *&file, ImageConfigNS::LayerConfig &layer, } target_file = new_gzfile(target_file, gz_index, true); if (image_service.global_conf.gzipCacheConfig().enable() && layer.targetDigest() != "") { - target_file = image_service.global_fs.gzcache_fs-> - open_cached_gzip_file(target_file, layer.targetDigest().c_str()); + target_file = image_service.global_fs.gzcache_fs->open_cached_gzip_file( + target_file, layer.targetDigest().c_str()); } } if (target_file != nullptr) { @@ -395,7 +400,8 @@ LSMT::IFileRW *ImageFile::open_upper(ImageConfigNS::UpperConfig &upper) { } if (upper.target() != "") { - LOG_INFO("turboOCIv1 upper layer : `, `, `, `", upper.index(), upper.data(), upper.target()); + LOG_INFO("turboOCIv1 upper layer : `, `, `, `", upper.index(), upper.data(), + upper.target()); target_file = open_localfile_adaptor(upper.target().c_str(), O_RDWR, 0644); if (!target_file) { LOG_ERROR("open(`,flags), `:`", upper.target(), errno, strerror(errno)); @@ -411,16 +417,16 @@ LSMT::IFileRW *ImageFile::open_upper(ImageConfigNS::UpperConfig &upper) { } ret = LSMT::open_warpfile_rw(idx_file, data_file, target_file, true); if (!ret) { - LOG_ERROR("LSMT::open_warpfile_rw(`,`,`,`) return NULL", - (uint64_t)data_file, (uint64_t)idx_file, (uint64_t)target_file, true); + LOG_ERROR("LSMT::open_warpfile_rw(`,`,`,`) return NULL", (uint64_t)data_file, + (uint64_t)idx_file, (uint64_t)target_file, true); goto ERROR_EXIT; } } else { LOG_INFO("overlaybd upper layer : ` , `", upper.index(), upper.data()); ret = LSMT::open_file_rw(data_file, idx_file, true); if (!ret) { - LOG_ERROR("LSMT::open_file_rw(`,`,`) return NULL", - (uint64_t)data_file, (uint64_t)idx_file, true); + LOG_ERROR("LSMT::open_file_rw(`,`,`) return NULL", (uint64_t)data_file, + (uint64_t)idx_file, true); goto ERROR_EXIT; } } @@ -453,8 +459,7 @@ int ImageFile::init_image_file() { LOG_INFO("Acceleration layer found at `, ignore the last lower", accel_layer); std::string trace_file = accel_layer + "/trace"; - if (Prefetcher::detect_mode(trace_file) == - Prefetcher::Mode::Replay) { + if (Prefetcher::detect_mode(trace_file) == Prefetcher::Mode::Replay) { m_prefetcher = new_prefetcher(trace_file, concurrency); } @@ -519,8 +524,8 @@ void ImageFile::set_auth_failed() { } } -template -void ImageFile::set_failed(const Ts&...xs) { +template +void ImageFile::set_failed(const Ts &...xs) { if (m_status == 0) // only set exit in image boot phase { m_status = -1; diff --git a/src/image_file.h b/src/image_file.h index 344487e5..ac7a6738 100644 --- a/src/image_file.h +++ b/src/image_file.h @@ -119,6 +119,8 @@ class ImageFile : public photon::fs::ForwardFile { template void set_failed(const Ts&...xs); LSMT::IFileRO *open_lowers(std::vector &, bool &); LSMT::IFileRW *open_upper(ImageConfigNS::UpperConfig &); + + IFile *open_localfile(ImageConfigNS::LayerConfig &layer, std::string &opened); IFile *__open_ro_file(const std::string &); IFile *__open_ro_target_file(const std::string &); IFile *__open_ro_remote(const std::string &dir, const std::string &, const uint64_t, int); diff --git a/src/main.cpp b/src/main.cpp index 982fa049..652b1a71 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -456,6 +456,8 @@ int main(int argc, char **argv) { LOG_INFO("reset netlink done"); } + LOG_INFO("current version: `", OVERLAYBD_VERSION); + std::vector handlers; struct tcmulib_handler overlaybd_handler; overlaybd_handler.name = "Handler for overlaybd devices";