diff --git a/.github/workflows/clang_linux.yml b/.github/workflows/clang_linux.yml index 0794d4bb8b..b0cfe2cdb4 100644 --- a/.github/workflows/clang_linux.yml +++ b/.github/workflows/clang_linux.yml @@ -31,6 +31,10 @@ jobs: id: regular PROJ_CMAKE_BUILD_OPTIONS: "" + - name: EMBED_RESOURCE_FILES + id: EMBED_RESOURCE_FILES + PROJ_CMAKE_BUILD_OPTIONS: "-DEMBED_RESOURCE_FILES=ON" + - name: Without TIFF id: without_tiff PROJ_CMAKE_BUILD_OPTIONS: "-DENABLE_TIFF=OFF" diff --git a/.github/workflows/doc_checks.yml b/.github/workflows/doc_checks.yml index 704846b814..4d583db2f2 100644 --- a/.github/workflows/doc_checks.yml +++ b/.github/workflows/doc_checks.yml @@ -26,6 +26,11 @@ jobs: steps: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - name: Update sphinx-rtd-theme + shell: bash -l {0} + run: | + pip install -U "sphinx-rtd-theme>=3.0.0" + - name: Print versions shell: bash -l {0} run: | diff --git a/.github/workflows/fedora_rawhide.yml b/.github/workflows/fedora_rawhide.yml new file mode 100644 index 0000000000..99b6caf3f3 --- /dev/null +++ b/.github/workflows/fedora_rawhide.yml @@ -0,0 +1,37 @@ +name: Fedora Rawhide + +on: + push: + paths-ignore: + - 'docs/**' + pull_request: + paths-ignore: + - 'docs/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + + fedora_rawhide: + runs-on: ubuntu-latest + if: "!contains(github.event.head_commit.message, '[ci skip]') && !contains(github.event.head_commit.message, '[skip ci]')" + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Cache + uses: actions/cache@v4 + id: cache + with: + path: | + ${{ github.workspace }}/ccache.tar.gz + key: ${{ runner.os }}-cache-fedora_rawhide-${{ github.run_id }} + restore-keys: ${{ runner.os }}-cache-fedora_rawhide- + + - name: Run + run: docker run -e CI -e WORK_DIR="$PWD" -v $PWD:$PWD fedora:rawhide $PWD/.github/workflows/fedora_rawhide/start.sh diff --git a/.github/workflows/fedora_rawhide/start.sh b/.github/workflows/fedora_rawhide/start.sh new file mode 100755 index 0000000000..ff219fb606 --- /dev/null +++ b/.github/workflows/fedora_rawhide/start.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +set -e + +dnf install -y cmake clang ccache ninja-build sqlite-devel libtiff-devel libcurl-devel diffutils + +cd "$WORK_DIR" + +if test -f "$WORK_DIR/ccache.tar.gz"; then + echo "Restoring ccache..." + (cd $HOME && tar xzf "$WORK_DIR/ccache.tar.gz") +fi + +export CCACHE_CPP2=yes + +ccache -M 500M +ccache -s + +mkdir build +cd build +CC=clang CXX=clang++ cmake .. \ + -DEMBED_RESOURCE_FILES=ON -DUSE_ONLY_EMBEDDED_RESOURCE_FILES=ON -DUSE_CCACHE=ON -DPROJ_DB_CACHE_DIR=$HOME/.ccache .. +make -j$(nproc) +ctest -j$(nproc) +cd .. + +ccache -s + +echo "Saving ccache..." +rm -f "$WORK_DIR/ccache.tar.gz" +(cd $HOME && tar czf "$WORK_DIR/ccache.tar.gz" .cache) diff --git a/CMakeLists.txt b/CMakeLists.txt index 98a278631d..844c482ad4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -349,6 +349,45 @@ else() message(STATUS "Testing disabled") endif() +################################################################################ +# Whether we should embed resources +################################################################################ + +function (is_sharp_embed_available res) + if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.21 AND + ((CMAKE_C_COMPILER_ID STREQUAL "GNU") OR (CMAKE_C_COMPILER_ID STREQUAL "Clang"))) + # CMAKE_C_STANDARD=23 only supported since CMake 3.21 + set(TEST_SHARP_EMBED + "static const unsigned char embedded[] = {\n#embed __FILE__\n};\nint main() { (void)embedded; return 0;}" + ) + set(CMAKE_C_STANDARD_BACKUP "${CMAKE_C_STANDARD}") + set(CMAKE_C_STANDARD "23") + check_c_source_compiles("${TEST_SHARP_EMBED}" _TEST_SHARP_EMBED) + set(CMAKE_C_STANDARD "${CMAKE_C_STANDARD_BACKUP}") + if (_TEST_SHARP_EMBED) + set(${res} ON PARENT_SCOPE) + else() + set(${res} OFF PARENT_SCOPE) + endif() + else() + set(${res} OFF PARENT_SCOPE) + endif() +endfunction() + +is_sharp_embed_available(IS_SHARP_EMBED_AVAILABLE_RES) +if (NOT BUILD_SHARED_LIBS) + set(DEFAULT_EMBED_RESOURCE_FILES ON) +else() + set(DEFAULT_EMBED_RESOURCE_FILES OFF) +endif() +option(EMBED_RESOURCE_FILES "Whether resource files (limited to proj.db) should be embedded into the PROJ library" ${DEFAULT_EMBED_RESOURCE_FILES}) + +option(USE_ONLY_EMBEDDED_RESOURCE_FILES "Whether embedded resource files (limited to proj.db) should be used (should nominally be used together with EMBED_RESOURCE_FILES=ON, otherwise this will result in non-functional builds)" OFF) + +if (USE_ONLY_EMBEDDED_RESOURCE_FILES AND NOT EMBED_RESOURCE_FILES) + message(WARNING "USE_ONLY_EMBEDDED_RESOURCE_FILES=ON set but EMBED_RESOURCE_FILES=OFF: some drivers will lack required resource files") +endif() + ################################################################################ # Build configured components ################################################################################ @@ -432,3 +471,5 @@ endif() configure_file(cmake/uninstall.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/proj_uninstall.cmake @ONLY) add_custom_target(uninstall COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/proj_uninstall.cmake) + +message(STATUS "EMBED_RESOURCE_FILES=${EMBED_RESOURCE_FILES}") diff --git a/cmake/FileEmbed.cmake b/cmake/FileEmbed.cmake new file mode 100644 index 0000000000..c40c4cb651 --- /dev/null +++ b/cmake/FileEmbed.cmake @@ -0,0 +1,99 @@ +# Derived from https://gitlab.com/jhamberg/cmake-examples/-/blob/master/cmake/FileEmbed.cmake +# MIT licensed +# Copyright (c) 2022 Jonathan Hamberg + +function(FileEmbedSetup) + + if (NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/file_embed) + file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}file_embed) + endif () + + if (NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/file_embed/file_embed_empty.c) + file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/file_embed/file_embed_empty.c "") + endif () + + add_library(file_embed ${CMAKE_CURRENT_BINARY_DIR}/file_embed/file_embed_empty.c) + target_include_directories(file_embed PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/file_embed) + +endfunction() + +function(FileEmbedAdd file) + FileEmbedGenerate(${file} var) + target_sources(file_embed PUBLIC ${var}) + + add_custom_command( + OUTPUT ${var} + COMMAND ${CMAKE_COMMAND} + -DRUN_FILE_EMBED_GENERATE=1 + "-DFILE_EMBED_GENERATE_PATH=${file}" + -P ${PROJECT_SOURCE_DIR}/cmake/FileEmbed.cmake + MAIN_DEPENDENCY ${file} + ) +endfunction() + +function(FileEmbedGenerate file generated_c) + + get_filename_component(base_filename ${file} NAME) + set(output_filename "${base_filename}.c") + string(MAKE_C_IDENTIFIER ${base_filename} c_name) + file(READ ${file} content HEX) + + string(LENGTH "${content}" size) + math(EXPR size_mult_16 "${size} - (${size} % 16)") + string(SUBSTRING "${content}" 0 ${size_mult_16} content_mult_16) + + string(REGEX REPLACE + "([A-Fa-f0-9][A-Fa-f0-9])\ +([A-Fa-f0-9][A-Fa-f0-9])\ +([A-Fa-f0-9][A-Fa-f0-9])\ +([A-Fa-f0-9][A-Fa-f0-9])\ +([A-Fa-f0-9][A-Fa-f0-9])\ +([A-Fa-f0-9][A-Fa-f0-9])\ +([A-Fa-f0-9][A-Fa-f0-9])\ +([A-Fa-f0-9][A-Fa-f0-9])" + "0x\\1,0x\\2,0x\\3,0x\\4,0x\\5,0x\\6,0x\\7,0x\\8,\n" SEPARATED_HEX "${content_mult_16}") + + set(SEPARATED_HEX_REMAINDER "") + if (NOT ${size_mult_16} EQUAL ${size}) + string(SUBSTRING "${content}" ${size_mult_16} 16 content_remainder) + string(REGEX REPLACE "([A-Fa-f0-9][A-Fa-f0-9])" "0x\\1," SEPARATED_HEX_REMAINDER "${content_remainder}") + endif() + + set(output_c "${SEPARATED_HEX}${SEPARATED_HEX_REMAINDER}") + + set(output_c " +#include \"${c_name}.h\" +const uint8_t ${c_name}_data[] = { + ${output_c} +}\; +const unsigned ${c_name}_size = sizeof(${c_name}_data)\; +") + + set(output_h " +#ifndef ${c_name}_H +#define ${c_name}_H +#include +extern const uint8_t ${c_name}_data[]\; +extern const unsigned ${c_name}_size\; +#endif // ${c_name}_H + ") + + + if (NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/file_embed) + file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}file_embed) + endif () + + + file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/file_embed/${c_name}.c + ${output_c}) + + file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/file_embed/${c_name}.h + ${output_h}) + + set(${generated_c} ${CMAKE_CURRENT_BINARY_DIR}/file_embed/${c_name}.c PARENT_SCOPE) + +endfunction() + +if (RUN_FILE_EMBED_GENERATE) + FileEmbedGenerate(${FILE_EMBED_GENERATE_PATH} var) +endif () diff --git a/data/CMakeLists.txt b/data/CMakeLists.txt index 2c1f2047a6..105b4e0d24 100644 --- a/data/CMakeLists.txt +++ b/data/CMakeLists.txt @@ -11,7 +11,7 @@ if(NOT IS_DIRECTORY ${PROJ_DB_CACHE_DIR}) endif() endif() -set(CONFIG_FILES +set(PROJ_INI proj.ini ) @@ -60,7 +60,7 @@ add_custom_command( add_custom_target(generate_proj_db ALL DEPENDS ${PROJ_DB}) if(NOT "${CMAKE_CURRENT_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_BINARY_DIR}") - foreach(FILE ${CONFIG_FILES} ${PROJ_DICTIONARY} ${GRIDSHIFT_FILES}) + foreach(FILE ${PROJ_INI} ${PROJ_DICTIONARY} ${GRIDSHIFT_FILES}) configure_file(${FILE} ${FILE} COPYONLY) endforeach() endif() @@ -78,7 +78,7 @@ set(DATA_FOR_TESTS execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/for_tests) execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/for_tests/tests) -foreach(FILE ${DATA_FOR_TESTS} ${CONFIG_FILES}) +foreach(FILE ${DATA_FOR_TESTS} ${PROJ_INI}) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/${FILE} ${CMAKE_CURRENT_BINARY_DIR}/for_tests/${FILE} COPYONLY) endforeach() @@ -109,13 +109,16 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/tests/conus "${CMAKE_CURRENT_BINARY_D #install # set(ALL_DATA_FILE - ${CONFIG_FILES} ${PROJ_DICTIONARY} ${GRIDSHIFT_FILES} - ${PROJ_DB} ${SCHEMA_FILES} ${GEOTIFF_FILES} ) + +if (NOT USE_ONLY_EMBEDDED_RESOURCE_FILES) + list(APPEND ALL_DATA_FILE ${PROJ_INI} ${PROJ_DB}) +endif() + install( FILES ${ALL_DATA_FILE} DESTINATION ${PROJ_DATA_PATH} diff --git a/data/generate_proj_db.cmake b/data/generate_proj_db.cmake index fa268cc9f3..9dfb27a71e 100644 --- a/data/generate_proj_db.cmake +++ b/data/generate_proj_db.cmake @@ -14,6 +14,8 @@ file(READ ${ALL_SQL_IN} CONTENTS) string(REPLACE "\${PROJ_VERSION}" "${PROJ_VERSION}" CONTENTS_MOD "${CONTENTS}") file(WRITE "${ALL_SQL_IN}" "${CONTENTS_MOD}") +set(generate_proj_db ON) + if(IS_DIRECTORY ${PROJ_DB_CACHE_DIR}) set(USE_PROJ_DB_CACHE_DIR TRUE) set(PROJ_DB_SQL_MD5_FILE "${PROJ_DB_CACHE_DIR}/proj.db.sql.md5") @@ -26,18 +28,21 @@ if(IS_DIRECTORY ${PROJ_DB_CACHE_DIR}) message(STATUS "Reusing cached proj.db from ${PROJ_DB_CACHE_DIR}") get_filename_component(PROJ_DB_DIR "${PROJ_DB}" DIRECTORY) file(COPY "${CACHED_PROJ_DB}" DESTINATION "${PROJ_DB_DIR}") - return() + file(TOUCH "${PROJ_DB}") + set(generate_proj_db OFF) endif() endif() -execute_process(COMMAND "${EXE_SQLITE3}" "${PROJ_DB}" - INPUT_FILE "${ALL_SQL_IN}" - RESULT_VARIABLE STATUS) +if (generate_proj_db) + execute_process(COMMAND "${EXE_SQLITE3}" "${PROJ_DB}" + INPUT_FILE "${ALL_SQL_IN}" + RESULT_VARIABLE STATUS) -if(STATUS AND NOT STATUS EQUAL 0) - message(FATAL_ERROR "Build of proj.db failed") -elseif(USE_PROJ_DB_CACHE_DIR) - message(STATUS "Saving cache: ${CACHED_PROJ_DB}") - file(COPY "${PROJ_DB}" DESTINATION "${PROJ_DB_CACHE_DIR}") - file(WRITE "${PROJ_DB_SQL_MD5_FILE}" "${PROJ_DB_SQL_MD5}\n") + if(STATUS AND NOT STATUS EQUAL 0) + message(FATAL_ERROR "Build of proj.db failed") + elseif(USE_PROJ_DB_CACHE_DIR) + message(STATUS "Saving cache: ${CACHED_PROJ_DB}") + file(COPY "${PROJ_DB}" DESTINATION "${PROJ_DB_CACHE_DIR}") + file(WRITE "${PROJ_DB_SQL_MD5_FILE}" "${PROJ_DB_SQL_MD5}\n") + endif() endif() diff --git a/docs/source/conf.py b/docs/source/conf.py index bc0cf80801..5e8f52e8a2 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -9,8 +9,6 @@ import sys from datetime import date -import sphinx_rtd_theme - sys.path.insert(0, os.path.abspath(".")) sys.path.insert(0, os.path.abspath("_extensions")) @@ -119,7 +117,7 @@ html_theme_options = { "canonical_url": "https://proj.org", "logo_only": True, - "display_version": True, + "version_selector": True, "prev_next_buttons_location": "both", "style_external_links": False, "style_nav_header_background": "#353130", @@ -131,9 +129,6 @@ "titles_only": False, } -# Add any paths that contain custom themes here, relative to this directory -html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] - # Display "Edit on GitHub" link on each page # https://docs.readthedocs.io/en/stable/guides/edit-source-links-sphinx.html html_context = { @@ -154,7 +149,7 @@ # Add any paths that contain custom static files (such as style sheets) html_static_path = ["_static"] -htm_css_files = [ +html_css_files = [ "theme_overrides.css", # override wide tables in RTD theme ] diff --git a/docs/source/install.rst b/docs/source/install.rst index 3929f78ad2..beffee6f53 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -446,6 +446,26 @@ All cached entries can be viewed using ``cmake -LAH`` from a build directory. Embed ``PROJ_DATA`` hard-coded alternative path for data files location. Disable to avoid setting this non-relocatable hard-coded path. Default ON. +.. option:: EMBED_RESOURCE_FILES=ON/OFF + + .. versionadded:: 9.6 + + Default is OFF for shared library builds (BUILD_SHARED_LIBS=ON), and ON + for static library builds (BUILD_SHARED_LIBS=OFF). + When ON, :file:`proj.db` and :file:`proj.ini` will be embedded into the PROJ library. + +.. option:: USE_ONLY_EMBEDDED_RESOURCE_FILES=ON/OFF + + .. versionadded:: 9.6 + + Even if EMBED_RESOURCE_FILES=ON, by default PROJ will still try to locate + :file:`proj.db` and :file:`proj.ini` on the file system, and fallback to the + embedded version if not found. + By setting USE_ONLY_EMBEDDED_RESOURCE_FILES=ON, no attempt at locating + those files on the file system is made. Default is OFF. + Users will also typically want to set EMBED_PROJ_DATA_PATH=OFF if setting + USE_ONLY_EMBEDDED_RESOURCE_FILES=OFF. + Building on Windows with vcpkg and Visual Studio 2017 or 2019 -------------------------------------------------------------------------------- diff --git a/src/embedded_resources.c b/src/embedded_resources.c new file mode 100644 index 0000000000..04b721f11e --- /dev/null +++ b/src/embedded_resources.c @@ -0,0 +1,34 @@ +#include "embedded_resources.h" + +#if USE_SHARP_EMBED +const unsigned char *pj_get_embedded_proj_db(unsigned int *pnSize) { + static const unsigned char proj_db[] = { +#embed PROJ_DB + }; + *pnSize = (unsigned int)sizeof(proj_db); + return proj_db; +} + +const char *pj_get_embedded_proj_ini(unsigned int *pnSize) { + static const char proj_ini[] = { +#embed PROJ_INI + }; + *pnSize = (unsigned int)sizeof(proj_ini); + return proj_ini; +} + +#else + +#include "file_embed/proj_db.h" +const unsigned char *pj_get_embedded_proj_db(unsigned int *pnSize) { + *pnSize = proj_db_size; + return proj_db_data; +} + +#include "file_embed/proj_ini.h" +const char *pj_get_embedded_proj_ini(unsigned int *pnSize) { + *pnSize = proj_ini_size; + return (const char *)proj_ini_data; +} + +#endif diff --git a/src/embedded_resources.h b/src/embedded_resources.h new file mode 100644 index 0000000000..39a119b0a7 --- /dev/null +++ b/src/embedded_resources.h @@ -0,0 +1,15 @@ +#ifndef EMBEDDED_RESOURCES_H +#define EMBEDDED_RESOURCES_H + +#ifdef __cplusplus +extern "C" { +#endif + +const unsigned char *pj_get_embedded_proj_db(unsigned int *pnSize); +const char *pj_get_embedded_proj_ini(unsigned int *pnSize); + +#ifdef __cplusplus +} +#endif + +#endif /* EMBEDDED_RESOURCES_H */ diff --git a/src/filemanager.cpp b/src/filemanager.cpp index c176c70c5b..3c6bcef61d 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -69,6 +69,10 @@ #include #endif +#ifdef EMBED_RESOURCE_FILES +#include "embedded_resources.h" +#endif + //! @cond Doxygen_Suppress using namespace NS_PROJ::internal; @@ -1850,21 +1854,32 @@ void pj_load_ini(PJ_CONTEXT *ctx) { } ctx->iniFileLoaded = true; - auto file = std::unique_ptr( - reinterpret_cast(pj_open_lib_internal( - ctx, "proj.ini", "rb", pj_open_file_with_manager, nullptr, 0))); - if (!file) - return; - file->seek(0, SEEK_END); - const auto filesize = file->tell(); - if (filesize == 0 || filesize > 100 * 1024U) - return; - file->seek(0, SEEK_SET); std::string content; - content.resize(static_cast(filesize)); - const auto nread = file->read(&content[0], content.size()); - if (nread != content.size()) + std::unique_ptr file; +#ifndef USE_ONLY_EMBEDDED_RESOURCE_FILES + file.reset(reinterpret_cast(pj_open_lib_internal( + ctx, "proj.ini", "rb", pj_open_file_with_manager, nullptr, 0))); +#endif + if (!file) { +#ifdef EMBED_RESOURCE_FILES + unsigned int content_size = 0; + const char *c_content = pj_get_embedded_proj_ini(&content_size); + content.assign(c_content, content_size); +#else return; +#endif + } + if (file) { + file->seek(0, SEEK_END); + const auto filesize = file->tell(); + if (filesize == 0 || filesize > 100 * 1024U) + return; + file->seek(0, SEEK_SET); + content.resize(static_cast(filesize)); + const auto nread = file->read(&content[0], content.size()); + if (nread != content.size()) + return; + } content += '\n'; size_t pos = 0; while (pos != std::string::npos) { diff --git a/src/iso19111/factory.cpp b/src/iso19111/factory.cpp index 822d0cc6e1..56dd92f1b1 100644 --- a/src/iso19111/factory.cpp +++ b/src/iso19111/factory.cpp @@ -76,6 +76,10 @@ #include +#ifdef EMBED_RESOURCE_FILES +#include "embedded_resources.h" +#endif + // Custom SQLite VFS as our database is not supposed to be modified in // parallel. This is slightly faster #define ENABLE_CUSTOM_LOCKLESS_VFS @@ -130,6 +134,10 @@ constexpr int DATABASE_LAYOUT_VERSION_MINOR = 4; constexpr size_t N_MAX_PARAMS = 7; +#ifdef EMBED_RESOURCE_FILES +constexpr const char *EMBEDDED_PROJ_DB = "__embedded_proj_db__"; +#endif + // --------------------------------------------------------------------------- struct SQLValues { @@ -238,6 +246,7 @@ static void PROJ_SQLITE_intersects_bbox(sqlite3_context *pContext, // --------------------------------------------------------------------------- class SQLiteHandle { + std::string path_{}; sqlite3 *sqlite_handle_ = nullptr; bool close_handle_ = true; @@ -248,7 +257,7 @@ class SQLiteHandle { int nLayoutVersionMajor_ = 0; int nLayoutVersionMinor_ = 0; -#ifdef ENABLE_CUSTOM_LOCKLESS_VFS +#if defined(ENABLE_CUSTOM_LOCKLESS_VFS) || defined(EMBED_RESOURCE_FILES) std::unique_ptr vfs_{}; #endif @@ -270,6 +279,8 @@ class SQLiteHandle { public: ~SQLiteHandle(); + const std::string &path() const { return path_; } + sqlite3 *handle() { return sqlite_handle_; } #ifdef REOPEN_SQLITE_DB_AFTER_FORK @@ -312,8 +323,9 @@ SQLiteHandle::~SQLiteHandle() { // --------------------------------------------------------------------------- std::shared_ptr SQLiteHandle::open(PJ_CONTEXT *ctx, - const std::string &path) { + const std::string &pathIn) { + std::string path(pathIn); const int sqlite3VersionNumber = sqlite3_libversion_number(); // Minimum version for correct performance: 3.11 if (sqlite3VersionNumber < 3 * 1000000 + 11 * 1000) { @@ -323,9 +335,35 @@ std::shared_ptr SQLiteHandle::open(PJ_CONTEXT *ctx, } std::string vfsName; -#ifdef ENABLE_CUSTOM_LOCKLESS_VFS +#if defined(ENABLE_CUSTOM_LOCKLESS_VFS) || defined(EMBED_RESOURCE_FILES) std::unique_ptr vfs; - if (ctx->custom_sqlite3_vfs_name.empty()) { +#endif + +#ifdef EMBED_RESOURCE_FILES + if (path == EMBEDDED_PROJ_DB && ctx->custom_sqlite3_vfs_name.empty()) { + unsigned int proj_db_size = 0; + const unsigned char *proj_db = pj_get_embedded_proj_db(&proj_db_size); + + vfs = SQLite3VFS::createMem(proj_db, proj_db_size); + if (vfs == nullptr) { + throw FactoryException("Open of " + path + " failed"); + } + + std::ostringstream buffer; + buffer << "file:/proj.db?immutable=1&ptr="; + buffer << reinterpret_cast(proj_db); + buffer << "&sz="; + buffer << proj_db_size; + buffer << "&max="; + buffer << proj_db_size; + buffer << "&vfs="; + buffer << vfs->name(); + path = buffer.str(); + } else +#endif + +#ifdef ENABLE_CUSTOM_LOCKLESS_VFS + if (ctx->custom_sqlite3_vfs_name.empty()) { vfs = SQLite3VFS::create(false, true, true); if (vfs == nullptr) { throw FactoryException("Open of " + path + " failed"); @@ -350,10 +388,11 @@ std::shared_ptr SQLiteHandle::open(PJ_CONTEXT *ctx, } auto handle = std::shared_ptr(new SQLiteHandle(sqlite_handle, true)); -#ifdef ENABLE_CUSTOM_LOCKLESS_VFS +#if defined(ENABLE_CUSTOM_LOCKLESS_VFS) || defined(EMBED_RESOURCE_FILES) handle->vfs_ = std::move(vfs); #endif handle->initialize(); + handle->path_ = path; handle->checkDatabaseLayout(path, path, std::string()); return handle; } @@ -454,8 +493,12 @@ SQLResultSet SQLiteHandle::run(sqlite3_stmt *stmt, const std::string &sql, } else if (ret == SQLITE_DONE) { break; } else { - throw FactoryException("SQLite error on " + sql + ": " + - sqlite3_errmsg(sqlite_handle_)); + throw FactoryException(std::string("SQLite error on ") + .append(sql) + .append(": code = ") + .append(internal::toString(ret)) + .append(", msg = ") + .append(sqlite3_errmsg(sqlite_handle_))); } } return result; @@ -1200,18 +1243,25 @@ void DatabaseContext::Private::open(const std::string &databasePath, setPjCtxt(ctx); std::string path(databasePath); if (path.empty()) { +#ifndef USE_ONLY_EMBEDDED_RESOURCE_FILES path.resize(2048); const bool found = pj_find_file(pjCtxt(), "proj.db", &path[0], path.size() - 1) != 0; path.resize(strlen(path.c_str())); - if (!found) { + if (!found) +#endif + { +#ifdef EMBED_RESOURCE_FILES + path = EMBEDDED_PROJ_DB; +#else throw FactoryException("Cannot find proj.db"); +#endif } } sqlite_handle_ = SQLiteHandleCache::get().getHandle(path, ctx); - databasePath_ = std::move(path); + databasePath_ = sqlite_handle_->path(); } // --------------------------------------------------------------------------- diff --git a/src/lib_proj.cmake b/src/lib_proj.cmake index c39e5dfd4d..5f9fad3aaf 100644 --- a/src/lib_proj.cmake +++ b/src/lib_proj.cmake @@ -401,6 +401,47 @@ if("${CMAKE_C_COMPILER_ID}" STREQUAL "Intel") PROPERTIES COMPILE_FLAGS ${FP_PRECISE}) endif() +if (EMBED_RESOURCE_FILES AND NOT IS_SHARP_EMBED_AVAILABLE_RES) + set(EMBEDDED_PROJ_DB "file_embed/proj_db.c") + add_custom_command( + OUTPUT "${EMBEDDED_PROJ_DB}" + COMMAND ${CMAKE_COMMAND} + -DRUN_FILE_EMBED_GENERATE=1 + "-DFILE_EMBED_GENERATE_PATH=${PROJECT_BINARY_DIR}/data/proj.db" + -P ${PROJECT_SOURCE_DIR}/cmake/FileEmbed.cmake + DEPENDS generate_proj_db "${PROJECT_BINARY_DIR}/data/proj.db" + ) + target_sources(proj PRIVATE embedded_resources.c "${EMBEDDED_PROJ_DB}") + + set(EMBEDDED_PROJ_INI "file_embed/proj_ini.c") + add_custom_command( + OUTPUT "${EMBEDDED_PROJ_INI}" + COMMAND ${CMAKE_COMMAND} + -DRUN_FILE_EMBED_GENERATE=1 + "-DFILE_EMBED_GENERATE_PATH=${PROJECT_SOURCE_DIR}/data/proj.ini" + -P ${PROJECT_SOURCE_DIR}/cmake/FileEmbed.cmake + DEPENDS "${PROJECT_SOURCE_DIR}/data/proj.ini" + ) + target_sources(proj PRIVATE embedded_resources.c "${EMBEDDED_PROJ_DB}" "${EMBEDDED_PROJ_INI}") +elseif(EMBED_RESOURCE_FILES AND IS_SHARP_EMBED_AVAILABLE_RES) + add_library(proj_resources OBJECT embedded_resources.c) + target_compile_definitions(proj_resources PRIVATE "PROJ_DB=\"${PROJECT_BINARY_DIR}/data/proj.db\"") + target_compile_definitions(proj_resources PRIVATE "PROJ_INI=\"${PROJECT_SOURCE_DIR}/data/proj.ini\"") + target_compile_definitions(proj_resources PRIVATE USE_SHARP_EMBED) + add_dependencies(proj_resources generate_proj_db) + option(PROJ_OBJECT_LIBRARIES_POSITION_INDEPENDENT_CODE "Set ON to produce -fPIC code" ${BUILD_SHARED_LIBS}) + set_property(TARGET proj_resources PROPERTY POSITION_INDEPENDENT_CODE ${PROJ_OBJECT_LIBRARIES_POSITION_INDEPENDENT_CODE}) + set_target_properties(proj_resources PROPERTIES C_STANDARD 23) + target_sources(proj PRIVATE $) +endif() +if (EMBED_RESOURCE_FILES) + target_sources(proj PRIVATE memvfs.c) + target_compile_definitions(proj PRIVATE EMBED_RESOURCE_FILES) +endif() +if (USE_ONLY_EMBEDDED_RESOURCE_FILES) + target_compile_definitions(proj PRIVATE USE_ONLY_EMBEDDED_RESOURCE_FILES) +endif() + if(ENABLE_IPO) set_property(TARGET proj PROPERTY INTERPROCEDURAL_OPTIMIZATION TRUE) diff --git a/src/memvfs.c b/src/memvfs.c new file mode 100644 index 0000000000..96fe824a59 --- /dev/null +++ b/src/memvfs.c @@ -0,0 +1,448 @@ +/* Derived from https://www.sqlite.org/src/doc/tip/ext/misc/memvfs.c */ +/* +** 2016-09-07 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This is an in-memory VFS implementation. The application supplies +** a chunk of memory to hold the database file. +** +** Because there is place to store a rollback or wal journal, the database +** must use one of journal_mode=MEMORY or journal_mode=NONE. +** +** USAGE: +** +** sqlite3_open_v2("file:/whatever?ptr=0xf05538&sz=14336&max=65536", &db, +** SQLITE_OPEN_READWRITE | SQLITE_OPEN_URI, +** "memvfs"); +** +** These are the query parameters: +** +** ptr= The address of the memory buffer that holds the database. +** +** sz= The current size the database file +** +** maxsz= The maximum size of the database. In other words, the +** amount of space allocated for the ptr= buffer. +** +** The ptr= and sz= query parameters are required. If maxsz= is omitted, +** then it defaults to the sz= value. Parameter values can be in either +** decimal or hexadecimal. The filename in the URI is ignored. +*/ +#include + +#include +#include +#include + +#include "memvfs.h" + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#pragma GCC diagnostic ignored "-Wmissing-field-initializers" +#endif + +#ifdef _MSC_VER +#pragma warning(push) +// Ignore unreferenced formal parameter +#pragma warning(disable : 4100) +#endif + +/* +** Forward declaration of objects used by this utility +*/ +typedef struct sqlite3_vfs MemVfs; +typedef struct MemFile MemFile; + +typedef struct MemVfsAppData { + const unsigned char *buffer; + size_t bufferSize; + sqlite3_vfs *pBaseVFS; +} MemVfsAppData; + +/* Access to a lower-level VFS that (might) implement dynamic loading, +** access to randomness, etc. +*/ +#define ORIGVFS(p) (((MemVfsAppData *)((p)->pAppData))->pBaseVFS) + +/* An open file */ +struct MemFile { + sqlite3_file base; /* IO methods */ + sqlite3_int64 sz; /* Size of the file */ + sqlite3_int64 szMax; /* Space allocated to aData */ + const unsigned char *aData; /* content of the file */ +}; + +/* +** Methods for MemFile +*/ +static int memClose(sqlite3_file *); +static int memRead(sqlite3_file *, void *, int iAmt, sqlite3_int64 iOfst); +static int memWrite(sqlite3_file *, const void *, int iAmt, + sqlite3_int64 iOfst); +static int memTruncate(sqlite3_file *, sqlite3_int64 size); +static int memSync(sqlite3_file *, int flags); +static int memFileSize(sqlite3_file *, sqlite3_int64 *pSize); +static int memLock(sqlite3_file *, int); +static int memUnlock(sqlite3_file *, int); +static int memCheckReservedLock(sqlite3_file *, int *pResOut); +static int memFileControl(sqlite3_file *, int op, void *pArg); +static int memSectorSize(sqlite3_file *); +static int memDeviceCharacteristics(sqlite3_file *); +static int memShmMap(sqlite3_file *, int iPg, int pgsz, int, void volatile **); +static int memShmLock(sqlite3_file *, int offset, int n, int flags); +static void memShmBarrier(sqlite3_file *); +static int memShmUnmap(sqlite3_file *, int deleteFlag); +static int memFetch(sqlite3_file *, sqlite3_int64 iOfst, int iAmt, void **pp); +static int memUnfetch(sqlite3_file *, sqlite3_int64 iOfst, void *p); + +/* +** Methods for MemVfs +*/ +static int memOpen(sqlite3_vfs *, const char *, sqlite3_file *, int, int *); +static int memDelete(sqlite3_vfs *, const char *zName, int syncDir); +static int memAccess(sqlite3_vfs *, const char *zName, int flags, int *); +static int memFullPathname(sqlite3_vfs *, const char *zName, int, char *zOut); +static void *memDlOpen(sqlite3_vfs *, const char *zFilename); +static void memDlError(sqlite3_vfs *, int nByte, char *zErrMsg); +static void (*memDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void); +static void memDlClose(sqlite3_vfs *, void *); +static int memRandomness(sqlite3_vfs *, int nByte, char *zOut); +static int memSleep(sqlite3_vfs *, int microseconds); +static int memCurrentTime(sqlite3_vfs *, double *); +static int memGetLastError(sqlite3_vfs *, int, char *); +static int memCurrentTimeInt64(sqlite3_vfs *, sqlite3_int64 *); + +static sqlite3_vfs mem_vfs = { + 2, /* iVersion */ + 0, /* szOsFile (set when registered) */ + 1024, /* mxPathname */ + 0, /* pNext */ + "memvfs", /* zName */ + 0, /* pAppData (set when registered) */ + memOpen, /* xOpen */ + memDelete, /* xDelete */ + memAccess, /* xAccess */ + memFullPathname, /* xFullPathname */ + memDlOpen, /* xDlOpen */ + memDlError, /* xDlError */ + memDlSym, /* xDlSym */ + memDlClose, /* xDlClose */ + memRandomness, /* xRandomness */ + memSleep, /* xSleep */ + memCurrentTime, /* xCurrentTime */ + memGetLastError, /* xGetLastError */ + memCurrentTimeInt64 /* xCurrentTimeInt64 */ +}; + +static const sqlite3_io_methods mem_io_methods = { + 3, /* iVersion */ + memClose, /* xClose */ + memRead, /* xRead */ + memWrite, /* xWrite */ + memTruncate, /* xTruncate */ + memSync, /* xSync */ + memFileSize, /* xFileSize */ + memLock, /* xLock */ + memUnlock, /* xUnlock */ + memCheckReservedLock, /* xCheckReservedLock */ + memFileControl, /* xFileControl */ + memSectorSize, /* xSectorSize */ + memDeviceCharacteristics, /* xDeviceCharacteristics */ + memShmMap, /* xShmMap */ + memShmLock, /* xShmLock */ + memShmBarrier, /* xShmBarrier */ + memShmUnmap, /* xShmUnmap */ + memFetch, /* xFetch */ + memUnfetch /* xUnfetch */ +}; + +/* +** Close an mem-file. +** +** The pData pointer is owned by the application, so there is nothing +** to free. +*/ +static int memClose(sqlite3_file *pFile) { return SQLITE_OK; } + +/* +** Read data from an mem-file. +*/ +static int memRead(sqlite3_file *pFile, void *zBuf, int iAmt, + sqlite_int64 iOfst) { + MemFile *p = (MemFile *)pFile; + memcpy(zBuf, p->aData + iOfst, iAmt); + return SQLITE_OK; +} + +/* +** Write data to an mem-file. +*/ +static int memWrite(sqlite3_file *pFile, const void *z, int iAmt, + sqlite_int64 iOfst) { + return SQLITE_READONLY; +} + +/* +** Truncate an mem-file. +*/ +static int memTruncate(sqlite3_file *pFile, sqlite_int64 size) { + return SQLITE_READONLY; +} + +/* +** Sync an mem-file. +*/ +static int memSync(sqlite3_file *pFile, int flags) { return SQLITE_OK; } + +/* +** Return the current file-size of an mem-file. +*/ +static int memFileSize(sqlite3_file *pFile, sqlite_int64 *pSize) { + MemFile *p = (MemFile *)pFile; + *pSize = p->sz; + return SQLITE_OK; +} + +/* +** Lock an mem-file. +*/ +static int memLock(sqlite3_file *pFile, int eLock) { return SQLITE_OK; } + +/* +** Unlock an mem-file. +*/ +static int memUnlock(sqlite3_file *pFile, int eLock) { return SQLITE_OK; } + +/* +** Check if another file-handle holds a RESERVED lock on an mem-file. +*/ +static int memCheckReservedLock(sqlite3_file *pFile, int *pResOut) { + *pResOut = 0; + return SQLITE_OK; +} + +/* +** File control method. For custom operations on an mem-file. +*/ +static int memFileControl(sqlite3_file *pFile, int op, void *pArg) { + MemFile *p = (MemFile *)pFile; + int rc = SQLITE_NOTFOUND; + if (op == SQLITE_FCNTL_VFSNAME) { + *(char **)pArg = sqlite3_mprintf("mem(%p,%lld)", p->aData, p->sz); + rc = SQLITE_OK; + } + return rc; +} + +/* +** Return the sector-size in bytes for an mem-file. +*/ +static int memSectorSize(sqlite3_file *pFile) { return 1024; } + +/* +** Return the device characteristic flags supported by an mem-file. +*/ +static int memDeviceCharacteristics(sqlite3_file *pFile) { + return SQLITE_IOCAP_ATOMIC | SQLITE_IOCAP_POWERSAFE_OVERWRITE | + SQLITE_IOCAP_SAFE_APPEND | SQLITE_IOCAP_SEQUENTIAL; +} + +/* Create a shared memory file mapping */ +static int memShmMap(sqlite3_file *pFile, int iPg, int pgsz, int bExtend, + void volatile **pp) { + return SQLITE_IOERR_SHMMAP; +} + +/* Perform locking on a shared-memory segment */ +static int memShmLock(sqlite3_file *pFile, int offset, int n, int flags) { + return SQLITE_IOERR_SHMLOCK; +} + +/* Memory barrier operation on shared memory */ +static void memShmBarrier(sqlite3_file *pFile) { return; } + +/* Unmap a shared memory segment */ +static int memShmUnmap(sqlite3_file *pFile, int deleteFlag) { + return SQLITE_OK; +} + +/* Fetch a page of a memory-mapped file */ +static int memFetch(sqlite3_file *pFile, sqlite3_int64 iOfst, int iAmt, + void **pp) { + MemFile *p = (MemFile *)pFile; + *pp = (void *)(p->aData + iOfst); + return SQLITE_OK; +} + +/* Release a memory-mapped page */ +static int memUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage) { + return SQLITE_OK; +} + +/* +** Open an mem file handle. +*/ +static int memOpen(sqlite3_vfs *pVfs, const char *zName, sqlite3_file *pFile, + int flags, int *pOutFlags) { + MemFile *p = (MemFile *)pFile; + MemVfsAppData *appData = (MemVfsAppData *)(pVfs->pAppData); + memset(p, 0, sizeof(*p)); + if ((flags & SQLITE_OPEN_MAIN_DB) == 0) { + /* Modification w.r.t upstream: instead of returning SQLITE_CANTOPEN, + * delegate to orign VFS. Typically for temporary file creation. + */ + return ORIGVFS(pVfs)->xOpen(ORIGVFS(pVfs), zName, pFile, flags, + pOutFlags); + } + if ((uintptr_t)(appData->buffer) != + (uintptr_t)sqlite3_uri_int64(zName, "ptr", 0)) { + return SQLITE_CANTOPEN; + } + p->aData = appData->buffer; + p->sz = sqlite3_uri_int64(zName, "sz", 0); + if (p->sz < 0 || (size_t)p->sz != appData->bufferSize) + return SQLITE_CANTOPEN; + p->szMax = sqlite3_uri_int64(zName, "max", p->sz); + if (p->szMax < p->sz) + return SQLITE_CANTOPEN; + pFile->pMethods = &mem_io_methods; + return SQLITE_OK; +} + +/* +** Delete the file located at zPath. If the dirSync argument is true, +** ensure the file-system modifications are synced to disk before +** returning. +*/ +static int memDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync) { + return SQLITE_IOERR_DELETE; +} + +/* +** Test for access permissions. Return true if the requested permission +** is available, or false otherwise. +*/ +static int memAccess(sqlite3_vfs *pVfs, const char *zPath, int flags, + int *pResOut) { + *pResOut = 0; + return SQLITE_OK; +} + +/* +** Populate buffer zOut with the full canonical pathname corresponding +** to the pathname in zPath. zOut is guaranteed to point to a buffer +** of at least (INST_MAX_PATHNAME+1) bytes. +*/ +static int memFullPathname(sqlite3_vfs *pVfs, const char *zPath, int nOut, + char *zOut) { + sqlite3_snprintf(nOut, zOut, "%s", zPath); + return SQLITE_OK; +} + +/* +** Open the dynamic library located at zPath and return a handle. +*/ +static void *memDlOpen(sqlite3_vfs *pVfs, const char *zPath) { + return ORIGVFS(pVfs)->xDlOpen(ORIGVFS(pVfs), zPath); +} + +/* +** Populate the buffer zErrMsg (size nByte bytes) with a human readable +** utf-8 string describing the most recent error encountered associated +** with dynamic libraries. +*/ +static void memDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg) { + ORIGVFS(pVfs)->xDlError(ORIGVFS(pVfs), nByte, zErrMsg); +} + +/* +** Return a pointer to the symbol zSymbol in the dynamic library pHandle. +*/ +static void (*memDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void) { + return ORIGVFS(pVfs)->xDlSym(ORIGVFS(pVfs), p, zSym); +} + +/* +** Close the dynamic library handle pHandle. +*/ +static void memDlClose(sqlite3_vfs *pVfs, void *pHandle) { + ORIGVFS(pVfs)->xDlClose(ORIGVFS(pVfs), pHandle); +} + +/* +** Populate the buffer pointed to by zBufOut with nByte bytes of +** random data. +*/ +static int memRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut) { + return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut); +} + +/* +** Sleep for nMicro microseconds. Return the number of microseconds +** actually slept. +*/ +static int memSleep(sqlite3_vfs *pVfs, int nMicro) { + return ORIGVFS(pVfs)->xSleep(ORIGVFS(pVfs), nMicro); +} + +/* +** Return the current time as a Julian Day number in *pTimeOut. +*/ +static int memCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut) { + return ORIGVFS(pVfs)->xCurrentTime(ORIGVFS(pVfs), pTimeOut); +} + +static int memGetLastError(sqlite3_vfs *pVfs, int a, char *b) { + return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b); +} +static int memCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p) { + return ORIGVFS(pVfs)->xCurrentTimeInt64(ORIGVFS(pVfs), p); +} + +/* + * Register the new VFS. + */ +int pj_sqlite3_memvfs_init(sqlite3_vfs *vfs, const char *vfs_name, + const void *buffer, size_t bufferSize) { + memcpy(vfs, &mem_vfs, sizeof(mem_vfs)); + vfs->zName = vfs_name; + sqlite3_vfs *defaultVFS = sqlite3_vfs_find(NULL); + if (!defaultVFS) + return SQLITE_ERROR; + MemVfsAppData *appData = + (MemVfsAppData *)sqlite3_malloc(sizeof(MemVfsAppData)); + appData->buffer = buffer; + appData->bufferSize = bufferSize; + appData->pBaseVFS = defaultVFS; + vfs->pAppData = appData; + vfs->szOsFile = sizeof(MemFile); + /* Modification w.r.t upstream: as we might delegate file opening + * to default VFS for temporary files, we need to make sure szOsFile is + * the maximum of our own need and of the default VFS. + */ + if (vfs->szOsFile < defaultVFS->szOsFile) + vfs->szOsFile = defaultVFS->szOsFile; + return sqlite3_vfs_register(vfs, 0); +} + +void pj_sqlite3_memvfs_deallocate_user_data(sqlite3_vfs *vfs) { + sqlite3_free(vfs->pAppData); + vfs->pAppData = NULL; +} + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/src/memvfs.h b/src/memvfs.h new file mode 100644 index 0000000000..f8d7e412c7 --- /dev/null +++ b/src/memvfs.h @@ -0,0 +1,18 @@ +#ifndef MEMVFS_H +#define MEMVFS_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +int pj_sqlite3_memvfs_init(sqlite3_vfs *vfs, const char *vfs_name, + const void *buffer, size_t bufferSize); +void pj_sqlite3_memvfs_deallocate_user_data(sqlite3_vfs *vfs); + +#ifdef __cplusplus +} +#endif + +#endif /* MEMVFS_H */ diff --git a/src/sqlite3_utils.cpp b/src/sqlite3_utils.cpp index f21d67f5f1..5f37b0b8af 100644 --- a/src/sqlite3_utils.cpp +++ b/src/sqlite3_utils.cpp @@ -36,6 +36,10 @@ #pragma GCC diagnostic pop #endif +#ifdef EMBED_RESOURCE_FILES +#include "memvfs.h" +#endif + #include #include #include // std::ostringstream @@ -44,13 +48,17 @@ NS_PROJ_START // --------------------------------------------------------------------------- +pj_sqlite3_vfs::~pj_sqlite3_vfs() = default; + +// --------------------------------------------------------------------------- + SQLite3VFS::SQLite3VFS(pj_sqlite3_vfs *vfs) : vfs_(vfs) {} // --------------------------------------------------------------------------- SQLite3VFS::~SQLite3VFS() { if (vfs_) { - sqlite3_vfs_unregister(vfs_); + sqlite3_vfs_unregister(&(vfs_->base)); delete vfs_; } } @@ -84,10 +92,27 @@ static int VSFNoOpLockUnlockSync(sqlite3_file *, int) { return SQLITE_OK; } // --------------------------------------------------------------------------- +namespace { + +struct pj_sqlite3_customvfs_appdata { + sqlite3_vfs *defaultVFS = nullptr; + bool fakeSync = false; + bool fakeLock = false; +}; + +struct pj_sqlite3_customvfs : public pj_sqlite3_vfs { + ~pj_sqlite3_customvfs() override { + delete static_cast(base.pAppData); + base.pAppData = nullptr; + } +}; +} // namespace + static int VFSCustomOpen(sqlite3_vfs *vfs, const char *name, sqlite3_file *file, int flags, int *outFlags) { - auto realVFS = static_cast(vfs); - sqlite3_vfs *defaultVFS = static_cast(vfs->pAppData); + pj_sqlite3_customvfs_appdata *appdata = + static_cast(vfs->pAppData); + sqlite3_vfs *defaultVFS = appdata->defaultVFS; int ret = defaultVFS->xOpen(defaultVFS, name, file, flags, outFlags); if (ret == SQLITE_OK) { ClosePtr defaultClosePtr = file->pMethods->xClose; @@ -100,13 +125,13 @@ static int VFSCustomOpen(sqlite3_vfs *vfs, const char *name, sqlite3_file *file, } memcpy(methods, file->pMethods, sizeof(sqlite3_io_methods)); methods->xClose = VFSClose; - if (realVFS->fakeSync) { + if (appdata->fakeSync) { // Disable xSync because it can be significantly slow and we don't // need // that level of data integrity guarantee for the cache. methods->xSync = VSFNoOpLockUnlockSync; } - if (realVFS->fakeLock) { + if (appdata->fakeLock) { methods->xLock = VSFNoOpLockUnlockSync; methods->xUnlock = VSFNoOpLockUnlockSync; } @@ -138,19 +163,28 @@ static void projSqlite3LogCallback(void *, int iErrCode, const char *zMsg) { fprintf(stderr, "SQLite3 message: (code %d) %s\n", iErrCode, zMsg); } +namespace { +struct InstallSqliteLogger { + InstallSqliteLogger() { + if (getenv("PROJ_LOG_SQLITE3") != nullptr) { + sqlite3_config(SQLITE_CONFIG_LOG, projSqlite3LogCallback, nullptr); + } + } + + static InstallSqliteLogger &GetSingleton() { + static InstallSqliteLogger installSqliteLogger; + return installSqliteLogger; + } +}; +} // namespace + +// --------------------------------------------------------------------------- + std::unique_ptr SQLite3VFS::create(bool fakeSync, bool fakeLock, bool skipStatJournalAndWAL) { // Install SQLite3 logger if PROJ_LOG_SQLITE3 env var is defined - struct InstallSqliteLogger { - InstallSqliteLogger() { - if (getenv("PROJ_LOG_SQLITE3") != nullptr) { - sqlite3_config(SQLITE_CONFIG_LOG, projSqlite3LogCallback, - nullptr); - } - } - }; - static InstallSqliteLogger installSqliteLogger; + InstallSqliteLogger::GetSingleton(); // Call to sqlite3_initialize() is normally not needed, except for // people building SQLite3 with -DSQLITE_OMIT_AUTOINIT @@ -158,9 +192,7 @@ std::unique_ptr SQLite3VFS::create(bool fakeSync, bool fakeLock, sqlite3_vfs *defaultVFS = sqlite3_vfs_find(nullptr); assert(defaultVFS); - auto vfs = new pj_sqlite3_vfs(); - vfs->fakeSync = fakeSync; - vfs->fakeLock = fakeLock; + auto vfs = new pj_sqlite3_customvfs(); auto vfsUnique = std::unique_ptr(new SQLite3VFS(vfs)); @@ -168,26 +200,66 @@ std::unique_ptr SQLite3VFS::create(bool fakeSync, bool fakeLock, buffer << vfs; vfs->namePtr = buffer.str(); - vfs->iVersion = 1; - vfs->szOsFile = defaultVFS->szOsFile + sizeof(ClosePtr); - vfs->mxPathname = defaultVFS->mxPathname; - vfs->zName = vfs->namePtr.c_str(); - vfs->pAppData = defaultVFS; - vfs->xOpen = VFSCustomOpen; - vfs->xDelete = defaultVFS->xDelete; - vfs->xAccess = + vfs->base.iVersion = 1; + vfs->base.szOsFile = defaultVFS->szOsFile + sizeof(ClosePtr); + vfs->base.mxPathname = defaultVFS->mxPathname; + vfs->base.zName = vfs->namePtr.c_str(); + pj_sqlite3_customvfs_appdata *appdata = new pj_sqlite3_customvfs_appdata; + appdata->fakeSync = fakeSync; + appdata->fakeLock = fakeLock; + appdata->defaultVFS = defaultVFS; + vfs->base.pAppData = appdata; + vfs->base.xOpen = VFSCustomOpen; + vfs->base.xDelete = defaultVFS->xDelete; + vfs->base.xAccess = skipStatJournalAndWAL ? VFSCustomAccess : defaultVFS->xAccess; - vfs->xFullPathname = defaultVFS->xFullPathname; - vfs->xDlOpen = defaultVFS->xDlOpen; - vfs->xDlError = defaultVFS->xDlError; - vfs->xDlSym = defaultVFS->xDlSym; - vfs->xDlClose = defaultVFS->xDlClose; - vfs->xRandomness = defaultVFS->xRandomness; - vfs->xSleep = defaultVFS->xSleep; - vfs->xCurrentTime = defaultVFS->xCurrentTime; - vfs->xGetLastError = defaultVFS->xGetLastError; - vfs->xCurrentTimeInt64 = defaultVFS->xCurrentTimeInt64; - if (sqlite3_vfs_register(vfs, false) == SQLITE_OK) { + vfs->base.xFullPathname = defaultVFS->xFullPathname; + vfs->base.xDlOpen = defaultVFS->xDlOpen; + vfs->base.xDlError = defaultVFS->xDlError; + vfs->base.xDlSym = defaultVFS->xDlSym; + vfs->base.xDlClose = defaultVFS->xDlClose; + vfs->base.xRandomness = defaultVFS->xRandomness; + vfs->base.xSleep = defaultVFS->xSleep; + vfs->base.xCurrentTime = defaultVFS->xCurrentTime; + vfs->base.xGetLastError = defaultVFS->xGetLastError; + vfs->base.xCurrentTimeInt64 = defaultVFS->xCurrentTimeInt64; + if (sqlite3_vfs_register(&(vfs->base), false) == SQLITE_OK) { + return vfsUnique; + } + delete vfsUnique->vfs_; + vfsUnique->vfs_ = nullptr; + return nullptr; +} + +// --------------------------------------------------------------------------- + +#ifdef EMBED_RESOURCE_FILES + +struct pj_sqlite3_memvfs : public pj_sqlite3_vfs { + ~pj_sqlite3_memvfs() override { + pj_sqlite3_memvfs_deallocate_user_data(&base); + } +}; + +/* static */ +std::unique_ptr SQLite3VFS::createMem(const void *membuffer, + size_t bufferSize) { + // Install SQLite3 logger if PROJ_LOG_SQLITE3 env var is defined + InstallSqliteLogger::GetSingleton(); + + // Call to sqlite3_initialize() is normally not needed, except for + // people building SQLite3 with -DSQLITE_OMIT_AUTOINIT + sqlite3_initialize(); + + auto vfs = new pj_sqlite3_memvfs(); + + auto vfsUnique = std::unique_ptr(new SQLite3VFS(vfs)); + + std::ostringstream buffer; + buffer << vfs; + vfs->namePtr = buffer.str(); + if (pj_sqlite3_memvfs_init(&(vfs->base), vfs->namePtr.c_str(), membuffer, + bufferSize) == SQLITE_OK) { return vfsUnique; } delete vfsUnique->vfs_; @@ -195,6 +267,8 @@ std::unique_ptr SQLite3VFS::create(bool fakeSync, bool fakeLock, return nullptr; } +#endif + // --------------------------------------------------------------------------- SQLiteStatement::SQLiteStatement(sqlite3_stmt *hStmtIn) : hStmt(hStmtIn) {} diff --git a/src/sqlite3_utils.hpp b/src/sqlite3_utils.hpp index 42ec33af7b..e592e7604d 100644 --- a/src/sqlite3_utils.hpp +++ b/src/sqlite3_utils.hpp @@ -41,10 +41,11 @@ NS_PROJ_START // --------------------------------------------------------------------------- -struct pj_sqlite3_vfs : public sqlite3_vfs { +struct pj_sqlite3_vfs { + sqlite3_vfs base{}; std::string namePtr{}; - bool fakeSync = false; - bool fakeLock = false; + + virtual ~pj_sqlite3_vfs(); }; // --------------------------------------------------------------------------- @@ -62,8 +63,14 @@ class SQLite3VFS { static std::unique_ptr create(bool fakeSync, bool fakeLock, bool skipStatJournalAndWAL); + +#ifdef EMBED_RESOURCE_FILES + static std::unique_ptr createMem(const void *membuffer, + size_t bufferSize); +#endif + const char *name() const; - sqlite3_vfs *raw() { return vfs_; } + sqlite3_vfs *raw() { return &(vfs_->base); } }; // --------------------------------------------------------------------------- diff --git a/test/cli/test_cs2cs_various.yaml b/test/cli/test_cs2cs_various.yaml index b6c621c853..21b0841748 100644 --- a/test/cli/test_cs2cs_various.yaml +++ b/test/cli/test_cs2cs_various.yaml @@ -1372,6 +1372,7 @@ tests: PROJ_DATA: $tmpdir PROJ_DEBUG: 2 PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY: YES + PROJ_ONLY_BEST_DEFAULT: "" args: EPSG:4326+5773 EPSG:4326+5782 in: 39 -3 0 grep-v: pj_open_lib diff --git a/test/unit/CMakeLists.txt b/test/unit/CMakeLists.txt index 7a84e34aaf..860927fefd 100644 --- a/test/unit/CMakeLists.txt +++ b/test/unit/CMakeLists.txt @@ -184,7 +184,14 @@ set_property(TEST proj_test_cpp_api if(TIFF_ENABLED) target_compile_definitions(proj_test_cpp_api PRIVATE -DTIFF_ENABLED) endif() - + +if(EMBED_RESOURCE_FILES) + target_compile_definitions(proj_test_cpp_api PRIVATE EMBED_RESOURCE_FILES) +endif() +if (USE_ONLY_EMBEDDED_RESOURCE_FILES) + target_compile_definitions(proj_test_cpp_api PRIVATE USE_ONLY_EMBEDDED_RESOURCE_FILES) +endif() + add_executable(gie_self_tests main.cpp gie_self_tests.cpp) diff --git a/test/unit/test_c_api.cpp b/test/unit/test_c_api.cpp index 950596294d..4a6450979a 100644 --- a/test/unit/test_c_api.cpp +++ b/test/unit/test_c_api.cpp @@ -4677,6 +4677,7 @@ TEST_F(CApi, proj_as_projjson) { // --------------------------------------------------------------------------- +#if !defined(EMBED_RESOURCE_FILES) && !defined(USE_ONLY_EMBEDDED_RESOURCE_FILES) TEST_F(CApi, proj_context_copy_from_default) { auto c_path = proj_context_get_database_path(m_ctxt); ASSERT_TRUE(c_path != nullptr); @@ -4726,6 +4727,7 @@ TEST_F(CApi, proj_context_copy_from_default) { std::string new_db_path(c_new_path); ASSERT_EQ(new_db_path, tmp_filename); } +#endif // ---------------------------------------------------------------------------