Consolidated IPC extension into main project

CMakeLists.txt

@@ -34,9 +34,15 @@ set(NANOARROW_VERSION_PATCH "${nanoarrow_VERSION_PATCH}")
 
 # Feature options
 option(NANOARROW_IPC "Build IPC extension" OFF)
+option(NANOARROW_IPC_FLATCC_ROOT_DIR
+       "Root directory for flatcc include and lib directories" OFF)
+option(NANOARROW_FLATCC_INCLUDE_DIR "Include directory for flatcc includes" OFF)
+option(NANOARROW_FLATCC_LIB_DIR "Library directory that contains libflatccrt.a" OFF)
+
 option(NANOARROW_DEVICE "Build device extension" OFF)
 
 # Development options
+option(NANOARROW_BUILD_APPS "Build utility applications" OFF)
 option(NANOARROW_BUILD_TESTS "Build tests" OFF)
 option(NANOARROW_BUILD_BENCHMARKS "Build benchmarks" OFF)
 option(NANOARROW_BUILD_INTEGRATION_TESTS
@@ -63,6 +69,47 @@ endif()
 
 configure_file(src/nanoarrow/nanoarrow_config.h.in generated/nanoarrow_config.h)
 
+if(NANOARROW_IPC AND (NANOARROW_BUILD_TESTS OR NOT NANOARROW_BUNDLE))
+  # Add the flatcc (runtime) dependency
+  set(FLATCC_RTONLY
+    ON
+    CACHE INTERNAL "")
+  set(FLATCC_REFLECTION
+    OFF
+    CACHE INTERNAL "")
+
+  # A flatcc installation is unlikely, so default to building the vendored one
+  if(NOT NANOARROW_FLATCC_INCLUDE_DIR AND NOT NANOARROW_FLATCC_ROOT_DIR)
+    add_library(flatccrt STATIC
+      thirdparty/flatcc/src/runtime/builder.c
+      thirdparty/flatcc/src/runtime/emitter.c
+      thirdparty/flatcc/src/runtime/verifier.c
+      thirdparty/flatcc/src/runtime/refmap.c)
+
+    set(NANOARROW_FLATCC_INCLUDE_DIR
+      ${CMAKE_CURRENT_LIST_DIR}/thirdparty/flatcc/include)
+    target_include_directories(flatccrt
+      PUBLIC $<BUILD_INTERFACE:${NANOARROW_FLATCC_INCLUDE_DIR}>
+      $<INSTALL_INTERFACE:include>)
+  elseif(NOT NANOARROW_FLATCC_ROOT_DIR)
+    add_library(flatccrt STATIC IMPORTED)
+    set_target_properties(flatccrt
+      PROPERTIES IMPORTED_LOCATION
+      "${NANOARROW_FLATCC_LIB_DIR}/libflatccrt.a"
+      INTERFACE_INCLUDE_DIRECTORIES
+      "${NANOARROW_FLATCC_INCLUDE_DIR}")
+
+  elseif(NOT NANOARROW_FLATCC_INCLUDE_DIR)
+    set(NANOARROW_FLATCC_INCLUDE_DIR ${NANOARROW_FLATCC_ROOT_DIR}/include)
+    add_library(flatccrt STATIC IMPORTED)
+    set_target_properties(flatccrt
+      PROPERTIES IMPORTED_LOCATION
+      "${NANOARROW_FLATCC_ROOT_DIR}/lib/libflatccrt.a"
+      INTERFACE_INCLUDE_DIRECTORIES
+      "${NANOARROW_FLATCC_INCLUDE_DIR}")
+  endif()
+endif()
+
 if(NANOARROW_BUNDLE)
   # Combine all headers into amalgamation/nanoarrow.h in the build directory
   file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/amalgamation)
@@ -132,7 +179,61 @@ if(NANOARROW_BUNDLE)
                 ${NANOARROW_HPP_TEMP}
                 ${NANOARROW_GTEST_UTIL_HPP_TEMP}
                 ${NANOARROW_TESTING_HPP_TEMP}
-          DESTINATION ".")
+                DESTINATION ".")
+
+  if(NANOARROW_IPC)
+    # nanoarrow_ipc.h is already standalone
+    set(NANOARROW_IPC_H_TEMP ${CMAKE_BINARY_DIR}/amalgamation/nanoarrow/nanoarrow_ipc.h)
+    file(READ src/nanoarrow/nanoarrow_ipc.h SRC_FILE_CONTENTS)
+    file(WRITE ${NANOARROW_IPC_H_TEMP} "${SRC_FILE_CONTENTS}")
+
+    set(NANOARROW_IPC_C_TEMP ${CMAKE_BINARY_DIR}/amalgamation/nanoarrow/nanoarrow_ipc.c)
+    file(READ src/nanoarrow/nanoarrow_ipc_flatcc_generated.h SRC_FILE_CONTENTS)
+    file(WRITE ${NANOARROW_IPC_C_TEMP} "${SRC_FILE_CONTENTS}")
+    file(READ src/nanoarrow/nanoarrow_ipc_decoder.c SRC_FILE_CONTENTS)
+    file(APPEND ${NANOARROW_IPC_C_TEMP} "${SRC_FILE_CONTENTS}")
+    file(READ src/nanoarrow/nanoarrow_ipc_reader.c SRC_FILE_CONTENTS)
+    file(APPEND ${NANOARROW_IPC_C_TEMP} "${SRC_FILE_CONTENTS}")
+
+    # remove the include for the generated files in the bundled version
+    file(READ ${NANOARROW_IPC_C_TEMP} SRC_FILE_CONTENTS)
+    string(REGEX REPLACE "#include \"nanoarrow_ipc_flatcc_generated.h\"" ""
+                         SRC_FILE_CONTENTS "${SRC_FILE_CONTENTS}")
+    file(WRITE ${NANOARROW_IPC_C_TEMP} "${SRC_FILE_CONTENTS}")
+
+    # combine the flatcc sources
+    set(FLATCC_C_TEMP ${CMAKE_BINARY_DIR}/amalgamation/nanoarrow/flatcc.c)
+    file(READ thirdparty/flatcc/src/runtime/builder.c SRC_FILE_CONTENTS)
+    file(WRITE ${FLATCC_C_TEMP} "${SRC_FILE_CONTENTS}")
+    file(READ thirdparty/flatcc/src/runtime/emitter.c SRC_FILE_CONTENTS)
+    file(APPEND ${FLATCC_C_TEMP} "${SRC_FILE_CONTENTS}")
+    file(READ thirdparty/flatcc/src/runtime/verifier.c SRC_FILE_CONTENTS)
+    file(APPEND ${FLATCC_C_TEMP} "${SRC_FILE_CONTENTS}")
+    file(READ thirdparty/flatcc/src/runtime/refmap.c SRC_FILE_CONTENTS)
+    file(APPEND ${FLATCC_C_TEMP} "${SRC_FILE_CONTENTS}")
+
+    # Add a library that the tests can link against (but don't install it)
+    if(NANOARROW_BUILD_TESTS)
+      # Bundle flatcc into the nanoarrow_ipc library instead of
+      # link to the flatccrt static lib we declared above.
+      add_library(nanoarrow_ipc ${CMAKE_BINARY_DIR}/amalgamation/nanoarrow/nanoarrow_ipc.c
+        ${CMAKE_BINARY_DIR}/amalgamation/nanoarrow/flatcc.c)
+
+      target_include_directories(nanoarrow_ipc
+        PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src>
+        $<BUILD_INTERFACE:${nanoarrow_SOURCE_DIR}/src/nanoarrow>
+        $<BUILD_INTERFACE:${nanoarrow_BINARY_DIR}/generated>
+        #$<BUILD_INTERFACE:${NANOARROW_IPC_FLATCC_INCLUDE_DIR}>
+      )
+    endif()
+
+    # Install the amalgamated header and sources
+    install(FILES ${NANOARROW_IPC_H_TEMP} ${NANOARROW_IPC_C_TEMP} ${FLATCC_C_TEMP}
+            DESTINATION ".")
+
+    # Also install the flatcc headers
+    install(DIRECTORY thirdparty/flatcc/include/flatcc DESTINATION ".")
+  endif()
 else()
   add_library(nanoarrow src/nanoarrow/array.c src/nanoarrow/schema.c
                         src/nanoarrow/array_stream.c src/nanoarrow/utils.c)
@@ -215,6 +316,24 @@ else()
               NAMESPACE nanoarrow::)
     endif()
   endforeach()
+
+  if(NANOARROW_IPC)
+    add_library(nanoarrow_ipc src/nanoarrow/nanoarrow_ipc_decoder.c
+                              src/nanoarrow/nanoarrow_ipc_reader.c)
+    target_link_libraries(nanoarrow_ipc PRIVATE flatccrt)
+
+    target_include_directories(nanoarrow_ipc
+                               PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src>
+                                      $<BUILD_INTERFACE:${nanoarrow_SOURCE_DIR}/src/nanoarrow>
+                                      $<BUILD_INTERFACE:${nanoarrow_BINARY_DIR}/generated>
+                                      #$<BUILD_INTERFACE:${NANOARROW_IPC_FLATCC_INCLUDE_DIR}>
+                                      $<INSTALL_INTERFACE:include>)
+
+    install(TARGETS nanoarrow_ipc DESTINATION lib)
+    install(FILES src/nanoarrow/nanoarrow_ipc.h
+                  src/nanoarrow/nanoarrow_ipc_flatcc_generated.h
+            DESTINATION include/nanoarrow)
+  endif()
 endif()
 
 # Always build integration test if building tests
@@ -310,10 +429,70 @@ if(NANOARROW_BUILD_TESTS)
   gtest_discover_tests(nanoarrow_hpp_test DISCOVERY_TIMEOUT 10)
   gtest_discover_tests(nanoarrow_testing_test DISCOVERY_TIMEOUT 10)
   gtest_discover_tests(c_data_integration_test DISCOVERY_TIMEOUT 10)
+
+  if(NANOARROW_IPC)
+
+    # zlib to decode gzipped integration testing JSON files
+    # We don't use Arrow C++ for this because building Arrow C++ with zlib
+    # is not trivial on Windows.
+    find_package(ZLIB)
+    if(NOT ZLIB_FOUND)
+      # Wrapper around FetchContent that better isolates the zlib CMakeLists.txt
+      message(STATUS "Using FetchContent to build a static zlib")
+      add_subdirectory(thirdparty/zlib)
+    endif()
+
+    enable_testing()
+
+    add_executable(nanoarrow_ipc_decoder_test src/nanoarrow/nanoarrow_ipc_decoder_test.cc)
+    add_executable(nanoarrow_ipc_reader_test src/nanoarrow/nanoarrow_ipc_reader_test.cc)
+    add_executable(nanoarrow_ipc_files_test src/nanoarrow/nanoarrow_ipc_files_test.cc)
+    add_executable(nanoarrow_ipc_hpp_test src/nanoarrow/nanoarrow_ipc_hpp_test.cc)
+
+    if(NANOARROW_CODE_COVERAGE)
+      target_compile_options(ipc_coverage_config INTERFACE -O0 -g --coverage)
+      target_link_options(ipc_coverage_config INTERFACE --coverage)
+      target_link_libraries(nanoarrow_ipc PRIVATE ipc_coverage_config)
+    endif()
+    target_link_libraries(nanoarrow_ipc_decoder_test
+      nanoarrow_ipc
+      nanoarrow
+      ${NANOARROW_ARROW_TARGET}
+      gtest_main
+      ipc_coverage_config)
+    target_link_libraries(nanoarrow_ipc_reader_test
+      nanoarrow_ipc
+      nanoarrow
+      gtest_main
+      ipc_coverage_config)
+    target_link_libraries(nanoarrow_ipc_files_test
+      nanoarrow_ipc
+      nanoarrow
+      ${NANOARROW_ARROW_TARGET}
+      nlohmann_json
+      ZLIB::ZLIB
+      gtest_main
+      ipc_coverage_config)
+    target_link_libraries(nanoarrow_ipc_hpp_test
+      nanoarrow_ipc
+      nanoarrow
+      ${NANOARROW_ARROW_TARGET}
+      gtest_main
+      ipc_coverage_config)
+
+    include(GoogleTest)
+    gtest_discover_tests(nanoarrow_ipc_decoder_test)
+    gtest_discover_tests(nanoarrow_ipc_reader_test)
+    gtest_discover_tests(nanoarrow_ipc_files_test)
+    gtest_discover_tests(nanoarrow_ipc_hpp_test)
+  endif()
 endif()
 
-if(NANOARROW_BUILD_BENCHMARKS OR NANOARROW_IPC)
-  add_subdirectory(extensions/nanoarrow_ipc)
+if(NANOARROW_BUILD_APPS)
+  if(NANOARROW_IPC)
+    add_executable(dump_stream src/apps/dump_stream.c)
+    target_link_libraries(dump_stream nanoarrow_ipc nanoarrow)
+  endif()
 endif()
 
 if(NANOARROW_DEVICE)

extensions/nanoarrow_ipc/dev/update_fbs.sh → dev/update_fbs.sh

@@ -24,7 +24,7 @@ curl -L https://github.com/apache/arrow/raw/main/format/Message.fbs --output Mes
 curl -L https://github.com/apache/arrow/raw/main/format/File.fbs --output File.fbs
 
 # compile using flatcc
-flatcc --common --reader --builder --verifier --recursive --outfile ../../src/nanoarrow_ipc/nanoarrow_ipc_flatcc_generated.h *.fbs
+flatcc --common --reader --builder --verifier --recursive --outfile ../src/nanoarrow/nanoarrow_ipc_flatcc_generated.h *.fbs
 
 # clean up
 cd ..

extensions/nanoarrow_ipc/dev/update_vendored_flatcc.sh → dev/update_vendored_flatcc.sh

@@ -47,7 +47,7 @@ makedepend -s#: -f- -- -I. -DFLATCC_PORTABLE -- 2>/dev/null \
   ../src/runtime/builder.c \
   ../src/runtime/verifier.c \
   ../src/runtime/refmap.c \
-  ../../../../src/nanoarrow/nanoarrow_ipc.c | \
+  ../src/nanoarrow/nanoarrow_ipc.c | \
   # Remove the '<src file>.o: ' prefix
   sed 's/[^:]*: *//' | \
   # Spaces to new lines