sys-fs/dwarfs: unbundling WIP

Package-Manager: Portage-3.0.18, Repoman-3.0.3 Signed-off-by: Alessandro Barbieri <lssndrbarbieri@gmail.com>
2026-07-20 20:43:09 -04:00 · 2021-05-02 03:49:27 +02:00
parent 6acfa62b63
commit 7f85b03b40
2 changed files with 182 additions and 3 deletions
--- a/sys-fs/dwarfs/dwarfs-0.5.4-r3.ebuild
+++ b/sys-fs/dwarfs/dwarfs-0.5.4-r3.ebuild
@@ -14,13 +14,13 @@ SRC_URI="https://github.com/mhx/dwarfs/releases/download/v${PV}/${P}.tar.bz2"

 LICENSE="GPL-3"
 SLOT="0"
-KEYWORDS=""
+KEYWORDS="~amd64"

 IUSE="python +jemalloc test"
 RESTRICT="!test? ( test )"
 REQUIRED_USE="python? ( ${PYTHON_REQUIRED_USE} )"

-PATCHES=( "${WORKDIR}/unbundle-folly-fbthrift.patch" )
+PATCHES=( "${FILESDIR}/unbundle.patch" )
 #TODO: unbundle libfsst

 RDEPEND="
@@ -34,6 +34,7 @@ RDEPEND="
 	dev-cpp/folly:=
 	dev-cpp/gflags
 	dev-cpp/glog[gflags]
+	dev-cpp/parallel-hashmap:=
 	dev-cpp/sparsehash
 	dev-libs/boost[context,threads,python?]
 	dev-libs/double-conversion
@@ -68,13 +69,14 @@ CMAKE_IN_SOURCE_BUILD=1
 CMAKE_WARN_UNUSED_CLI=0

 src_prepare(){
+	rm -r fsst zstd fbthrift folly xxHash parallel-hashmap || die
 	cmake_src_prepare
 	einfo "setting library path to $(get_libdir)"
 	sed "s/DESTINATION lib/DESTINATION $(get_libdir)/" -i CMakeLists.txt || die
 }

 src_configure(){
-	append-cxxflag "-I/usr/include/folly"
+	append-cxxflags "-I/usr/include"

 	einfo "setting configuration flags to:"
 	mycmakeargs=(
--- a/sys-fs/dwarfs/files/unbundle.patch
+++ b/sys-fs/dwarfs/files/unbundle.patch
@@ -0,0 +1,177 @@
+--- a/CMakeLists.txt
+++ b/CMakeLists.txt
+@@ -142,8 +142,6 @@
+     ON
+     CACHE BOOL "only build thrift compiler")
+ 
+-add_subdirectory(folly EXCLUDE_FROM_ALL)
+-add_subdirectory(fbthrift EXCLUDE_FROM_ALL)
+ if(NOT (ZSTD_FOUND AND PREFER_SYSTEM_ZSTD))
+   add_subdirectory(zstd/build/cmake EXCLUDE_FROM_ALL)
+ endif()
+@@ -298,7 +296,6 @@
+ if(WITH_TESTS OR WITH_BENCHMARKS)
+   add_library(test_helpers test/test_helpers.cpp test/test_strings.cpp
+                            test/loremipsum.cpp)
+-  target_link_libraries(test_helpers dwarfs folly)
+   set_property(TARGET test_helpers PROPERTY CXX_STANDARD 17)
+ endif()
+ 
+@@ -364,70 +361,7 @@
+ 
+ list(
+   APPEND
+-  FROZEN_THRIFT_SRC
+-  ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift/gen-cpp2/frozen_data.h
+-  ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift/gen-cpp2/frozen_data.cpp
+-  ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift/gen-cpp2/frozen_types.h
+-  ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift/gen-cpp2/frozen_types.tcc
+-  ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift/gen-cpp2/frozen_types.cpp
+-  ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift/gen-cpp2/frozen_types_custom_protocol.h
+-  ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift/gen-cpp2/frozen_constants.h
+-  ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift/gen-cpp2/frozen_constants.cpp
+-  ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift/gen-cpp2/frozen_metadata.h
+-  ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift/gen-cpp2/frozen_metadata.cpp
+-  ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift/gen-cpp2/frozen_visitation.h
+-  ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift/gen-cpp2/frozen_for_each_field.h
+-  ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift/gen-cpp2/frozen_visit_union.h)
+-
+-list(
+-  APPEND
+-  METADATA_THRIFT_SRC
+-  ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/metadata_constants.cpp
+-  ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/metadata_constants.h
+-  ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/metadata_data.cpp
+-  ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/metadata_data.h
+-  ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/metadata_for_each_field.h
+-  ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/metadata_layouts.cpp
+-  ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/metadata_layouts.h
+-  ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/metadata_metadata.cpp
+-  ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/metadata_metadata.h
+-  ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/metadata_types.cpp
+-  ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/metadata_types.h
+-  ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/metadata_types.tcc
+-  ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/metadata_types_custom_protocol.h
+-  ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/metadata_visit_union.h
+-  ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/metadata_visitation.h)
+-
+-add_custom_command(
+-  OUTPUT ${FROZEN_THRIFT_SRC}
+-  COMMAND mkdir -p ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift
+-  COMMAND
+-    cp ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift/thrift/lib/thrift/frozen.thrift
+-    ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift/
+-  COMMAND cd ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift &&
+-          ${CMAKE_CURRENT_BINARY_DIR}/bin/thrift1 --gen mstch_cpp2 frozen.thrift
+-  DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/bin/thrift1
+-          ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift/thrift/lib/thrift/frozen.thrift)
+-
+-add_custom_command(
+-  OUTPUT ${METADATA_THRIFT_SRC}
+-  COMMAND mkdir -p ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs
+-  COMMAND cp ${CMAKE_CURRENT_SOURCE_DIR}/thrift/metadata.thrift
+-          thrift/dwarfs/metadata.thrift
+-  COMMAND
+-    cd ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs &&
+-    ${CMAKE_CURRENT_BINARY_DIR}/bin/thrift1 --gen mstch_cpp2:frozen2
+-    metadata.thrift
+-  DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/bin/thrift1
+-          ${CMAKE_CURRENT_SOURCE_DIR}/thrift/metadata.thrift)
+-
+-list(
+-  APPEND
+   INCLUDE_DIRS
+-  ${CMAKE_CURRENT_BINARY_DIR}/folly
+-  ${CMAKE_CURRENT_BINARY_DIR}/thrift
+-  ${CMAKE_CURRENT_SOURCE_DIR}/folly
+-  ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift
+   ${CMAKE_CURRENT_BINARY_DIR})
+ 
+ if(NOT (ZSTD_FOUND AND PREFER_SYSTEM_ZSTD))
+@@ -445,39 +379,6 @@
+   list(APPEND INCLUDE_DIRS ${Jemalloc_INCLUDE_DIRS})
+ endif()
+ 
+-add_library(
+-  thrift_light
+-  ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift/thrift/lib/cpp2/FieldRef.cpp
+-  ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift/thrift/lib/cpp2/protocol/CompactProtocol.cpp
+-  ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift/thrift/lib/cpp2/protocol/BinaryProtocol.cpp
+-  ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift/thrift/lib/cpp2/protocol/DebugProtocol.cpp
+-  ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift/thrift/lib/cpp2/protocol/JSONProtocolCommon.cpp
+-  ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift/thrift/lib/cpp2/protocol/JSONProtocol.cpp
+-  ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift/thrift/lib/cpp/protocol/TProtocolException.cpp
+-  ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift/thrift/lib/cpp/util/VarintUtils.cpp
+-  ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift/thrift/lib/cpp2/gen/module_types_cpp.cpp
+-  ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift/thrift/lib/cpp2/frozen/Frozen.cpp
+-  ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift/thrift/lib/cpp2/frozen/FrozenUtil.cpp
+-  ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift/thrift/lib/cpp2/frozen/schema/MemorySchema.cpp
+-  ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift/gen-cpp2/frozen_data.cpp
+-  ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift/gen-cpp2/frozen_types.cpp)
+-
+-set_property(TARGET thrift_light PROPERTY CXX_STANDARD 17)
+-
+-target_include_directories(thrift_light PRIVATE ${INCLUDE_DIRS})
+-
+-add_library(
+-  metadata_thrift
+-  ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/metadata_layouts.cpp
+-  ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/metadata_types.cpp
+-  ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/metadata_data.cpp)
+-
+-set_property(TARGET metadata_thrift PROPERTY CXX_STANDARD 17)
+-
+-target_include_directories(metadata_thrift PRIVATE ${INCLUDE_DIRS})
+-
+-add_dependencies(metadata_thrift thrift_light)
+-
+ foreach(tgt dwarfs ${BINARY_TARGETS})
+   target_include_directories(
+     ${tgt} SYSTEM
+@@ -501,8 +402,6 @@
+   set_property(TARGET ${tgt} PROPERTY CXX_STANDARD_REQUIRED ON)
+   set_property(TARGET ${tgt} PROPERTY CXX_EXTENSIONS OFF)
+ 
+-  add_dependencies(${tgt} metadata_thrift)
+-
+   if(ENABLE_ASAN)
+     target_compile_options(${tgt} PRIVATE -fsanitize=address
+                                           -fno-omit-frame-pointer)
+@@ -535,25 +434,8 @@
+   endif()
+ endforeach()
+ 
+-# not sure why exactly, copied from fsst/CMakeLists.txt
+-if(CMAKE_BUILD_TYPE STREQUAL Release)
+-  set_source_files_properties(fsst/fsst_avx512.cpp PROPERTIES COMPILE_FLAGS -O1)
+-endif()
+-
+-add_library(
+-  fsst
+-  fsst/libfsst.cpp fsst/fsst_avx512.cpp fsst/fsst_avx512_unroll1.inc
+-  fsst/fsst_avx512_unroll2.inc fsst/fsst_avx512_unroll3.inc
+-  fsst/fsst_avx512_unroll4.inc)
+-
+-target_include_directories(dwarfs PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/fsst)
+-
+ target_link_libraries(
+   dwarfs
+-  metadata_thrift
+-  thrift_light
+-  folly
+-  fsst
+   ${Boost_LIBRARIES}
+   PkgConfig::LIBARCHIVE
+   PkgConfig::LIBLZ4
+@@ -582,11 +464,6 @@
+   if(USE_JEMALLOC)
+     target_link_libraries(${tgt} ${Jemalloc_LIBRARIES})
+   endif()
+-  if(TARGET folly_exception_tracer)
+-    target_link_libraries(
+-      ${tgt} -Wl,--whole-archive folly_exception_tracer_base
+-      folly_exception_tracer -Wl,--no-whole-archive)
+-  endif()
+ endforeach()
+ 
+ if(STATIC_BUILD_DO_NOT_USE)