mirrors/guru
1
0
Fork 0
mirror of https://github.com/gentoo-mirror/guru.git synced 2026-07-21 21:13:27 -04:00
Code Issues Packages Projects Releases Wiki Activity

dwarfs: unorphaned, work in progress

Browse Source 
Signed-off-by: Denis Reva <denis7774@gmail.com>
This commit is contained in:
Denis Reva
2023-02-28 17:53:52 +05:00
parent 5dc6a108df
commit 253b9fd4d7
4 changed files with 289 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
sys-fs/dwarfs/Manifest Normal file
View File

@@ -0,0 +1 @@
DIST dwarfs-0.7.0-RC4.tar.xz 14425176 BLAKE2B fdb2852da81f8d567c19dc83e3f7dd74e0de35e56579d50b6ce9ee6adda2d540bbbc7d5655a499f19ee340a2f904bfe9b2b977b9539e8e99dc9e8667b9b7435e SHA512 1e0d603507f93b6bcb79050877de0ac64eed931b192b43466d3b5e1ee55a759261ad937d33f24d145a61e8b3d431405a35bbd8ff95ead10ccdabe14ba4d840a5

103
sys-fs/dwarfs/dwarfs-0.7.0_rc4.ebuild Normal file
View File

@@ -0,0 +1,103 @@
# Copyright 1999-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
EAPI=7
PYTHON_COMPAT=( python3_{10..11} )
inherit check-reqs cmake flag-o-matic python-single-r1
MY_P="${P/_rc/-RC}"
DESCRIPTION="A fast very high compression read-only FUSE file system"
HOMEPAGE="https://github.com/mhx/dwarfs"
SRC_URI="https://github.com/mhx/dwarfs/releases/download/v0.7.0-RC4/dwarfs-0.7.0-RC4.tar.xz" #TODO: change to ${PV}
LICENSE="GPL-3"
SLOT="0"
KEYWORDS=""
IUSE="python +jemalloc test man"
S="${WORKDIR}/${MY_P}"
RDEPEND="
${PYTHON_DEPS}
app-arch/libarchive
app-arch/lz4
app-arch/snappy
app-arch/xz-utils
app-arch/zstd
dev-cpp/fbthrift:=
dev-cpp/folly:=
dev-cpp/gflags
dev-cpp/glog[gflags]
dev-cpp/parallel-hashmap:=
dev-cpp/sparsehash
dev-libs/boost[context,threads(+),python?]
dev-libs/double-conversion
dev-libs/fsst:=
dev-libs/libevent
dev-libs/libfmt
dev-libs/xxhash
sys-fs/fuse:3
sys-libs/binutils-libs
sys-libs/libunwind
sys-libs/zlib
jemalloc? ( >=dev-libs/jemalloc-5.3.0-r1 )
"
DEPEND="
${RDEPEND}
sys-devel/flex
!sys-fs/dwarfs-bin
"
BDEPEND="
man? ( || ( app-text/ronn app-text/ronn-ng ) )
sys-devel/bison
virtual/pkgconfig
test? ( dev-cpp/gtest )
"
DOCS=( "README.md" "CHANGES.md" "TODO" )
RESTRICT="!test? ( test )"
REQUIRED_USE="python? ( ${PYTHON_REQUIRED_USE} )"
PATCHES=( "${FILESDIR}/${P}-unbundle.patch" )
CHECKREQS_DISK_BUILD="1300M"
CMAKE_IN_SOURCE_BUILD=1
CMAKE_WARN_UNUSED_CLI=0
src_prepare(){
rm -r fsst zstd fbthrift/* folly xxHash parallel-hashmap || die
cmake_src_prepare
sed "s/DESTINATION lib/DESTINATION $(get_libdir)/" -i CMakeLists.txt || die
}
src_configure(){
append-cxxflags "-I/usr/include"
mycmakeargs=(
-DUSE_JEMALLOC=$(usex jemalloc ON OFF)
-DWITH_PYTHON=$(usex python ON OFF)
-DWITH_TESTS=$(usex test ON OFF)
-WITH_MAN_PAGES=$(usex man ON OFF)
-DPREFER_SYSTEM_ZSTD=1
-DPREFER_SYSTEM_XXHASH=1
-DPREFER_SYSTEM_GTEST=1
-DWITH_LEGACY_FUSE=0
)
use python && mycmakeargs+=( "-DWITH_PYTHON_VERSION=${EPYTHON#python}" )
cmake_src_configure
}
src_install(){
cmake_src_install
dolib.so libdwarfs.so
}
pkg_postinst(){
elog "You may find more information in the"
elog "${HOMEPAGE}"
elog "About creating: ${HOMEPAGE}/blob/main/doc/mkdwarfs.md"
elog "About mounting: ${HOMEPAGE}/blob/main/doc/dwarfs.md"
}

161
sys-fs/dwarfs/files/dwarfs-0.7.0_rc4-unbundle.patch Normal file
View File

@@ -0,0 +1,161 @@
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -151,8 +151,6 @@
ON
CACHE BOOL "only build thrift compiler")
-add_subdirectory(folly EXCLUDE_FROM_ALL)
-add_subdirectory(fbthrift EXCLUDE_FROM_ALL)
if(NOT (ZSTD_FOUND AND PREFER_SYSTEM_ZSTD))
add_subdirectory(zstd/build/cmake EXCLUDE_FROM_ALL)
endif()
@@ -383,23 +381,6 @@
list(
APPEND
- FROZEN_THRIFT_SRC
- ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift/gen-cpp2/frozen_data.h
- ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift/gen-cpp2/frozen_data.cpp
- ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift/gen-cpp2/frozen_types.h
- ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift/gen-cpp2/frozen_types.tcc
- ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift/gen-cpp2/frozen_types.cpp
- ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift/gen-cpp2/frozen_types_custom_protocol.h
- ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift/gen-cpp2/frozen_constants.h
- ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift/gen-cpp2/frozen_constants.cpp
- ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift/gen-cpp2/frozen_metadata.h
- ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift/gen-cpp2/frozen_metadata.cpp
- ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift/gen-cpp2/frozen_visitation.h
- ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift/gen-cpp2/frozen_for_each_field.h
- ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift/gen-cpp2/frozen_visit_union.h)
-
-list(
- APPEND
METADATA_THRIFT_SRC
${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/metadata_constants.cpp
${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/metadata_constants.h
@@ -418,35 +399,20 @@
${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/metadata_visitation.h)
add_custom_command(
- OUTPUT ${FROZEN_THRIFT_SRC}
- COMMAND mkdir -p ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift
- COMMAND
- cp ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift/thrift/lib/thrift/frozen.thrift
- ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift/
- COMMAND cd ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift &&
- ${CMAKE_CURRENT_BINARY_DIR}/bin/thrift1 --gen mstch_cpp2 frozen.thrift
- DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/bin/thrift1
- ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift/thrift/lib/thrift/frozen.thrift)
-
-add_custom_command(
OUTPUT ${METADATA_THRIFT_SRC}
COMMAND mkdir -p ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs
COMMAND cp ${CMAKE_CURRENT_SOURCE_DIR}/thrift/metadata.thrift
thrift/dwarfs/metadata.thrift
COMMAND
cd ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs &&
- ${CMAKE_CURRENT_BINARY_DIR}/bin/thrift1 --gen mstch_cpp2:frozen2
+ thrift1 --gen mstch_cpp2:frozen2
metadata.thrift
- DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/bin/thrift1
- ${CMAKE_CURRENT_SOURCE_DIR}/thrift/metadata.thrift)
+ DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/thrift/metadata.thrift)
list(
APPEND
INCLUDE_DIRS
- ${CMAKE_CURRENT_BINARY_DIR}/folly
${CMAKE_CURRENT_BINARY_DIR}/thrift
- ${CMAKE_CURRENT_SOURCE_DIR}/folly
- ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift
${CMAKE_CURRENT_BINARY_DIR})
if(NOT (ZSTD_FOUND AND PREFER_SYSTEM_ZSTD))
@@ -472,43 +438,24 @@
endif()
add_library(
- thrift_light
- ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift/thrift/lib/cpp2/FieldRef.cpp
- ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift/thrift/lib/cpp2/protocol/CompactProtocol.cpp
- ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift/thrift/lib/cpp2/protocol/BinaryProtocol.cpp
- ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift/thrift/lib/cpp2/protocol/DebugProtocol.cpp
- ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift/thrift/lib/cpp2/protocol/JSONProtocolCommon.cpp
- ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift/thrift/lib/cpp2/protocol/JSONProtocol.cpp
- ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift/thrift/lib/cpp/protocol/TProtocolException.cpp
- ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift/thrift/lib/cpp/util/VarintUtils.cpp
- ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift/thrift/lib/cpp2/gen/module_types_cpp.cpp
- ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift/thrift/lib/cpp2/frozen/Frozen.cpp
- ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift/thrift/lib/cpp2/frozen/FrozenUtil.cpp
- ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift/thrift/lib/cpp2/frozen/schema/MemorySchema.cpp
- ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift/gen-cpp2/frozen_data.cpp
- ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift/gen-cpp2/frozen_types.cpp)
-
-set_property(TARGET thrift_light PROPERTY CXX_STANDARD 17)
-
-target_include_directories(thrift_light PRIVATE ${INCLUDE_DIRS})
-
-add_library(
metadata_thrift
+ STATIC
${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/metadata_layouts.cpp
${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/metadata_types.cpp
${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/gen-cpp2/metadata_data.cpp)
set_property(TARGET metadata_thrift PROPERTY CXX_STANDARD 17)
-
+set_property(TARGET metadata_thrift PROPERTY POSITION_INDEPENDENT_CODE ON)
+set_property(TARGET metadata_thrift PROPERTY CXX_VISIBILITY_PRESET hidden)
+set_property(TARGET metadata_thrift PROPERTY VISIBILITY_INLINES_HIDDEN 1)
target_include_directories(metadata_thrift PRIVATE ${INCLUDE_DIRS})
-
-add_dependencies(metadata_thrift thrift_light)
+target_link_libraries(metadata_thrift fmt glog folly)
foreach(tgt dwarfs ${BINARY_TARGETS})
target_include_directories(
${tgt} SYSTEM
PRIVATE ${Boost_INCLUDE_DIRS} ${Python3_INCLUDE_DIRS} ${INCLUDE_DIRS}
- ${CMAKE_CURRENT_SOURCE_DIR}/parallel-hashmap)
+ )
target_include_directories(${tgt} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
@@ -565,23 +512,11 @@
endif()
endforeach()
-# not sure why exactly, copied from fsst/CMakeLists.txt
-if(CMAKE_BUILD_TYPE STREQUAL Release)
- set_source_files_properties(fsst/fsst_avx512.cpp PROPERTIES COMPILE_FLAGS -O1)
-endif()
-
-add_library(
- fsst
- fsst/libfsst.cpp fsst/fsst_avx512.cpp fsst/fsst_avx512_unroll1.inc
- fsst/fsst_avx512_unroll2.inc fsst/fsst_avx512_unroll3.inc
- fsst/fsst_avx512_unroll4.inc)
-
-target_include_directories(dwarfs PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/fsst)
-
target_link_libraries(
dwarfs
+ thriftprotocol
+ thriftfrozen2
metadata_thrift
- thrift_light
folly
fsst
${Boost_LIBRARIES}
@@ -612,11 +547,6 @@
if(USE_JEMALLOC)
target_link_libraries(${tgt} ${Jemalloc_LIBRARIES})
endif()
- if(DWARFS_USE_EXCEPTION_TRACER)
- target_link_libraries(
- ${tgt} -Wl,--whole-archive folly_exception_tracer_base
- folly_exception_tracer -Wl,--no-whole-archive)
- endif()
endforeach()
if(STATIC_BUILD_DO_NOT_USE)

24
sys-fs/dwarfs/metadata.xml Normal file
View File

@@ -0,0 +1,24 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE pkgmetadata SYSTEM 'http://www.gentoo.org/dtd/metadata.dtd'>
<pkgmetadata>
<maintainer type="person">
<email>denis7774@gmail.com</email>
<name>Denis Reva</name>
<description>rarogcmex</description>
</maintainer>
<longdescription lang="en">
DwarFS is a read-only file system with a focus on achieving very high compression ratios in particular for very redundant data.
This probably doesn't sound very exciting, because if it's redundant, it should compress well. However, I found that other read-only, compressed file systems don't do a very good job at making use of this redundancy. See here for a comparison with other compressed file systems.
DwarFS also doesn't compromise on speed and for my use cases I've found it to be on par with or perform better than SquashFS. For my primary use case, DwarFS compression is an order of magnitude better than SquashFS compression, it's 4 times faster to build the file system, it's typically faster to access files on DwarFS and it uses less CPU resources.
Distinct features of DwarFS are:
* Clustering of files by similarity using a similarity hash function. This makes it easier to exploit the redundancy across file boundaries.
* Segmentation analysis across file system blocks in order to reduce the size of the uncompressed file system. This saves memory when using the compressed file system and thus potentially allows for higher cache hit rates as more data can be kept in the cache.
* Highly multi-threaded implementation. Both the file system creation tool as well as the FUSE driver are able to make good use of the many cores of your system.
* Optional experimental Lua support to provide custom filtering and ordering functionality.
</longdescription>
<upstream>
<bugs-to>https://github.com/mhx/dwarfs/issues</bugs-to>
<remote-id type="github">mhx/dwarfs</remote-id>
</upstream>
</pkgmetadata>