ryan77627/guix
1
0
Fork 0
mirror of https://git.in.rschanz.org/ryan77627/guix.git synced 2024-12-26 14:28:15 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

gnu: ceph: Update to 16.2.6.

Browse source 
* gnu/packages/storage.scm (ceph): Update to 16.2.6.
[source](patches): Add two new patches.  Remove one obsolete.
[source](snippet): Unbundle fmt.
[arguments]: Remove -DWITH_PYTHON3 from #:configure-flags. Add
-DWITH_SYSTEMD.  Adjust RUNPATH substitution, and the wrap-python-scripts
phase, for upstream changes.  Remove trailing #t's.
[inputs]: Remove PYTHON-SIX and PYTHON-WRAPPER.  Add FMT, ICU4C,
PYTHON-PYYAML, PYTHON, and SQLITE.
* gnu/packages/patches/ceph-fix-snappy-breaking-change.patch: Delete file.
* gnu/packages/patches/ceph-boost-compat.patch,
gnu/packages/patches/ceph-rocksdb-compat.patch: New files.
* gnu/local.mk (dist_patch_DATA): Adjust accordingly.
This commit is contained in:
Marius Bakke 2021-11-05 22:49:43 +01:00
parent 075df3d3e2
commit 1525443e02
No known key found for this signature in database
GPG key ID: A2A06DF2A33A54FA
5 changed files with 354 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
gnu/local.mk
View file

@ -942,8 +942,9 @@ dist_patch_DATA = \
%D%/packages/patches/cdparanoia-fpic.patch \
%D%/packages/patches/cdrtools-3.01-mkisofs-isoinfo.patch \
%D%/packages/patches/ceph-disable-cpu-optimizations.patch \
%D%/packages/patches/ceph-boost-compat.patch \
%D%/packages/patches/ceph-rocksdb-compat.patch \
%D%/packages/patches/cheese-vala-update.patch \
%D%/packages/patches/ceph-fix-snappy-breaking-change.patch \
%D%/packages/patches/chez-scheme-build-util-paths-backport.patch \
%D%/packages/patches/chmlib-inttypes.patch \
%D%/packages/patches/cl-asdf-config-directories.patch \

18
gnu/packages/patches/ceph-boost-compat.patch Normal file
View file

@ -0,0 +1,18 @@
Add extra includes required for Boost 1.75 and later.
Taken from upstram:
https://github.com/ceph/ceph/commit/ebf3a0398f18eab67d2ba25e6a10b41ff140f6a4
diff --git a/src/rgw/rgw_string.h b/src/rgw/rgw_string.h
index 257daa9c1fe6e..90e64f98a2587 100644
--- a/src/rgw/rgw_string.h
+++ b/src/rgw/rgw_string.h
@@ -8,5 +8,7 @@
#include <stdlib.h>
#include <limits.h>
#include <string_view>
+#include <string>
+#include <stdexcept>
#include <boost/container/small_vector.hpp>

13
gnu/packages/patches/ceph-fix-snappy-breaking-change.patch
View file

@ -1,13 +0,0 @@
Patch tracked upstream at https://tracker.ceph.com/issues/50934
--- a/src/compressor/snappy/SnappyCompressor.h
+++ b/src/compressor/snappy/SnappyCompressor.h
@@ -96,7 +96,7 @@ class SnappyCompressor : public Compressor {
if (qat_enabled)
return qat_accel.decompress(p, compressed_len, dst);
#endif
- snappy::uint32 res_len = 0;
+ uint32_t res_len = 0;
BufferlistSource source_1(p, compressed_len);
if (!snappy::GetUncompressedLength(&source_1, &res_len)) {
return -1;

303
gnu/packages/patches/ceph-rocksdb-compat.patch Normal file
View file

@ -0,0 +1,303 @@
Adjust for newer versions of RocksDB.
Taken from upstream:
https://github.com/ceph/ceph/pull/42815
https://github.com/ceph/ceph/commit/ff7f192ea3cf88ca1098bcf9396ff4f8ed1e8792.diff
diff --git a/src/kv/rocksdb_cache/BinnedLRUCache.cc b/src/kv/rocksdb_cache/BinnedLRUCache.cc
index 0d657883e92de..47c56e2ddd769 100644
--- a/src/kv/rocksdb_cache/BinnedLRUCache.cc
+++ b/src/kv/rocksdb_cache/BinnedLRUCache.cc
@@ -151,13 +151,20 @@ void BinnedLRUCacheShard::EraseUnRefEntries() {
}
}
-void BinnedLRUCacheShard::ApplyToAllCacheEntries(void (*callback)(void*, size_t),
- bool thread_safe) {
+void BinnedLRUCacheShard::ApplyToAllCacheEntries(
+ const std::function<void(const rocksdb::Slice& key,
+ void* value,
+ size_t charge,
+ DeleterFn)>& callback,
+ bool thread_safe)
+{
if (thread_safe) {
mutex_.lock();
}
table_.ApplyToAllCacheEntries(
- [callback](BinnedLRUHandle* h) { callback(h->value, h->charge); });
+ [callback](BinnedLRUHandle* h) {
+ callback(h->key(), h->value, h->charge, h->deleter);
+ });
if (thread_safe) {
mutex_.unlock();
}
@@ -345,7 +352,7 @@ bool BinnedLRUCacheShard::Release(rocksdb::Cache::Handle* handle, bool force_era
rocksdb::Status BinnedLRUCacheShard::Insert(const rocksdb::Slice& key, uint32_t hash, void* value,
size_t charge,
- void (*deleter)(const rocksdb::Slice& key, void* value),
+ DeleterFn deleter,
rocksdb::Cache::Handle** handle, rocksdb::Cache::Priority priority) {
auto e = new BinnedLRUHandle();
rocksdb::Status s;
@@ -464,6 +471,12 @@ std::string BinnedLRUCacheShard::GetPrintableOptions() const {
return std::string(buffer);
}
+DeleterFn BinnedLRUCacheShard::GetDeleter(rocksdb::Cache::Handle* h) const
+{
+ auto* handle = reinterpret_cast<BinnedLRUHandle*>(h);
+ return handle->deleter;
+}
+
BinnedLRUCache::BinnedLRUCache(CephContext *c,
size_t capacity,
int num_shard_bits,
@@ -519,6 +532,13 @@ void BinnedLRUCache::DisownData() {
#endif // !__SANITIZE_ADDRESS__
}
+#if (ROCKSDB_MAJOR >= 6 && ROCKSDB_MINOR >= 22)
+DeleterFn BinnedLRUCache::GetDeleter(Handle* handle) const
+{
+ return reinterpret_cast<const BinnedLRUHandle*>(handle)->deleter;
+}
+#endif
+
size_t BinnedLRUCache::TEST_GetLRUSize() {
size_t lru_size_of_all_shards = 0;
for (int i = 0; i < num_shards_; i++) {
diff --git a/src/kv/rocksdb_cache/BinnedLRUCache.h b/src/kv/rocksdb_cache/BinnedLRUCache.h
index 85608be0e5734..88bf4502e8927 100644
--- a/src/kv/rocksdb_cache/BinnedLRUCache.h
+++ b/src/kv/rocksdb_cache/BinnedLRUCache.h
@@ -56,7 +56,7 @@ std::shared_ptr<rocksdb::Cache> NewBinnedLRUCache(
struct BinnedLRUHandle {
void* value;
- void (*deleter)(const rocksdb::Slice&, void* value);
+ DeleterFn deleter;
BinnedLRUHandle* next_hash;
BinnedLRUHandle* next;
BinnedLRUHandle* prev;
@@ -189,7 +189,7 @@ class alignas(CACHE_LINE_SIZE) BinnedLRUCacheShard : public CacheShard {
// Like Cache methods, but with an extra "hash" parameter.
virtual rocksdb::Status Insert(const rocksdb::Slice& key, uint32_t hash, void* value,
size_t charge,
- void (*deleter)(const rocksdb::Slice& key, void* value),
+ DeleterFn deleter,
rocksdb::Cache::Handle** handle,
rocksdb::Cache::Priority priority) override;
virtual rocksdb::Cache::Handle* Lookup(const rocksdb::Slice& key, uint32_t hash) override;
@@ -205,13 +205,19 @@ class alignas(CACHE_LINE_SIZE) BinnedLRUCacheShard : public CacheShard {
virtual size_t GetUsage() const override;
virtual size_t GetPinnedUsage() const override;
- virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
- bool thread_safe) override;
+ virtual void ApplyToAllCacheEntries(
+ const std::function<void(const rocksdb::Slice& key,
+ void* value,
+ size_t charge,
+ DeleterFn)>& callback,
+ bool thread_safe) override;
virtual void EraseUnRefEntries() override;
virtual std::string GetPrintableOptions() const override;
+ virtual DeleterFn GetDeleter(rocksdb::Cache::Handle* handle) const override;
+
void TEST_GetLRUList(BinnedLRUHandle** lru, BinnedLRUHandle** lru_low_pri);
// Retrieves number of elements in LRU, for unit test purpose only
@@ -304,7 +310,9 @@ class BinnedLRUCache : public ShardedCache {
virtual size_t GetCharge(Handle* handle) const override;
virtual uint32_t GetHash(Handle* handle) const override;
virtual void DisownData() override;
-
+#if (ROCKSDB_MAJOR >= 6 && ROCKSDB_MINOR >= 22)
+ virtual DeleterFn GetDeleter(Handle* handle) const override;
+#endif
// Retrieves number of elements in LRU, for unit test purpose only
size_t TEST_GetLRUSize();
// Sets the high pri pool ratio
diff --git a/src/kv/rocksdb_cache/ShardedCache.cc b/src/kv/rocksdb_cache/ShardedCache.cc
index 367140a94d8be..6cbd89ad6472c 100644
--- a/src/kv/rocksdb_cache/ShardedCache.cc
+++ b/src/kv/rocksdb_cache/ShardedCache.cc
@@ -44,7 +44,7 @@ void ShardedCache::SetStrictCapacityLimit(bool strict_capacity_limit) {
}
rocksdb::Status ShardedCache::Insert(const rocksdb::Slice& key, void* value, size_t charge,
- void (*deleter)(const rocksdb::Slice& key, void* value),
+ DeleterFn deleter,
rocksdb::Cache::Handle** handle, Priority priority) {
uint32_t hash = HashSlice(key);
return GetShard(Shard(hash))
@@ -109,13 +109,36 @@ size_t ShardedCache::GetPinnedUsage() const {
return usage;
}
+#if (ROCKSDB_MAJOR >= 6 && ROCKSDB_MINOR >= 22)
+DeleterFn ShardedCache::GetDeleter(Handle* handle) const
+{
+ uint32_t hash = GetHash(handle);
+ return GetShard(Shard(hash))->GetDeleter(handle);
+}
+
+void ShardedCache::ApplyToAllEntries(
+ const std::function<void(const rocksdb::Slice& key, void* value, size_t charge,
+ DeleterFn deleter)>& callback,
+ const ApplyToAllEntriesOptions& opts)
+{
+ int num_shards = 1 << num_shard_bits_;
+ for (int s = 0; s < num_shards; s++) {
+ GetShard(s)->ApplyToAllCacheEntries(callback, true /* thread_safe */);
+ }
+}
+#else
void ShardedCache::ApplyToAllCacheEntries(void (*callback)(void*, size_t),
bool thread_safe) {
int num_shards = 1 << num_shard_bits_;
for (int s = 0; s < num_shards; s++) {
- GetShard(s)->ApplyToAllCacheEntries(callback, thread_safe);
+ GetShard(s)->ApplyToAllCacheEntries(
+ [callback](const rocksdb::Slice&, void* value, size_t charge, DeleterFn) {
+ callback(value, charge);
+ },
+ thread_safe);
}
}
+#endif
void ShardedCache::EraseUnRefEntries() {
int num_shards = 1 << num_shard_bits_;
@@ -131,7 +154,7 @@ std::string ShardedCache::GetPrintableOptions() const {
char buffer[kBufferSize];
{
std::lock_guard<std::mutex> l(capacity_mutex_);
- snprintf(buffer, kBufferSize, " capacity : %" ROCKSDB_PRIszt "\n",
+ snprintf(buffer, kBufferSize, " capacity : %zu\n",
capacity_);
ret.append(buffer);
snprintf(buffer, kBufferSize, " num_shard_bits : %d\n", num_shard_bits_);
diff --git a/src/kv/rocksdb_cache/ShardedCache.h b/src/kv/rocksdb_cache/ShardedCache.h
index 4d64893ab1c7b..f98421a09a33a 100644
--- a/src/kv/rocksdb_cache/ShardedCache.h
+++ b/src/kv/rocksdb_cache/ShardedCache.h
@@ -14,6 +14,7 @@
#include <string>
#include <mutex>
+#include "rocksdb/version.h"
#include "rocksdb/cache.h"
#include "include/ceph_hash.h"
#include "common/PriorityCache.h"
@@ -22,10 +23,11 @@
#ifndef CACHE_LINE_SIZE
#define CACHE_LINE_SIZE 64 // XXX arch-specific define
#endif
-#define ROCKSDB_PRIszt "zu"
namespace rocksdb_cache {
+using DeleterFn = void (*)(const rocksdb::Slice& key, void* value);
+
// Single cache shard interface.
class CacheShard {
public:
@@ -34,7 +36,7 @@ class CacheShard {
virtual rocksdb::Status Insert(const rocksdb::Slice& key, uint32_t hash, void* value,
size_t charge,
- void (*deleter)(const rocksdb::Slice& key, void* value),
+ DeleterFn deleter,
rocksdb::Cache::Handle** handle, rocksdb::Cache::Priority priority) = 0;
virtual rocksdb::Cache::Handle* Lookup(const rocksdb::Slice& key, uint32_t hash) = 0;
virtual bool Ref(rocksdb::Cache::Handle* handle) = 0;
@@ -44,10 +46,15 @@ class CacheShard {
virtual void SetStrictCapacityLimit(bool strict_capacity_limit) = 0;
virtual size_t GetUsage() const = 0;
virtual size_t GetPinnedUsage() const = 0;
- virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
- bool thread_safe) = 0;
+ virtual void ApplyToAllCacheEntries(
+ const std::function<void(const rocksdb::Slice& key,
+ void* value,
+ size_t charge,
+ DeleterFn)>& callback,
+ bool thread_safe) = 0;
virtual void EraseUnRefEntries() = 0;
virtual std::string GetPrintableOptions() const { return ""; }
+ virtual DeleterFn GetDeleter(rocksdb::Cache::Handle* handle) const = 0;
};
// Generic cache interface which shards cache by hash of keys. 2^num_shard_bits
@@ -57,34 +64,43 @@ class ShardedCache : public rocksdb::Cache, public PriorityCache::PriCache {
public:
ShardedCache(size_t capacity, int num_shard_bits, bool strict_capacity_limit);
virtual ~ShardedCache() = default;
+ // rocksdb::Cache
virtual const char* Name() const override = 0;
- virtual CacheShard* GetShard(int shard) = 0;
- virtual const CacheShard* GetShard(int shard) const = 0;
- virtual void* Value(Handle* handle) override = 0;
- virtual size_t GetCharge(Handle* handle) const = 0;
- virtual uint32_t GetHash(Handle* handle) const = 0;
- virtual void DisownData() override = 0;
-
- virtual void SetCapacity(size_t capacity) override;
- virtual void SetStrictCapacityLimit(bool strict_capacity_limit) override;
-
virtual rocksdb::Status Insert(const rocksdb::Slice& key, void* value, size_t charge,
- void (*deleter)(const rocksdb::Slice& key, void* value),
+ DeleterFn,
rocksdb::Cache::Handle** handle, Priority priority) override;
virtual rocksdb::Cache::Handle* Lookup(const rocksdb::Slice& key, rocksdb::Statistics* stats) override;
virtual bool Ref(rocksdb::Cache::Handle* handle) override;
virtual bool Release(rocksdb::Cache::Handle* handle, bool force_erase = false) override;
+ virtual void* Value(Handle* handle) override = 0;
virtual void Erase(const rocksdb::Slice& key) override;
virtual uint64_t NewId() override;
- virtual size_t GetCapacity() const override;
+ virtual void SetCapacity(size_t capacity) override;
+ virtual void SetStrictCapacityLimit(bool strict_capacity_limit) override;
virtual bool HasStrictCapacityLimit() const override;
+ virtual size_t GetCapacity() const override;
virtual size_t GetUsage() const override;
virtual size_t GetUsage(rocksdb::Cache::Handle* handle) const override;
virtual size_t GetPinnedUsage() const override;
+ virtual size_t GetCharge(Handle* handle) const = 0;
+#if (ROCKSDB_MAJOR >= 6 && ROCKSDB_MINOR >= 22)
+ virtual DeleterFn GetDeleter(Handle* handle) const override;
+#endif
+ virtual void DisownData() override = 0;
+#if (ROCKSDB_MAJOR >= 6 && ROCKSDB_MINOR >= 22)
+ virtual void ApplyToAllEntries(
+ const std::function<void(const rocksdb::Slice& key, void* value, size_t charge,
+ DeleterFn deleter)>& callback,
+ const ApplyToAllEntriesOptions& opts) override;
+#else
virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
bool thread_safe) override;
+#endif
virtual void EraseUnRefEntries() override;
virtual std::string GetPrintableOptions() const override;
+ virtual CacheShard* GetShard(int shard) = 0;
+ virtual const CacheShard* GetShard(int shard) const = 0;
+ virtual uint32_t GetHash(Handle* handle) const = 0;
int GetNumShardBits() const { return num_shard_bits_; }
@@ -120,7 +136,7 @@ class ShardedCache : public rocksdb::Cache, public PriorityCache::PriCache {
// return Hash(s.data(), s.size(), 0);
}
- uint32_t Shard(uint32_t hash) {
+ uint32_t Shard(uint32_t hash) const {
// Note, hash >> 32 yields hash in gcc, not the zero we expect!
return (num_shard_bits_ > 0) ? (hash >> (32 - num_shard_bits_)) : 0;
}

59
gnu/packages/storage.scm
View file

@ -39,6 +39,7 @@ (define-module (gnu packages storage)
#:use-module (gnu packages disk)
#:use-module (gnu packages gperf)
#:use-module (gnu packages jemalloc)
#:use-module (gnu packages icu4c)
#:use-module (gnu packages linux)
#:use-module (gnu packages lua)
#:use-module (gnu packages ncurses)
@ -46,9 +47,11 @@ (define-module (gnu packages storage)
#:use-module (gnu packages nss)
#:use-module (gnu packages openldap)
#:use-module (gnu packages pkg-config)
#:use-module (gnu packages pretty-print)
#:use-module (gnu packages python)
#:use-module (gnu packages python-xyz)
#:use-module (gnu packages sphinx)
#:use-module (gnu packages sqlite)
#:use-module (gnu packages tls)
#:use-module (gnu packages web)
#:use-module (gnu packages xml))
@ -56,18 +59,19 @@ (define-module (gnu packages storage)
(define-public ceph
(package
(name "ceph")
(version "14.2.16")
(version "16.2.6")
(source (origin
(method url-fetch)
(uri (string-append "https://download.ceph.com/tarballs/ceph-"
version ".tar.gz"))
(sha256
(base32
"0lmdri415hqczc9565s5m5568pnj97ipqxgnw6085kps0flwq5zh"))
"104xmc84d4ycdn7f4z09kvzwl1vlywxp3hbfxhgq0kcmgikb4wad"))
(patches
(search-patches
"ceph-disable-cpu-optimizations.patch"
"ceph-fix-snappy-breaking-change.patch"))
"ceph-boost-compat.patch"
"ceph-rocksdb-compat.patch"))
(modules '((guix build utils)))
(snippet
'(begin
@ -78,9 +82,8 @@ (define-public ceph
;"src/xxHash"
;"src/zstd"
;"src/civetweb"
;"src/seastar/fmt"
"src/test/downloads"
"src/c-ares"
"src/fmt"
"src/googletest"
"src/rapidjson"
"src/spdk"
@ -113,7 +116,7 @@ (define-public ceph
"-DBUILD_SHARED_LIBS=ON"
"-DWITH_SYSTEM_ROCKSDB=ON"
"-DWITH_SYSTEM_BOOST=ON"
"-DWITH_PYTHON3=ON"
;; TODO: Enable these when available in Guix.
"-DWITH_MGR_DASHBOARD_FRONTEND=OFF" ;requires node + nodeenv
"-DWITH_BABELTRACE=OFF"
@ -124,6 +127,9 @@ (define-public ceph
;; Use jemalloc instead of tcmalloc.
"-DALLOCATOR=jemalloc"
;; Don't install systemd unit files.
"-DWITH_SYSTEMD=OFF"
;; Do not bother building the tests; we are not currently running
;; them, and they do not build with system googletest as of 14.2.5.
"-DWITH_TESTS=OFF"))
@ -141,15 +147,12 @@ (define-public ceph
(substitute* "cmake/modules/Distutils.cmake"
;; Prevent creation of Python eggs.
(("setup.py install")
"setup.py install --single-version-externally-managed --root=/"))
(substitute* (find-files "src/pybind" "^setup\\.py$")
;; Here we inject an extra line to the `setup.py' of the
"setup.py install --single-version-externally-managed --root=/")
;; Inject the -rpath linker argument when linking
;; Python C libraries so RUNPATH gets set up correctly.
(("^([[:blank:]]+)extra_compile_args=(.*)$" _ indent args)
(string-append indent "extra_compile_args=" args
indent "extra_link_args=['-Wl,-rpath="
lib "/lib'],\n")))
(("LDFLAGS=(.*)\n" _ flags)
(string-append "LDFLAGS=\\\"" flags
" -Wl,-rpath=" lib "/lib\\\"\n")))
;; Statically link libcrc32 because it does not get installed,
;; yet several libraries end up referring to it.
@ -159,8 +162,7 @@ (define-public ceph
(substitute* "udev/50-rbd.rules"
(("/usr/bin/ceph-rbdnamer")
(string-append out "/bin/ceph-rbdnamer")))
#t)))
(string-append out "/bin/ceph-rbdnamer"))))))
(add-before 'install 'set-install-environment
(lambda* (#:key outputs #:allow-other-keys)
(let* ((out (assoc-ref outputs "out"))
@ -175,24 +177,22 @@ (define-public ceph
(add-after 'install 'wrap-python-scripts
(lambda* (#:key inputs outputs #:allow-other-keys)
(let* ((out (assoc-ref outputs "out"))
(scripts '("ceph" "ceph-mgr" "ceph-volume"))
(prettytable (assoc-ref inputs "python-prettytable"))
(six (assoc-ref inputs "python-six"))
(scripts '("bin/ceph" "bin/cephfs-top" "sbin/ceph-volume"))
(dependencies (map (lambda (input)
(assoc-ref inputs input))
'("python-prettytable" "python-pyyaml")))
(sitedir (lambda (package)
(string-append package
"/lib/python"
,(version-major+minor
(package-version python))
"/site-packages")))
(PYTHONPATH (string-append
(sitedir out) ":"
(sitedir six) ":"
(sitedir prettytable))))
(PYTHONPATH (string-join (map sitedir (cons out dependencies))
":")))
(for-each (lambda (executable)
(wrap-program (string-append out "/bin/" executable)
(wrap-program (string-append out "/" executable)
`("GUIX_PYTHONPATH" ":" prefix (,PYTHONPATH))))
scripts)
#t))))))
scripts)))))))
(outputs
'("out" "lib"))
(native-inputs
@ -207,7 +207,9 @@ (define-public ceph
("cryptsetup" ,cryptsetup)
("expat" ,expat)
("fcgi" ,fcgi)
("fmt" ,fmt)
("fuse" ,fuse)
("icu4c" ,icu4c)
("jemalloc" ,jemalloc)
("keyutils" ,keyutils)
("leveldb" ,leveldb)
@ -224,12 +226,13 @@ (define-public ceph
("ncurses" ,ncurses)
("nss" ,nss)
("python-prettytable" ,python-prettytable) ;used by ceph_daemon.py
("python-six" ,python-six) ;for ceph-mgr + plugins
("python" ,python-wrapper)
("python-pyyaml" ,python-pyyaml) ;from python-common/setup.py
("python" ,python)
("rapidjson" ,rapidjson)
("rdma-core" ,rdma-core)
("rocksdb" ,rocksdb)
("snappy" ,snappy)
("sqlite" ,sqlite)
("udev" ,eudev)
("util-linux" ,util-linux)
("util-linux:lib" ,util-linux "lib")