Enable the CacheLibWrapper class as a RocksDB Plugin

cachelib/adaptor/rocks_secondary_cache/CMakeLists.txt

@@ -0,0 +1,13 @@
+cmake_minimum_required(VERSION 3.4)
+
+find_package(cachelib REQUIRED)
+
+# Suppresses errors from folly exceptions
+set(cachelib_SOURCES CachelibWrapper.cpp PARENT_SCOPE)
+set(cachelib_INCLUDE_PATHS ${CACHELIB_INCLUDE_DIR}/.. PARENT_SCOPE)
+set(cachelib_COMPILE_FLAGS -Wno-error=class-memaccess -Wno-error=sign-compare -Wno-error=deprecated-declarationsPARENT_SCOPE)
+#set(cachelib_LIBS "cachelib_allocator" "cachelib_shm" "cachelib_navy" "cachelib_common" "thriftcpp2" "thriftfrozen2" "thriftmetadata" "thriftanyrep" "thrifttype" "thrifttyperep" "thriftannotation" "thriftprotocol" "async" "wangle" "fizz" "sodium" "rt" PARENT_SCOPE)
+set(cachelib_LIBS "cachelib_allocator" "cachelib_shm" "cachelib_navy" "cachelib_common" "thriftcpp2" "thriftfrozen2" "thriftmetadata" "thriftanyrep" "thrifttype" "thrifttyperep" "thriftannotation" "thriftprotocol" "async" "wangle" "fizz" "folly" "folly_test_util" "fmtd" "glogd" "sodium" "boost_context" "rt" PARENT_SCOPE)
+set(cachelib_FUNC register_CachelibObjects PARENT_SCOPE)
+set(cachelib_LINK_PATHS ${CMAKE_PREFIX_PATH}/lib PARENT_SCOPE)
+set(cachelib_TESTS tests/CachelibWrapperTest.cpp PARENT_SCOPE)

cachelib/adaptor/rocks_secondary_cache/CachelibWrapper.cpp

@@ -14,18 +14,19 @@
  * limitations under the License.
  */
 
-#include "cachelib/adaptor/rocks_secondary_cache/CachelibWrapper.h"
+#include "CachelibWrapper.h"
 
-#include "cachelib/facebook/utils/FbInternalRuntimeUpdateWrapper.h"
 #include "folly/init/Init.h"
 #include "folly/synchronization/Rcu.h"
+#include "rocksdb/convenience.h"
 #include "rocksdb/version.h"
+#include "rocksdb/utilities/object_registry.h"
+#include "rocksdb/utilities/options_type.h"
 
 namespace facebook {
 namespace rocks_secondary_cache {
 
 #define FB_CACHE_MAX_ITEM_SIZE 4 << 20
-using ApiWrapper = cachelib::FbInternalRuntimeUpdateWrapper<FbCache>;
 
 namespace {
 // We use a separate RCU domain since read side critical sections can block
@@ -129,8 +130,68 @@ class RocksCachelibWrapperHandle : public rocksdb::SecondaryCacheResultHandle {
     }
   }
 };
+
+static std::unordered_map<std::string, ROCKSDB_NAMESPACE::OptionTypeInfo>
+rocks_cachelib_type_info = {
+#ifndef ROCKSDB_LITE
+  {"cachename", {offsetof(struct RocksCachelibOptions, cacheName), ROCKSDB_NAMESPACE::OptionType::kString}},
+  {"filename",  {offsetof(struct RocksCachelibOptions, fileName), ROCKSDB_NAMESPACE::OptionType::kString}},
+  {"size",       {offsetof(struct RocksCachelibOptions, size), ROCKSDB_NAMESPACE::OptionType::kSizeT}},
+  {"block_size", {offsetof(struct RocksCachelibOptions, blockSize), ROCKSDB_NAMESPACE::OptionType::kSizeT}},
+  {"region_size",{offsetof(struct RocksCachelibOptions, regionSize), ROCKSDB_NAMESPACE::OptionType::kSizeT}},
+  {"policy",     {offsetof(struct RocksCachelibOptions, admPolicy), ROCKSDB_NAMESPACE::OptionType::kString}},
+  {"probability",{offsetof(struct RocksCachelibOptions, admProbability), ROCKSDB_NAMESPACE::OptionType::kDouble}},
+  {"max_write_rate",{offsetof(struct RocksCachelibOptions, maxWriteRate), ROCKSDB_NAMESPACE::OptionType::kUInt64T}},
+  {"admission_write_rate",{offsetof(struct RocksCachelibOptions, admissionWriteRate), ROCKSDB_NAMESPACE::OptionType::kUInt64T}},
+  {"volatile_size",{offsetof(struct RocksCachelibOptions, volatileSize), ROCKSDB_NAMESPACE::OptionType::kSizeT}},
+  {"bucket_power", {offsetof(struct RocksCachelibOptions, bktPower), ROCKSDB_NAMESPACE::OptionType::kUInt32T}},
+  {"lock_power",   {offsetof(struct RocksCachelibOptions, lockPower), ROCKSDB_NAMESPACE::OptionType::kUInt32T}},
+#endif // ROCKSDB_LITE
+};
 } // namespace
 
+RocksCachelibWrapper::RocksCachelibWrapper(const RocksCachelibOptions& options)
+  : options_(options), cache_(nullptr) {
+  RegisterOptions(&options_, &rocks_cachelib_type_info);
+}
+
+ROCKSDB_NAMESPACE::Status RocksCachelibWrapper::PrepareOptions(const ROCKSDB_NAMESPACE::ConfigOptions& opts) {
+  FbCache* cache = cache_.load();
+
+  if (!cache) {
+    cachelib::PoolId defaultPool;
+    FbCacheConfig config;
+    NvmCacheConfig nvmConfig;
+
+    nvmConfig.navyConfig.setBlockSize(options_.blockSize);
+    nvmConfig.navyConfig.setSimpleFile(options_.fileName,
+				       options_.size,
+				       /*truncateFile=*/true);
+    nvmConfig.navyConfig.blockCache().setRegionSize(options_.regionSize);
+    if (options_.admPolicy == "random") {
+      nvmConfig.navyConfig.enableRandomAdmPolicy().setAdmProbability(
+       options_.admProbability);
+    } else {
+      nvmConfig.navyConfig.enableDynamicRandomAdmPolicy()
+          .setMaxWriteRate(options_.maxWriteRate)
+          .setAdmWriteRate(options_.admissionWriteRate);
+    }
+    nvmConfig.enableFastNegativeLookups = true;
+
+    config.setCacheSize(options_.volatileSize)
+      .setCacheName(options_.cacheName)
+      .setAccessConfig(
+          {options_.bktPower /* bucket power */, options_.lockPower /* lock power */})
+      .enableNvmCache(nvmConfig)
+      .validate(); // will throw if bad config
+    auto new_cache = std::make_unique<FbCache>(config);
+    pool_ =
+      new_cache->addPool("default", new_cache->getCacheMemoryStats().cacheSize);
+    cache_.store(new_cache.release());
+  }
+  return SecondaryCache::PrepareOptions(opts);
+}
+
 RocksCachelibWrapper::~RocksCachelibWrapper() { Close(); }
 
 rocksdb::Status RocksCachelibWrapper::Insert(
@@ -225,64 +286,31 @@ void RocksCachelibWrapper::Close() {
     // sections already started to finish, and then delete the cache
     cache_.store(nullptr);
     GetRcuDomain().synchronize();
-    admin_.reset();
     delete cache;
   }
 }
 
-bool RocksCachelibWrapper::UpdateMaxWriteRateForDynamicRandom(
-    uint64_t maxRate) {
-  FbCache* cache = cache_.load();
-  bool ret = false;
-  if (cache) {
-    ret = ApiWrapper::updateMaxRateForDynamicRandomAP(*cache, maxRate);
-  }
-  return ret;
-}
-
 // Global cache object and a default cache pool
 std::unique_ptr<rocksdb::SecondaryCache> NewRocksCachelibWrapper(
     const RocksCachelibOptions& opts) {
-  std::unique_ptr<FbCache> cache;
-  std::unique_ptr<cachelib::CacheAdmin> admin;
-  cachelib::PoolId defaultPool;
-  FbCacheConfig config;
-  NvmCacheConfig nvmConfig;
-
-  nvmConfig.navyConfig.setBlockSize(opts.blockSize);
-  nvmConfig.navyConfig.setSimpleFile(opts.fileName,
-                                     opts.size,
-                                     /*truncateFile=*/true);
-  nvmConfig.navyConfig.blockCache().setRegionSize(opts.regionSize);
-  if (opts.admPolicy == "random") {
-    nvmConfig.navyConfig.enableRandomAdmPolicy().setAdmProbability(
-        opts.admProbability);
-  } else {
-    nvmConfig.navyConfig.enableDynamicRandomAdmPolicy()
-        .setMaxWriteRate(opts.maxWriteRate)
-        .setAdmWriteRate(opts.admissionWriteRate);
-  }
-  nvmConfig.enableFastNegativeLookups = true;
-
-  config.setCacheSize(opts.volatileSize)
-      .setCacheName(opts.cacheName)
-      .setAccessConfig(
-          {opts.bktPower /* bucket power */, opts.lockPower /* lock power */})
-      .enableNvmCache(nvmConfig)
-      .validate(); // will throw if bad config
-  cache = std::make_unique<FbCache>(config);
-  defaultPool =
-      cache->addPool("default", cache->getCacheMemoryStats().cacheSize);
-
-  if (opts.fb303Stats) {
-    cachelib::CacheAdmin::Config adminConfig;
-    adminConfig.oncall = opts.oncallName;
-    admin = std::make_unique<cachelib::CacheAdmin>(*cache, adminConfig);
-  }
-
-  return std::unique_ptr<rocksdb::SecondaryCache>(new RocksCachelibWrapper(
-      std::move(cache), std::move(admin), std::move(defaultPool)));
+  std::unique_ptr<rocksdb::SecondaryCache> secondary = std::make_unique<RocksCachelibWrapper>(opts);
+  assert(secondary->PrepareOptions(ROCKSDB_NAMESPACE::ConfigOptions()).ok());
+  return secondary;
 }
 
+#ifndef ROCKSDB_LITE
+int register_CachelibObjects(ROCKSDB_NAMESPACE::ObjectLibrary& library, const std::string&) {
+  library.AddFactory<ROCKSDB_NAMESPACE::SecondaryCache>(RocksCachelibWrapper::kClassName(), 
+      [](const std::string& uri, std::unique_ptr<ROCKSDB_NAMESPACE::SecondaryCache>* guard,
+         std::string* /*errmsg*/) {
+	RocksCachelibOptions options;
+	guard->reset(new RocksCachelibWrapper(options));
+        return guard->get();
+      });
+  return 1;
+}
+#endif // ROCKSDB_LITE
 } // namespace rocks_secondary_cache
 } // namespace facebook
+
+

cachelib/adaptor/rocks_secondary_cache/CachelibWrapper.h

@@ -16,15 +16,20 @@
 
 #pragma once
 #include "cachelib/allocator/CacheAllocator.h"
-#include "cachelib/facebook/admin/CacheAdmin.h"
 #include "rocksdb/secondary_cache.h"
 #include "rocksdb/types.h"
 #include "rocksdb/version.h"
 
+namespace ROCKSDB_NAMESPACE {
+class ObjectLibrary;
+} // namespace ROCKSDB_NAMESPACE
+
 namespace facebook {
 namespace rocks_secondary_cache {
 // Options structure for configuring a Cachelib SecondaryCache instance
 struct RocksCachelibOptions {
+  static const char *kName() { return "RocksCachelibOptions"; }
+
   // A name for the use case
   std::string cacheName;
 
@@ -82,37 +87,33 @@ using FbCacheReadHandle = typename FbCache::ReadHandle;
 using FbCacheItem = typename FbCache::Item;
 
 // The RocksCachelibWrapper is a concrete implementation of
-// rocksdb::SecondaryCache. It can be allocated using
+// ROCKSDB_NAMESPACE::SecondaryCache. It can be allocated using
 // NewRocksCachelibWrapper() and the resulting pointer
-// can be passed in rocksdb::LRUCacheOptions to
-// rocksdb::NewLRUCache().
+// can be passed in ROCKSDB_NAMESPACE::LRUCacheOptions to
+// ROCKSDB_NAMESPACE::NewLRUCache().
 //
 // Users can also cast a pointer to it and call methods on
 // it directly, especially custom methods that may be added
 // in the future.  For example -
-// std::unique_ptr<rocksdb::SecondaryCache> cache =
+// std::unique_ptr<ROCKSDB_NAMESPACE::SecondaryCache> cache =
 //      NewRocksCachelibWrapper(opts);
 // static_cast<RocksCachelibWrapper*>(cache.get())->Erase(key);
-class RocksCachelibWrapper : public rocksdb::SecondaryCache {
+class RocksCachelibWrapper : public ROCKSDB_NAMESPACE::SecondaryCache {
  public:
-  RocksCachelibWrapper(std::unique_ptr<FbCache>&& cache,
-                       std::unique_ptr<cachelib::CacheAdmin>&& admin,
-                       cachelib::PoolId pool)
-      : cache_(std::move(cache).release()),
-        admin_(std::move(admin)),
-        pool_(pool) {}
+  RocksCachelibWrapper(const RocksCachelibOptions& options);
   ~RocksCachelibWrapper() override;
 
-  const char* Name() const override { return "RocksCachelibWrapper"; }
+  static const char* kClassName() { return "RocksCachelibWrapper"; }
+  const char* Name() const override { return kClassName(); }
 
-  rocksdb::Status Insert(
-      const rocksdb::Slice& key,
+  ROCKSDB_NAMESPACE::Status Insert(
+      const ROCKSDB_NAMESPACE::Slice& key,
       void* value,
-      const rocksdb::Cache::CacheItemHelper* helper) override;
+      const ROCKSDB_NAMESPACE::Cache::CacheItemHelper* helper) override;
 
-  std::unique_ptr<rocksdb::SecondaryCacheResultHandle> Lookup(
-      const rocksdb::Slice& key,
-      const rocksdb::Cache::CreateCallback& create_cb,
+  std::unique_ptr<ROCKSDB_NAMESPACE::SecondaryCacheResultHandle> Lookup(
+      const ROCKSDB_NAMESPACE::Slice& key,
+      const ROCKSDB_NAMESPACE::Cache::CreateCallback& create_cb,
       bool wait
 #if ROCKSDB_MAJOR > 7 || (ROCKSDB_MAJOR == 7 && ROCKSDB_MINOR >= 7)
       ,
@@ -125,34 +126,36 @@ class RocksCachelibWrapper : public rocksdb::SecondaryCache {
   bool SupportForceErase() const override { return false; }
 #endif
 
-  void Erase(const rocksdb::Slice& key) override;
+  void Erase(const ROCKSDB_NAMESPACE::Slice& key) override;
 
   void WaitAll(
-      std::vector<rocksdb::SecondaryCacheResultHandle*> handles) override;
+      std::vector<ROCKSDB_NAMESPACE::SecondaryCacheResultHandle*> handles) override;
 
   // TODO
   std::string GetPrintableOptions() const override { return ""; }
-
+  ROCKSDB_NAMESPACE::Status PrepareOptions(const ROCKSDB_NAMESPACE::ConfigOptions& /*options*/) override;
+
   // Calling Close() persists the cachelib state to the file and frees the
   // cachelib object. After calling Close(), subsequent lookups will fail,
   // and subsequent inserts will be silently ignored. Close() is not thread
   // safe, i.e only one thread can call it at a time. It doesn't require
   // ongoing lookups and inserts by other threads to be quiesced.
   void Close();
 
-  // If the admPolicy in RocksCachelibOptions was set to "dynamic_random",
-  // then this function can be called to update the max write rate for that
-  // policy.
-  bool UpdateMaxWriteRateForDynamicRandom(uint64_t maxRate);
-
  private:
+  RocksCachelibOptions options_;
   std::atomic<FbCache*> cache_;
-  std::unique_ptr<cachelib::CacheAdmin> admin_;
   cachelib::PoolId pool_;
 };
 
-// Allocate a new Cache instance with a rocksdb::TieredCache wrapper around it
-extern std::unique_ptr<rocksdb::SecondaryCache> NewRocksCachelibWrapper(
+// Allocate a new Cache instance with a ROCKSDB_NAMESPACE::TieredCache wrapper around it
+extern std::unique_ptr<ROCKSDB_NAMESPACE::SecondaryCache> NewRocksCachelibWrapper(
     const RocksCachelibOptions& opts);
+#ifndef ROCKSDB_LITE
+extern "C" {
+int register_CachelibObjects(ROCKSDB_NAMESPACE::ObjectLibrary& library, const std::string&);
+} // extern "C"
+
+#endif // ROCKSDB_LITE
 } // namespace rocks_secondary_cache
 } // namespace facebook

cachelib/adaptor/rocks_secondary_cache/README.md

@@ -0,0 +1,44 @@
+# RocksDB CacheLib Secondary Cache
+
+This directory contains files and tests for building Cachelib as a SecondaryCache for RocksDB.
+
+# Build
+
+Currently the build is only supported via cmake.
+
+First, build cachelib through the normal build procedure.
+
+To build under RocksDB, link this directory adaptor/rocksdb_secondary_cache) to the plugins/cachelib directory under RocksDB:
+```
+$ln -s .../secondary_cache .../plugins/cachelib
+```
+This will allow RocksDB to find and build the CacheLib plugin code.
+
+Next, under the RocksDB build directory, instruct RocksDB to build and include the cachelib plugin:
+```
+$cmake -DROCKSDB_PLUGINS=cachelib -DCMAKE_PREFIX_PATH=<Path to CacheLib> -DCMAKE_MODULE_PATH="<Path to CacheLib>"  ..
+```
+where  the prefix path points to the opt/cachelib directory and module path points to the cachelib/cmake directory.
+
+Finally, build RocksDB using "make".
+
+# Using the RocksDB Cachelib Secondary Cache
+
+The Secondary Cache can be created via either the SecondaryCache::CreateFromString or NewRocksCachelibWrapper APIs.
+```
+SecondaryCache::CreateFromString(..., "id=RocksCachelibWrapper; ...");
+
+RoksCachelibOptions options;
+NewRocksCachelibWrapper(options);
+```
+
+When using CreateFromString API, the options can be specified as name-value pairs on the command line. The mapping between the field names and options is found in CacheLibWrapper.cpp.
+
+# Issues and TODO
+
+* There are incompatibilities between RocksDB and Cachelib on the version of GTEST required and supported.  In order to successfully build, you must use the GTEST version in Cachelib and remove the gtest from the RocksDB build.  This requirement is true whether or not you are building tests.  In the RocksDB build, you must remove GTEST from the "include_directories" and "add_subdirectories" and add " -Wno-error=deprecated-declarations" to the CMAKE_CXX_FLAGS.  You may also need to add "-Wno-error=sign-compare" to build the RocksDB tests, depending on your compiler.
+
+* RocksDB does not currently enable plugins to build their own tests (see PR11052).  When the CachelibWrapperTest is added, additional link libraries may be required (See the CMakeLists.txt file).  No investigation as to the reason has been undertaken.
+
+
+