From f2a2e51863d59bbca110bd46e2d031d7fa6b19ce Mon Sep 17 00:00:00 2001 From: Lenar Fatikhov Date: Mon, 2 Oct 2023 08:39:33 -0700 Subject: [PATCH] RootRoute changes to accommodate distribution cases Summary: Distribution now can be set up for broadcast and cross-region directed deletes. This requires a DistributionRoute below the local routing tree branch to work. Reviewed By: stuclar Differential Revision: D49598728 fbshipit-source-id: a7786a9fa57a5b29e07666d84f27b1bddef89d2f --- mcrouter/McrouterFiberContext.h | 15 +- mcrouter/ProxyConfig-inl.h | 25 +- mcrouter/lib/test/RouteHandleTestUtil.h | 14 + mcrouter/routes/ProxyRoute-inl.h | 8 +- mcrouter/routes/ProxyRoute.h | 3 +- mcrouter/routes/RootRoute.h | 132 ++++++-- mcrouter/routes/test/RootRouteTest.cpp | 387 ++++++++++++++++++++++++ 7 files changed, 541 insertions(+), 43 deletions(-) create mode 100644 mcrouter/routes/test/RootRouteTest.cpp diff --git a/mcrouter/McrouterFiberContext.h b/mcrouter/McrouterFiberContext.h index d1dfe8cc1..fafd18e33 100644 --- a/mcrouter/McrouterFiberContext.h +++ b/mcrouter/McrouterFiberContext.h @@ -82,15 +82,16 @@ class fiber_local { std::bitset featureFlags; int32_t selectedIndex{-1}; uint32_t failoverCount{0}; - int64_t accumulatedBeforeReqInjectedLatencyUs{0}; - int64_t accumulatedAfterReqInjectedLatencyUs{0}; std::optional bucketId; + std::optional distributionTargetRegion; RequestClass requestClass; folly::StringPiece asynclogName; int64_t networkTransportTimeUs{0}; ServerLoad load{0}; std::vector extraDataCallbacks; std::shared_ptr axonCtx{nullptr}; + int64_t accumulatedBeforeReqInjectedLatencyUs{0}; + int64_t accumulatedAfterReqInjectedLatencyUs{0}; }; static auto makeGuardHelperBase(McrouterFiberContext&& tmp) { @@ -367,6 +368,16 @@ class fiber_local { static std::optional getBucketId() { return folly::fibers::local().bucketId; } + + static void setDistributionTargetRegion(std::string region) { + folly::fibers::local().distributionTargetRegion = + region; + } + + static std::optional getDistributionTargetRegion() { + return folly::fibers::local() + .distributionTargetRegion; + } }; } // namespace mcrouter diff --git a/mcrouter/ProxyConfig-inl.h b/mcrouter/ProxyConfig-inl.h index f49357547..2dafb3524 100644 --- a/mcrouter/ProxyConfig-inl.h +++ b/mcrouter/ProxyConfig-inl.h @@ -107,15 +107,28 @@ ProxyConfig::ProxyConfig( partialConfigs_ = provider.releasePartialConfigs(); } accessPoints_ = provider.releaseAccessPoints(); - bool disableBroadcastDeleteRpc = false; - if (auto* jDisableBroadcastDeleteRpc = - json.get_ptr("disable_broadcast_delete_rpc")) { - disableBroadcastDeleteRpc = - parseBool(*jDisableBroadcastDeleteRpc, "disable_broadcast_delete_rpc"); + bool enableCrossRegionDeleteRpc = true; + if (auto* jEnableCrossRegionDeleteRpc = + json.get_ptr("enable_cross_region_delete_rpc")) { + enableCrossRegionDeleteRpc = parseBool( + *jEnableCrossRegionDeleteRpc, "enable_cross_region_delete_rpc"); } + bool enableDeleteDistribution = false; + if (auto* jEnableDeleteDistribution = + json.get_ptr("enable_delete_distribution")) { + enableDeleteDistribution = + parseBool(*jEnableDeleteDistribution, "enable_delete_distribution"); + } + checkLogic( + enableDeleteDistribution || enableCrossRegionDeleteRpc, + "ProxyConfig: cannot disable cross-region delete rpc if distribution is disabled"); + proxyRoute_ = std::make_shared>( - proxy, routeSelectors, disableBroadcastDeleteRpc); + proxy, + routeSelectors, + enableDeleteDistribution, + enableCrossRegionDeleteRpc); serviceInfo_ = std::make_shared>(proxy, *this); } diff --git a/mcrouter/lib/test/RouteHandleTestUtil.h b/mcrouter/lib/test/RouteHandleTestUtil.h index f33925818..9b17c4a6e 100644 --- a/mcrouter/lib/test/RouteHandleTestUtil.h +++ b/mcrouter/lib/test/RouteHandleTestUtil.h @@ -155,6 +155,8 @@ struct TestHandleImpl { std::vector sawQueryTags; + std::vector distributionRegionInFiber; + bool isTko; bool isPaused; @@ -302,6 +304,17 @@ struct RecordingRoute { std::enable_if_t::value, void> recordBucketId( const Request&) const {} + template + void recordDistributionTargetRegion(const Request&) const { + if (mcrouter::fiber_local::getDistributionTargetRegion() + .has_value()) { + h_->distributionRegionInFiber.push_back( + mcrouter::fiber_local< + MemcacheRouterInfo>::getDistributionTargetRegion() + .value()); + } + } + template bool traverse(const Request& req, const RouteHandleTraverser&) const { @@ -351,6 +364,7 @@ struct RecordingRoute { h_->sawFlags.push_back(getFlagsIfExist(req)); h_->sawQueryTags.push_back(getQueryTagsIfExists(req)); recordBucketId(req); + recordDistributionTargetRegion(req); if (carbon::GetLike::value) { reply.result_ref() = h_->resultGenerator_.hasValue() ? (*h_->resultGenerator_)(req.key_ref()->fullKey().str()) diff --git a/mcrouter/routes/ProxyRoute-inl.h b/mcrouter/routes/ProxyRoute-inl.h index 92484eaaf..e49a3dc92 100644 --- a/mcrouter/routes/ProxyRoute-inl.h +++ b/mcrouter/routes/ProxyRoute-inl.h @@ -42,12 +42,14 @@ template ProxyRoute::ProxyRoute( Proxy& proxy, const RouteSelectorMap& routeSelectors, - bool disableBroadcastDeleteRpc) + bool enableDeleteDistribution, + bool enableCrossRegionDeleteRpc) : proxy_(proxy), - root_(makeRouteHandle( + root_(makeRouteHandleWithInfo( proxy_, routeSelectors, - disableBroadcastDeleteRpc)) { + enableDeleteDistribution, + enableCrossRegionDeleteRpc)) { if (proxy_.getRouterOptions().big_value_split_threshold != 0) { root_ = detail::wrapWithBigValueRoute( std::move(root_), proxy_.getRouterOptions()); diff --git a/mcrouter/routes/ProxyRoute.h b/mcrouter/routes/ProxyRoute.h index 46f09eb5e..2da9d111f 100644 --- a/mcrouter/routes/ProxyRoute.h +++ b/mcrouter/routes/ProxyRoute.h @@ -44,7 +44,8 @@ class ProxyRoute { Proxy& proxy, const RouteSelectorMap& routeSelectors, - bool disableBroadcastDeleteRpc = false); + bool enableDeleteDistribution = false, + bool enableCrossRegionDeleteRpc = true); template bool traverse( diff --git a/mcrouter/routes/RootRoute.h b/mcrouter/routes/RootRoute.h index a1f715784..06837497d 100644 --- a/mcrouter/routes/RootRoute.h +++ b/mcrouter/routes/RootRoute.h @@ -26,7 +26,7 @@ namespace facebook { namespace memcache { namespace mcrouter { -template +template class RootRoute { public: static std::string routeName() { @@ -35,20 +35,24 @@ class RootRoute { RootRoute( ProxyBase& proxy, - const RouteSelectorMap& routeSelectors, - bool disableBroadcastDeleteRpc = false) + const RouteSelectorMap& + routeSelectors, + bool enableDeleteDistribution = false, + bool enableCrossRegionDeleteRpc = true) : opts_(proxy.getRouterOptions()), rhMap_( routeSelectors, opts_.default_route, opts_.send_invalid_route_to_default, opts_.enable_route_policy_v2), - disableBroadcastDeleteRpc_(disableBroadcastDeleteRpc) {} + defaultRoute_(opts_.default_route), + enableDeleteDistribution_(enableDeleteDistribution), + enableCrossRegionDeleteRpc_(enableCrossRegionDeleteRpc) {} template bool traverse( const Request& req, - const RouteHandleTraverser& t) const { + const RouteHandleTraverser& t) const { const auto* rhPtr = rhMap_.getTargetsForKeyFast( req.key_ref()->routingPrefix(), req.key_ref()->routingKey()); if (FOLLY_LIKELY(rhPtr != nullptr)) { @@ -76,16 +80,7 @@ class RootRoute { run in the background. This is a good default for /star/star/ requests. */ - const auto* rhPtr = rhMap_.getTargetsForKeyFast( - req.key_ref()->routingPrefix(), req.key_ref()->routingKey()); - - auto reply = FOLLY_UNLIKELY(rhPtr == nullptr) - ? routeImpl( - rhMap_.getTargetsForKeySlow( - req.key_ref()->routingPrefix(), req.key_ref()->routingKey()), - req) - : routeImpl(*rhPtr, req); - + auto reply = getTargetsAndRoute(req.key_ref()->routingPrefix(), req); if (isErrorResult(*reply.result_ref()) && opts_.group_remote_errors) { reply = ReplyT(carbon::Result::REMOTE_ERROR); } @@ -93,14 +88,63 @@ class RootRoute { return reply; } + McDeleteReply route(const McDeleteRequest& req) const { + // If distribution is enabled, route deletes to the default route where + // DistributionRoute will route cross region. + // + // NOTE: if enableCrossRegionDeleteRpc flag is not set it defaults to true, + // the distribution step will be followed by a duplicating RPC step, and the + // return value will be the reply returned by the RPC step. + McDeleteReply reply; + if (enableDeleteDistribution_ && !req.key_ref()->routingPrefix().empty()) { + auto routingPrefix = RoutingPrefix(req.key_ref()->routingPrefix()); + if (routingPrefix.str() != defaultRoute_.str() && + req.key_ref()->routingPrefix() != kBroadcastPrefix) { + reply = fiber_local::runWithLocals( + [this, &req, &routingPrefix]() { + fiber_local::setDistributionTargetRegion( + routingPrefix.getRegion().str()); + return getTargetsAndRoute(defaultRoute_, req); + }); + + if (!enableCrossRegionDeleteRpc_) { + return reply; + } + } + } + reply = getTargetsAndRoute(req.key_ref()->routingPrefix(), req); + if (isErrorResult(*reply.result_ref()) && opts_.group_remote_errors) { + reply = McDeleteReply(carbon::Result::REMOTE_ERROR); + } + return reply; + } + private: const McrouterOptions& opts_; - RouteHandleMap rhMap_; - bool disableBroadcastDeleteRpc_; + RouteHandleMap rhMap_; + RoutingPrefix defaultRoute_; + bool enableDeleteDistribution_; + bool enableCrossRegionDeleteRpc_; + + template + FOLLY_ALWAYS_INLINE ReplyT getTargetsAndRoute( + folly::StringPiece routingPrefix, + const Request& req) const { + const auto* rhPtr = + rhMap_.getTargetsForKeyFast(routingPrefix, req.key_ref()->routingKey()); + + return UNLIKELY(rhPtr == nullptr) + ? routeImpl( + rhMap_.getTargetsForKeySlow( + routingPrefix, req.key_ref()->routingKey()), + req) + : routeImpl(*rhPtr, req); + } template ReplyT routeImpl( - const std::vector>& rh, + const std::vector>& + rh, const Request& req, carbon::GetLikeT = 0) const { auto reply = doRoute(rh, req); @@ -123,7 +167,8 @@ class RootRoute { template ReplyT routeImpl( - const std::vector>& rh, + const std::vector>& + rh, const Request& req, carbon::ArithmeticLikeT = 0) const { auto reply = opts_.allow_only_gets ? createReply(DefaultReply, req) @@ -137,7 +182,8 @@ class RootRoute { template ReplyT routeImpl( - const std::vector>& rh, + const std::vector>& + rh, const Request& req, carbon::OtherThanT, carbon::ArithmeticLike<>> = 0) const { @@ -150,29 +196,53 @@ class RootRoute { template ReplyT doRoute( - const std::vector>& rh, + const std::vector>& + rh, const Request& req) const { if (FOLLY_LIKELY(rh.size() == 1)) { return rh[0]->route(req); } if (!rh.empty()) { - // Broadcast delete via Distribution, route only - // to the first (local) route handle - if constexpr (folly::IsOneOf::value) { - if (disableBroadcastDeleteRpc_ && - req.key_ref()->routingPrefix() == kBroadcastPrefix) { - return rh[0]->route(req); - } - } + return routeToAll(rh, req); + } + return createReply(ErrorReply); + } - auto reqCopy = std::make_shared(req); + template + ReplyT routeToAll( + const std::vector>& + rh, + const Request& req) const { + auto reqCopy = std::make_shared(req); + for (size_t i = 1, e = rh.size(); i < e; ++i) { + auto r = rh[i]; + folly::fibers::addTask([r, reqCopy]() { r->route(*reqCopy); }); + } + return rh[0]->route(req); + } + + McDeleteReply routeToAll( + const std::vector>& + rh, + const McDeleteRequest& req) const { + if (enableCrossRegionDeleteRpc_) { + auto reqCopy = std::make_shared(req); for (size_t i = 1, e = rh.size(); i < e; ++i) { auto r = rh[i]; folly::fibers::addTask([r, reqCopy]() { r->route(*reqCopy); }); } + } + if (enableDeleteDistribution_ && + req.key_ref()->routingPrefix() == kBroadcastPrefix) { + return fiber_local::runWithLocals([&req, &rh]() { + // DistributionRoute will read empty string as "broadcast", i.e. + // distribute to all regions: + fiber_local::setDistributionTargetRegion(""); + return rh[0]->route(req); + }); + } else { return rh[0]->route(req); } - return createReply(ErrorReply); } }; } // namespace mcrouter diff --git a/mcrouter/routes/test/RootRouteTest.cpp b/mcrouter/routes/test/RootRouteTest.cpp new file mode 100644 index 000000000..5666f1b05 --- /dev/null +++ b/mcrouter/routes/test/RootRouteTest.cpp @@ -0,0 +1,387 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include + +#include +#include + +#include "mcrouter/McrouterFiberContext.h" +#include "mcrouter/lib/network/gen/MemcacheMessages.h" +#include "mcrouter/lib/network/gen/MemcacheRouterInfo.h" +#include "mcrouter/routes/test/RouteHandleTestUtil.h" + +namespace facebook::memcache::mcrouter { + +class RootRouteTest : public ::testing::Test { + using PrefixSelector = PrefixSelectorRoute; + + public: + RootRouteTest() { + srHandleVec1.push_back(std::make_shared( + GetRouteTestData(carbon::Result::FOUND, "a"))); + srHandleVec2.push_back(std::make_shared( + GetRouteTestData(carbon::Result::FOUND, "b"))); + srHandleVec3.push_back(std::make_shared( + GetRouteTestData(carbon::Result::FOUND, "c"))); + mockSrHandle1 = get_route_handles(srHandleVec1)[0]; + mockSrHandle2 = get_route_handles(srHandleVec2)[0]; + mockSrHandle3 = get_route_handles(srHandleVec3)[0]; + srHandleVecs.push_back(srHandleVec1[0]); + srHandleVecs.push_back(srHandleVec2[0]); + srHandleVecs.push_back(srHandleVec3[0]); + + PrefixSelector selector1; + selector1.wildcard = mockSrHandle1; + auto ptr1 = std::make_shared(selector1); + PrefixSelector selector2; + selector2.wildcard = mockSrHandle2; + auto ptr2 = std::make_shared(selector2); + PrefixSelector selector3; + selector3.wildcard = mockSrHandle3; + auto ptr3 = std::make_shared(selector3); + + routeSelectors[prefix1] = ptr1; + routeSelectors[prefix2] = ptr2; + routeSelectors[prefix3] = ptr3; + } + + auto& getRouteSelectors() { + return routeSelectors; + } + + std::vector> getMockSrHandles() { + return {mockSrHandle1, mockSrHandle2, mockSrHandle3}; + } + + std::shared_ptr& getTestHandle(size_t index) { + return srHandleVecs[index]; + } + + private: + std::string prefix1 = "/././"; + std::string prefix2 = "/virginia/c/"; + std::string prefix3 = "/georgia/d/"; + std::vector> srHandleVec1; + std::vector> srHandleVec2; + std::vector> srHandleVec3; + std::vector> srHandleVecs; + std::shared_ptr mockSrHandle1; + std::shared_ptr mockSrHandle2; + std::shared_ptr mockSrHandle3; + RouteSelectorMap routeSelectors; +}; + +TEST_F(RootRouteTest, NoRoutingPrefixGetRoutesToDefault) { + mockFiberContext(); + auto proxy = &fiber_local::getSharedCtx()->proxy(); + RootRoute rr{*proxy, getRouteSelectors()}; + auto reply = rr.route(McGetRequest("getReq")); + EXPECT_FALSE(getTestHandle(0)->saw_keys.empty()); + EXPECT_TRUE(getTestHandle(1)->saw_keys.empty()); + EXPECT_TRUE(getTestHandle(2)->saw_keys.empty()); + + EXPECT_EQ(getTestHandle(0)->saw_keys.size(), 1); + + EXPECT_EQ(getTestHandle(0)->saw_keys[0], "getReq"); + + EXPECT_TRUE(getTestHandle(0)->distributionRegionInFiber.empty()); + EXPECT_TRUE(getTestHandle(1)->distributionRegionInFiber.empty()); + EXPECT_TRUE(getTestHandle(2)->distributionRegionInFiber.empty()); +} + +TEST_F(RootRouteTest, BroadcastPrefixGetRoutesToAll) { + mockFiberContext(); + auto proxy = &fiber_local::getSharedCtx()->proxy(); + RootRoute rr{*proxy, getRouteSelectors()}; + TestFiberManager fm; + fm.runAll({[&]() { rr.route(McGetRequest("/*/*/getReq")); }}); + EXPECT_FALSE(getTestHandle(0)->saw_keys.empty()); + EXPECT_FALSE(getTestHandle(1)->saw_keys.empty()); + EXPECT_FALSE(getTestHandle(2)->saw_keys.empty()); + + EXPECT_EQ(getTestHandle(0)->saw_keys.size(), 1); + EXPECT_EQ(getTestHandle(1)->saw_keys.size(), 1); + EXPECT_EQ(getTestHandle(2)->saw_keys.size(), 1); + + EXPECT_EQ(getTestHandle(0)->saw_keys[0], "/*/*/getReq"); + EXPECT_EQ(getTestHandle(1)->saw_keys[0], "/*/*/getReq"); + EXPECT_EQ(getTestHandle(2)->saw_keys[0], "/*/*/getReq"); + + EXPECT_TRUE(getTestHandle(0)->distributionRegionInFiber.empty()); + EXPECT_TRUE(getTestHandle(1)->distributionRegionInFiber.empty()); + EXPECT_TRUE(getTestHandle(2)->distributionRegionInFiber.empty()); +} + +TEST_F(RootRouteTest, DirectedCrossRegionPrefixGetRoutesToOne) { + mockFiberContext(); + auto proxy = &fiber_local::getSharedCtx()->proxy(); + RootRoute rr{*proxy, getRouteSelectors()}; + TestFiberManager fm; + fm.runAll({[&]() { rr.route(McGetRequest("/virginia/c/getReq")); }}); + EXPECT_TRUE(getTestHandle(0)->saw_keys.empty()); + EXPECT_FALSE(getTestHandle(1)->saw_keys.empty()); + EXPECT_TRUE(getTestHandle(2)->saw_keys.empty()); + + EXPECT_EQ(getTestHandle(1)->saw_keys.size(), 1); + + EXPECT_EQ(getTestHandle(1)->saw_keys[0], "/virginia/c/getReq"); + + EXPECT_TRUE(getTestHandle(0)->distributionRegionInFiber.empty()); + EXPECT_TRUE(getTestHandle(1)->distributionRegionInFiber.empty()); + EXPECT_TRUE(getTestHandle(2)->distributionRegionInFiber.empty()); +} + +TEST_F(RootRouteTest, NoRoutingPrefixSetRoutesToDefault) { + mockFiberContext(); + auto proxy = &fiber_local::getSharedCtx()->proxy(); + RootRoute rr{*proxy, getRouteSelectors()}; + auto reply = rr.route(McSetRequest("setReq")); + EXPECT_FALSE(getTestHandle(0)->saw_keys.empty()); + EXPECT_TRUE(getTestHandle(1)->saw_keys.empty()); + EXPECT_TRUE(getTestHandle(2)->saw_keys.empty()); + + EXPECT_EQ(getTestHandle(0)->saw_keys.size(), 1); + + EXPECT_EQ(getTestHandle(0)->saw_keys[0], "setReq"); + + EXPECT_TRUE(getTestHandle(0)->distributionRegionInFiber.empty()); + EXPECT_TRUE(getTestHandle(1)->distributionRegionInFiber.empty()); + EXPECT_TRUE(getTestHandle(2)->distributionRegionInFiber.empty()); +} + +TEST_F(RootRouteTest, BroadcastPrefixSetRoutesToAll) { + mockFiberContext(); + auto proxy = &fiber_local::getSharedCtx()->proxy(); + RootRoute rr{*proxy, getRouteSelectors()}; + TestFiberManager fm; + fm.runAll({[&]() { rr.route(McSetRequest("/*/*/setReq")); }}); + EXPECT_FALSE(getTestHandle(0)->saw_keys.empty()); + EXPECT_FALSE(getTestHandle(1)->saw_keys.empty()); + EXPECT_FALSE(getTestHandle(2)->saw_keys.empty()); + + EXPECT_EQ(getTestHandle(0)->saw_keys.size(), 1); + EXPECT_EQ(getTestHandle(1)->saw_keys.size(), 1); + EXPECT_EQ(getTestHandle(2)->saw_keys.size(), 1); + + EXPECT_EQ(getTestHandle(0)->saw_keys[0], "/*/*/setReq"); + EXPECT_EQ(getTestHandle(1)->saw_keys[0], "/*/*/setReq"); + EXPECT_EQ(getTestHandle(2)->saw_keys[0], "/*/*/setReq"); + + EXPECT_TRUE(getTestHandle(0)->distributionRegionInFiber.empty()); + EXPECT_TRUE(getTestHandle(1)->distributionRegionInFiber.empty()); + EXPECT_TRUE(getTestHandle(2)->distributionRegionInFiber.empty()); +} + +TEST_F(RootRouteTest, DirectedCrossRegionPrefixSetRoutesToOne) { + mockFiberContext(); + auto proxy = &fiber_local::getSharedCtx()->proxy(); + RootRoute rr{*proxy, getRouteSelectors()}; + TestFiberManager fm; + fm.runAll({[&]() { rr.route(McSetRequest("/virginia/c/setReq")); }}); + EXPECT_TRUE(getTestHandle(0)->saw_keys.empty()); + EXPECT_FALSE(getTestHandle(1)->saw_keys.empty()); + EXPECT_TRUE(getTestHandle(2)->saw_keys.empty()); + + EXPECT_EQ(getTestHandle(1)->saw_keys.size(), 1); + + EXPECT_EQ(getTestHandle(1)->saw_keys[0], "/virginia/c/setReq"); + + EXPECT_TRUE(getTestHandle(0)->distributionRegionInFiber.empty()); + EXPECT_TRUE(getTestHandle(1)->distributionRegionInFiber.empty()); + EXPECT_TRUE(getTestHandle(2)->distributionRegionInFiber.empty()); +} + +TEST_F(RootRouteTest, NoRoutingPrefixDeleteRoutesToDefault) { + mockFiberContext(); + auto proxy = &fiber_local::getSharedCtx()->proxy(); + RootRoute rr{*proxy, getRouteSelectors()}; + auto reply = rr.route(McDeleteRequest("delReq")); + EXPECT_FALSE(getTestHandle(0)->saw_keys.empty()); + EXPECT_TRUE(getTestHandle(1)->saw_keys.empty()); + EXPECT_TRUE(getTestHandle(2)->saw_keys.empty()); + + EXPECT_EQ(getTestHandle(0)->saw_keys.size(), 1); + + EXPECT_EQ(getTestHandle(0)->saw_keys[0], "delReq"); + + EXPECT_TRUE(getTestHandle(0)->distributionRegionInFiber.empty()); + EXPECT_TRUE(getTestHandle(1)->distributionRegionInFiber.empty()); + EXPECT_TRUE(getTestHandle(2)->distributionRegionInFiber.empty()); +} + +TEST_F(RootRouteTest, BroadcastPrefixDeleteRoutesToAll) { + mockFiberContext(); + auto proxy = &fiber_local::getSharedCtx()->proxy(); + RootRoute rr{*proxy, getRouteSelectors()}; + TestFiberManager fm; + fm.runAll({[&]() { rr.route(McDeleteRequest("/*/*/delReq")); }}); + EXPECT_FALSE(getTestHandle(0)->saw_keys.empty()); + EXPECT_FALSE(getTestHandle(1)->saw_keys.empty()); + EXPECT_FALSE(getTestHandle(2)->saw_keys.empty()); + + EXPECT_EQ(getTestHandle(0)->saw_keys.size(), 1); + EXPECT_EQ(getTestHandle(1)->saw_keys.size(), 1); + EXPECT_EQ(getTestHandle(2)->saw_keys.size(), 1); + + EXPECT_EQ(getTestHandle(0)->saw_keys[0], "/*/*/delReq"); + EXPECT_EQ(getTestHandle(1)->saw_keys[0], "/*/*/delReq"); + EXPECT_EQ(getTestHandle(2)->saw_keys[0], "/*/*/delReq"); + + EXPECT_TRUE(getTestHandle(0)->distributionRegionInFiber.empty()); + EXPECT_TRUE(getTestHandle(1)->distributionRegionInFiber.empty()); + EXPECT_TRUE(getTestHandle(2)->distributionRegionInFiber.empty()); +} + +TEST_F(RootRouteTest, DirectedCrossRegionPrefixDeleteRoutesToOne) { + mockFiberContext(); + auto proxy = &fiber_local::getSharedCtx()->proxy(); + RootRoute rr{*proxy, getRouteSelectors()}; + TestFiberManager fm; + fm.runAll({[&]() { rr.route(McDeleteRequest("/virginia/c/delReq")); }}); + EXPECT_TRUE(getTestHandle(0)->saw_keys.empty()); + EXPECT_FALSE(getTestHandle(1)->saw_keys.empty()); + EXPECT_TRUE(getTestHandle(2)->saw_keys.empty()); + + EXPECT_EQ(getTestHandle(1)->saw_keys.size(), 1); + + EXPECT_EQ(getTestHandle(1)->saw_keys[0], "/virginia/c/delReq"); + + EXPECT_TRUE(getTestHandle(0)->distributionRegionInFiber.empty()); + EXPECT_TRUE(getTestHandle(1)->distributionRegionInFiber.empty()); + EXPECT_TRUE(getTestHandle(2)->distributionRegionInFiber.empty()); +} + +TEST_F(RootRouteTest, NoRoutingPrefixDeleteWithDistributionOnRoutesToDefault) { + mockFiberContext(); + auto proxy = &fiber_local::getSharedCtx()->proxy(); + RootRoute rr{ + *proxy, + getRouteSelectors(), + /*enableDeleteDistribution*/ true, + /*enableBroadcastDeleteRpc*/ true}; + auto reply = rr.route(McDeleteRequest("delReq")); + EXPECT_FALSE(getTestHandle(0)->saw_keys.empty()); + EXPECT_TRUE(getTestHandle(1)->saw_keys.empty()); + EXPECT_TRUE(getTestHandle(2)->saw_keys.empty()); + + EXPECT_EQ(getTestHandle(0)->saw_keys.size(), 1); + + EXPECT_EQ(getTestHandle(0)->saw_keys[0], "delReq"); + + EXPECT_TRUE(getTestHandle(0)->distributionRegionInFiber.empty()); + EXPECT_TRUE(getTestHandle(1)->distributionRegionInFiber.empty()); + EXPECT_TRUE(getTestHandle(2)->distributionRegionInFiber.empty()); +} + +TEST_F(RootRouteTest, BroadcastPrefixDeleteWithDistributionOnRoutesToAll) { + mockFiberContext(); + auto proxy = &fiber_local::getSharedCtx()->proxy(); + RootRoute rr{ + *proxy, + getRouteSelectors(), + /*enableDeleteDistribution*/ true, + /*enableBroadcastDeleteRpc*/ true}; + TestFiberManager fm; + fm.runAll({[&]() { rr.route(McDeleteRequest("/*/*/delReq")); }}); + EXPECT_FALSE(getTestHandle(0)->saw_keys.empty()); + EXPECT_FALSE(getTestHandle(1)->saw_keys.empty()); + EXPECT_FALSE(getTestHandle(2)->saw_keys.empty()); + + EXPECT_EQ(getTestHandle(0)->saw_keys.size(), 1); + EXPECT_EQ(getTestHandle(1)->saw_keys.size(), 1); + EXPECT_EQ(getTestHandle(2)->saw_keys.size(), 1); + + EXPECT_EQ(getTestHandle(0)->saw_keys[0], "/*/*/delReq"); + EXPECT_EQ(getTestHandle(1)->saw_keys[0], "/*/*/delReq"); + EXPECT_EQ(getTestHandle(2)->saw_keys[0], "/*/*/delReq"); + + EXPECT_EQ(getTestHandle(0)->distributionRegionInFiber.size(), 1); + EXPECT_EQ(getTestHandle(0)->distributionRegionInFiber[0], ""); + EXPECT_TRUE(getTestHandle(1)->distributionRegionInFiber.empty()); + EXPECT_TRUE(getTestHandle(2)->distributionRegionInFiber.empty()); +} + +TEST_F( + RootRouteTest, + DirectedCrossRegionPrefixDeleteWithDistributionOnRoutesToTwo) { + mockFiberContext(); + auto proxy = &fiber_local::getSharedCtx()->proxy(); + RootRoute rr{ + *proxy, + getRouteSelectors(), + /*enableDeleteDistribution*/ true, + /*enableBroadcastDeleteRpc*/ true}; + TestFiberManager fm; + fm.runAll({[&]() { rr.route(McDeleteRequest("/virginia/c/delReq")); }}); + EXPECT_FALSE(getTestHandle(0)->saw_keys.empty()); + EXPECT_FALSE(getTestHandle(1)->saw_keys.empty()); + EXPECT_TRUE(getTestHandle(2)->saw_keys.empty()); + + EXPECT_EQ(getTestHandle(0)->saw_keys.size(), 1); + EXPECT_EQ(getTestHandle(1)->saw_keys.size(), 1); + + EXPECT_EQ(getTestHandle(0)->saw_keys[0], "/virginia/c/delReq"); + EXPECT_EQ(getTestHandle(1)->saw_keys[0], "/virginia/c/delReq"); + EXPECT_EQ(getTestHandle(0)->distributionRegionInFiber.size(), 1); + EXPECT_EQ(getTestHandle(0)->distributionRegionInFiber[0], "virginia"); + EXPECT_TRUE(getTestHandle(1)->distributionRegionInFiber.empty()); + EXPECT_TRUE(getTestHandle(2)->distributionRegionInFiber.empty()); +} + +TEST_F( + RootRouteTest, + BroadcastPrefixDeleteWithDistributionOnRcpOffRoutesToLocal) { + mockFiberContext(); + auto proxy = &fiber_local::getSharedCtx()->proxy(); + RootRoute rr{ + *proxy, + getRouteSelectors(), + /*enableDeleteDistribution*/ true, + /*enableBroadcastDeleteRpc*/ false}; + TestFiberManager fm; + fm.runAll({[&]() { rr.route(McDeleteRequest("/*/*/delReq")); }}); + EXPECT_FALSE(getTestHandle(0)->saw_keys.empty()); + EXPECT_TRUE(getTestHandle(1)->saw_keys.empty()); + EXPECT_TRUE(getTestHandle(2)->saw_keys.empty()); + + EXPECT_EQ(getTestHandle(0)->saw_keys.size(), 1); + + EXPECT_EQ(getTestHandle(0)->saw_keys[0], "/*/*/delReq"); + EXPECT_EQ(getTestHandle(0)->distributionRegionInFiber.size(), 1); + EXPECT_EQ(getTestHandle(0)->distributionRegionInFiber[0], ""); + EXPECT_TRUE(getTestHandle(1)->distributionRegionInFiber.empty()); + EXPECT_TRUE(getTestHandle(2)->distributionRegionInFiber.empty()); +} + +TEST_F( + RootRouteTest, + DirectedCrossRegionPrefixDeleteWithDistributionOnRpcOffRoutesToLocal) { + mockFiberContext(); + auto proxy = &fiber_local::getSharedCtx()->proxy(); + RootRoute rr{ + *proxy, + getRouteSelectors(), + /*enableDeleteDistribution*/ true, + /*enableBroadcastDeleteRpc*/ false}; + TestFiberManager fm; + fm.runAll({[&]() { rr.route(McDeleteRequest("/virginia/c/delReq")); }}); + EXPECT_FALSE(getTestHandle(0)->saw_keys.empty()); + EXPECT_TRUE(getTestHandle(1)->saw_keys.empty()); + EXPECT_TRUE(getTestHandle(2)->saw_keys.empty()); + + EXPECT_EQ(getTestHandle(0)->saw_keys.size(), 1); + + EXPECT_EQ(getTestHandle(0)->saw_keys[0], "/virginia/c/delReq"); + + EXPECT_EQ(getTestHandle(0)->distributionRegionInFiber.size(), 1); + EXPECT_EQ(getTestHandle(0)->distributionRegionInFiber[0], "virginia"); + EXPECT_TRUE(getTestHandle(1)->distributionRegionInFiber.empty()); + EXPECT_TRUE(getTestHandle(2)->distributionRegionInFiber.empty()); +} + +} // namespace facebook::memcache::mcrouter