From 5161060b5f471b87c84130647c14d416625f4b41 Mon Sep 17 00:00:00 2001
From: Radoslaw Zarzynski <rzarzyns@redhat.com>
Date: Tue, 1 Jan 2019 23:53:45 +0100
Subject: [PATCH] fixup: MonClient can live in sharded environment.

Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
---
 src/crimson/mon/MonClient.cc | 12 +++++++++---
 src/crimson/mon/MonClient.h  | 12 ++++++++----
 src/crimson/net/Fwd.h        |  3 ++-
 3 files changed, 19 insertions(+), 8 deletions(-)

diff --git a/src/crimson/mon/MonClient.cc b/src/crimson/mon/MonClient.cc
index 9d492593e46d4f..3a1364f06a551c 100644
--- a/src/crimson/mon/MonClient.cc
+++ b/src/crimson/mon/MonClient.cc
@@ -257,7 +257,8 @@ AuthMethodList create_auth_methods(uint32_t entity_type)
 
 Client::Client(const EntityName& name,
                ceph::net::Messenger& messenger)
-  : entity_name{name},
+  : ForeignDispatcher(seastar::engine().cpu_id()),
+    entity_name{name},
     auth_methods{create_auth_methods(entity_name.get_type())},
     want_keys{CEPH_ENTITY_TYPE_MON |
               CEPH_ENTITY_TYPE_OSD |
@@ -296,9 +297,11 @@ bool Client::is_hunting() const {
 }
 
 seastar::future<>
-Client::ms_dispatch(ceph::net::ConnectionRef conn, MessageRef m)
+Client::fms_dispatch(ceph::net::ConnectionFRef conn, Client::MessageFRef m)
 {
   logger().info("ms_dispatch {}", *m);
+#if 0
+  // TODO: need move to MessageFRef or MessengerXRef.
   // we only care about these message types
   switch (m->get_type()) {
   case CEPH_MSG_MON_MAP:
@@ -324,9 +327,12 @@ Client::ms_dispatch(ceph::net::ConnectionRef conn, MessageRef m)
   default:
     return seastar::now();
   }
+#else
+    return seastar::now();
+#endif
 }
 
-seastar::future<> Client::ms_handle_reset(ceph::net::ConnectionRef conn)
+seastar::future<> Client::fms_handle_reset(ceph::net::ConnectionFRef conn)
 {
   auto found = std::find_if(pending_conns.begin(), pending_conns.end(),
                             [peer_addr = conn->get_peer_addr()](auto& mc) {
diff --git a/src/crimson/mon/MonClient.h b/src/crimson/mon/MonClient.h
index cc291f7510ad9c..92824edf71d3aa 100644
--- a/src/crimson/mon/MonClient.h
+++ b/src/crimson/mon/MonClient.h
@@ -36,7 +36,11 @@ namespace ceph::mon {
 
 class Connection;
 
-class Client : public ceph::net::Dispatcher {
+// Suppose we don't want to shard MonClient to save resources -- one
+// instance is fine. Let's see how much effort is necessary to interact
+// with sharded world.
+class Client : public ceph::net::ForeignDispatcher<Client> {
+  friend ceph::net::ForeignDispatcher<Client>;
   const EntityName entity_name;
   KeyRing keyring;
   AuthMethodList auth_methods;
@@ -80,9 +84,9 @@ class Client : public ceph::net::Dispatcher {
 private:
   void tick();
 
-  seastar::future<> ms_dispatch(ceph::net::ConnectionRef conn,
-				MessageRef m) override;
-  seastar::future<> ms_handle_reset(ceph::net::ConnectionRef conn) override;
+  seastar::future<> fms_dispatch(ceph::net::ConnectionFRef conn,
+				 MessageFRef m) override;
+  seastar::future<> fms_handle_reset(ceph::net::ConnectionFRef conn) override;
 
   seastar::future<> handle_monmap(ceph::net::ConnectionRef conn,
 				  Ref<MMonMap> m);
diff --git a/src/crimson/net/Fwd.h b/src/crimson/net/Fwd.h
index 331c69d7a3af6b..17ffd082c880c1 100644
--- a/src/crimson/net/Fwd.h
+++ b/src/crimson/net/Fwd.h
@@ -29,9 +29,10 @@ using msgr_tag_t = uint8_t;
 
 class Connection;
 using ConnectionRef = seastar::shared_ptr<Connection>;
+using ConnectionFRef = seastar::foreign_ptr<ConnectionRef>;
 // NOTE: ConnectionXRef should only be used in seastar world, because
 // lw_shared_ptr<> is not safe to be accessed by unpinned alien threads.
-using ConnectionXRef = seastar::lw_shared_ptr<seastar::foreign_ptr<ConnectionRef>>;
+using ConnectionXRef = seastar::lw_shared_ptr<ConnectionFRef>;
 
 class Dispatcher;