Merge pull request #176 from xaya/rest

Add REST API and use it for bootstrap data

Makefile.am

@@ -1,3 +1,3 @@
 ACLOCAL_AMFLAGS = ${ACLOCAL_FLAGS} -Im4
 
-SUBDIRS = hexagonal proto database mapdata src gametest
+SUBDIRS = data hexagonal proto database mapdata src gametest

configure.ac

@@ -47,6 +47,7 @@ PKG_CHECK_MODULES([XAYAGAME], [libxayautil libxayagame])
 PKG_CHECK_MODULES([CHARON], [charon])
 PKG_CHECK_MODULES([SQLITE3], [sqlite3])
 PKG_CHECK_MODULES([JSON], [jsoncpp])
+PKG_CHECK_MODULES([MHD], [libmicrohttpd])
 PKG_CHECK_MODULES([GLOG], [libglog])
 PKG_CHECK_MODULES([GFLAGS], [gflags])
 PKG_CHECK_MODULES([GTEST], [gmock gtest])
@@ -68,6 +69,7 @@ AC_SUBST(GMP_LIBS, -lgmp)
 
 AC_CONFIG_FILES([
   Makefile \
+  data/Makefile \
   database/Makefile \
   gametest/Makefile \
   hexagonal/Makefile \

data/Makefile.am

@@ -0,0 +1 @@
+EXTRA_DIST = letsencrypt.pem

data/letsencrypt.pem

@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw
+TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
+cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4
+WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu
+ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY
+MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc
+h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+
+0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U
+A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW
+T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH
+B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC
+B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv
+KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn
+OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn
+jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw
+qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI
+rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV
+HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq
+hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL
+ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ
+3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK
+NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5
+ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur
+TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC
+jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc
+oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq
+4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA
+mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d
+emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=
+-----END CERTIFICATE-----

gametest/charon.py

@@ -46,6 +46,11 @@
 PUBSUB = "pubsub.chat.xaya.io"
 
 
+# Port and URL for the local REST API used for bootstrap data.
+REST_PORT = 18_042
+REST_URL = "http://localhost:%d" % REST_PORT
+
+
 def testAccountJid (acc):
   return "%s@%s" % (acc[0], XMPP_SERVER)
 
@@ -79,6 +84,7 @@ def __enter__ (self):
     args = [self.binary]
     args.extend (["--datadir", self.datadir])
     args.append ("--game_rpc_port=%d" % self.rpcport)
+    args.extend (["--rest_endpoint", REST_URL])
     args.extend (["--charon", "client"])
     args.extend (["--charon_server_jid", testAccountJid (TEST_ACCOUNTS[0])])
     args.extend (["--charon_client_jid", testAccountJid (TEST_ACCOUNTS[1])])
@@ -142,6 +148,7 @@ def run (self):
     args.extend (["--charon_pubsub_service", PUBSUB])
     args.extend (["--charon_server_jid", testAccountJid (TEST_ACCOUNTS[0])])
     args.extend (["--charon_password", TEST_ACCOUNTS[0][1]])
+    args.extend (["--rest_port", str (REST_PORT)])
     self.startGameDaemon (extraArgs=args)
 
     self.mainLogger.info ("Starting tauriond as Charon client...")
@@ -163,6 +170,10 @@ def run (self):
       del res["server"]
       self.assertEqual (res, srv)
 
+      self.mainLogger.info ("Testing bootstrap data via REST...")
+      res = client.rpc.getbootstrapdata ()
+      self.assertEqual (res, self.rpc.game.getbootstrapdata ())
+
       self.mainLogger.info ("Testing invalid Charon method call...")
       self.expectError (-32602, ".*Invalid method parameters.*",
                         client.rpc.getregions, 42)

src/Makefile.am

@@ -74,22 +74,24 @@ libtaurionheaders = \
 tauriond_CXXFLAGS = \
   -I$(top_srcdir) \
   $(XAYAGAME_CFLAGS) $(CHARON_CFLAGS) \
-  $(JSON_CFLAGS) \
+  $(JSON_CFLAGS) $(MHD_CLFGAS) \
   $(GLOG_CFLAGS) $(GFLAGS_CFLAGS) $(PROTOBUF_CFLAGS)
 tauriond_LDADD = \
   $(builddir)/libtaurion.la \
   $(top_builddir)/mapdata/libmapdata.la \
   $(top_builddir)/database/libdatabase.la \
   $(XAYAGAME_LIBS) $(CHARON_LIBS) \
-  $(JSON_LIBS) \
+  $(JSON_LIBS) $(MHD_LIBS) \
   $(GLOG_LIBS) $(GFLAGS_LIBS) $(PROTOBUF_LIBS)
 tauriond_SOURCES = main.cpp \
   charon.cpp \
   pxrpcserver.cpp \
+  rest.cpp \
   version.cpp
 tauriondheaders = \
   charon.hpp \
   pxrpcserver.hpp \
+  rest.hpp \
   version.hpp \
   \
   rpc-stubs/nonstaterpcserverstub.h \

src/charon.cpp

@@ -21,6 +21,7 @@
 #include "config.h"
 
 #include "pxrpcserver.hpp"
+#include "rest.hpp"
 
 #include <charon/notifications.hpp>
 #include <charon/rpcserver.hpp>
@@ -62,6 +63,12 @@ DEFINE_int32 (charon_timeout_ms, 3000,
               "Timeout in ms that the Charon client will wait"
               " for a server response");
 
+DEFINE_string (rest_endpoint, "https://rest.taurion.io",
+               "URL for the REST API that is used in the Charon client");
+DEFINE_string (cafile, "",
+               "if set, trust these certificates for TLS"
+               " instead of the cURL default");
+
 /** Interval for Charon server reconnects.  */
 const auto RECONNECT_INTERVAL = std::chrono::seconds (5);
 
@@ -115,12 +122,6 @@ const std::map<std::string, PXRpcMethod> CHARON_METHODS = {
   {"getserviceinfo", &PXRpcServer::getserviceinfoI},
 
   {"getversion", &PXRpcServer::getversionI},
-
-  /* FIXME: Instead of handling that through Charon, use an HTTP server to
-     download the bootstrap data.
-
-     See also https://github.com/xaya/taurion_gsp/issues/162.  */
-  {"getbootstrapdata", &PXRpcServer::getbootstrapdataI},
 };
 
 /**
@@ -377,6 +378,9 @@ class RealCharonClient : public CharonClient
   /** The Charon client.  */
   charon::Client client;
 
+  /** The REST client.  */
+  RestClient rest;
+
   /** The RPC server, if one has been started / set up.  */
   std::unique_ptr<RpcServer> rpc;
 
@@ -394,11 +398,15 @@ class RealCharonClient : public CharonClient
   explicit RealCharonClient (const std::string& serverJid,
                              const std::string& clientJid,
                              const std::string& password)
-    : client(serverJid, GetBackendVersion (), clientJid, password)
+    : client(serverJid, GetBackendVersion (), clientJid, password),
+      rest(FLAGS_rest_endpoint)
   {
     LOG (INFO)
         << "Using " << serverJid << " as Charon server,"
         << " requiring backend version " << GetBackendVersion ();
+    LOG (INFO)
+        << "REST endpoint: " << FLAGS_rest_endpoint;
+    rest.SetCaFile (FLAGS_cafile);
   }
 
   /**
@@ -435,6 +443,8 @@ RealCharonClient::RpcServer::RpcServer (RealCharonClient& p,
   jsonrpc::Procedure stopProc("stop", jsonrpc::PARAMS_BY_POSITION, nullptr);
   bindAndAddNotification (stopProc, &RpcServer::stop);
 
+  AddMethod ("getbootstrapdata");
+
   for (const auto& entry : CHARON_METHODS)
     AddMethod (entry.first);
   for (const auto& entry : NONSTATE_METHODS)
@@ -451,6 +461,21 @@ RealCharonClient::RpcServer::HandleMethodCall (jsonrpc::Procedure& proc,
 {
   const auto& method = proc.GetProcedureName ();
 
+  if (method == "getbootstrapdata")
+    {
+      VLOG (1) << "Getting bootstrap data through REST...";
+      try
+        {
+          result = parent.rest.GetBootstrapData ();
+          return;
+        }
+      catch (const std::runtime_error& exc)
+        {
+          throw jsonrpc::JsonRpcException (
+              jsonrpc::Errors::ERROR_RPC_INTERNAL_ERROR, exc.what ());
+        }
+    }
+
   if (CHARON_METHODS.find (method) != CHARON_METHODS.end ())
     {
       VLOG (1) << "Forwarding method " << method << " through Charon";

src/main.cpp

@@ -22,6 +22,7 @@
 #include "logic.hpp"
 #include "pending.hpp"
 #include "pxrpcserver.hpp"
+#include "rest.hpp"
 #include "version.hpp"
 
 #include <xayagame/defaultmain.hpp>
@@ -49,6 +50,9 @@ DEFINE_int32 (game_rpc_port, 0,
 DEFINE_bool (game_rpc_listen_locally, true,
              "whether the game's JSON-RPC server should listen locally");
 
+DEFINE_int32 (rest_port, 0,
+              "if non-zero, the port at which the REST interface should run");
+
 DEFINE_int32 (enable_pruning, -1,
               "if non-negative (including zero), old undo data will be pruned"
               " and only as many blocks as specified will be kept");
@@ -71,12 +75,21 @@ class PXInstanceFactory : public xaya::CustomisedInstanceFactory
    */
   pxd::PXLogic& rules;
 
+  /** The REST API port.  */
+  int restPort = 0;
+
 public:
 
   explicit PXInstanceFactory (pxd::PXLogic& r)
     : rules(r)
   {}
 
+  void
+  EnableRest (const int p)
+  {
+    restPort = p;
+  }
+
   std::unique_ptr<xaya::RpcServerInterface>
   BuildRpcServer (xaya::Game& game,
                   jsonrpc::AbstractServerConnector& conn) override
@@ -96,6 +109,9 @@ class PXInstanceFactory : public xaya::CustomisedInstanceFactory
     if (charonSrv != nullptr)
       res.push_back (std::move (charonSrv));
 
+    if (restPort != 0)
+      res.push_back (std::make_unique<pxd::RestApi> (game, rules, restPort));
+
     return res;
   }
 
@@ -175,6 +191,8 @@ main (int argc, char** argv)
 
   pxd::PXLogic rules;
   PXInstanceFactory instanceFact(rules);
+  if (FLAGS_rest_port != 0)
+    instanceFact.EnableRest (FLAGS_rest_port);
   config.InstanceFactory = &instanceFact;
 
   pxd::PendingMoves pending(rules);

src/rest.cpp

@@ -0,0 +1,126 @@
+// Copyright (C) 2020 The Xaya developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include "rest.hpp"
+
+#include "gamestatejson.hpp"
+
+#include <microhttpd.h>
+
+#include <gflags/gflags.h>
+#include <glog/logging.h>
+
+#include <chrono>
+
+namespace pxd
+{
+
+namespace
+{
+
+DEFINE_int32 (rest_bootstrap_refresh_seconds, 60 * 60,
+              "the refresh interval for bootstrap data in seconds");
+
+} // anonymous namespace
+
+std::shared_ptr<RestApi::SuccessResult>
+RestApi::ComputeBootstrapData ()
+{
+  const Json::Value val = logic.GetCustomStateData (game,
+    [] (GameStateJson& gsj)
+      {
+        return gsj.BootstrapData ();
+      });
+  auto res = std::make_shared<SuccessResult> (SuccessResult (val).Gzip ());
+
+  if (val["state"].asString () == "up-to-date")
+    {
+      LOG (INFO) << "Refreshing bootstrap-data cache";
+      std::lock_guard<std::mutex> lock(mutBootstrap);
+      bootstrapData = res;
+    }
+  else
+    LOG (WARNING) << "We are still catching up, not caching bootstrap data";
+
+  return res;
+}
+
+RestApi::SuccessResult
+RestApi::Process (const std::string& url)
+{
+  std::string remainder;
+  if (MatchEndpoint (url, "/bootstrap.json.gz", remainder) && remainder == "")
+    {
+      std::shared_ptr<SuccessResult> res;
+      {
+        std::lock_guard<std::mutex> lock(mutBootstrap);
+        res = bootstrapData;
+      }
+      if (res == nullptr)
+        res = ComputeBootstrapData ();
+      CHECK (res != nullptr);
+      return *res;
+    }
+
+  throw HttpError (MHD_HTTP_NOT_FOUND, "invalid API endpoint");
+}
+
+void
+RestApi::Start ()
+{
+  xaya::RestApi::Start ();
+
+  std::lock_guard<std::mutex> lock(mutStop);
+  shouldStop = false;
+  CHECK (bootstrapRefresher == nullptr);
+  bootstrapRefresher = std::make_unique<std::thread> ([this] ()
+    {
+      const auto intv
+          = std::chrono::seconds (FLAGS_rest_bootstrap_refresh_seconds);
+      while (true)
+        {
+          ComputeBootstrapData ();
+
+          std::unique_lock<std::mutex> lock(mutStop);
+          if (shouldStop)
+            break;
+          cvStop.wait_for (lock, intv);
+          if (shouldStop)
+            break;
+        }
+    });
+}
+
+void
+RestApi::Stop ()
+{
+  {
+    std::lock_guard<std::mutex> lock(mutStop);
+    shouldStop = true;
+    cvStop.notify_all ();
+  }
+
+  if (bootstrapRefresher != nullptr)
+    {
+      bootstrapRefresher->join ();
+      bootstrapRefresher.reset ();
+    }
+
+  xaya::RestApi::Stop ();
+}
+
+Json::Value
+RestClient::GetBootstrapData ()
+{
+  Request req(*this);
+  if (!req.Send ("/bootstrap.json.gz"))
+    throw std::runtime_error (req.GetError ());
+
+  if (req.GetType () != "application/json")
+    throw std::runtime_error ("response is not JSON");
+
+  return req.GetJson ();
+}
+
+} // namespace pxd