From fcd2c672079d03c2792a4ef1b79df4af31526957 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Date: Thu, 26 Jan 2017 11:37:16 +0100 Subject: [PATCH] Add caching of nodes for rol --- etc/oc-monolithic.conf | 6 ++++-- .../authenticationwebserviceclient.go | 18 ++++++++++++++---- .../datawebserviceclient.go | 14 ++++++++++++-- .../metadatawebserviceclient.go | 14 ++++++++++++-- .../proxiedauthenticationwebservice.go | 14 +++++++++++++- .../proxieddatawebservice.go | 14 +++++++++++++- .../proxiedmetadatawebservice.go | 14 +++++++++++++- .../proxiedocwebservice/proxiedocwebservice.go | 14 +++++++++++++- .../remotebasicauthmiddleware.go | 6 +++--- 9 files changed, 97 insertions(+), 17 deletions(-) diff --git a/etc/oc-monolithic.conf b/etc/oc-monolithic.conf index 8e73fcc..8945f72 100644 --- a/etc/oc-monolithic.conf +++ b/etc/oc-monolithic.conf @@ -16,9 +16,10 @@ "ocfsm_data_driver_temporary_folder": "/tmp", "ocfsm_data_driver_max_sql_iddle": 0, "ocfsm_data_driver_max_sql_concurrent": 0, - "ocfsm_data_driver_dsn": "root:passwd@tcp(localhost:3306)/owncloud", + "ocfsm_data_driver_dsn": "root:labkode@tcp(localhost:3306)/owncloud", "token_driver": "jwttokendriver", "jwt_token_driver_key": "", + "basic_auth_middleware": "local", "basic_auth_middleware_cookie_name": "oc_sessionpassphrase", "cors_middleware_enabled": true, "cors_middleware_access_control_allow_origin": "*", @@ -30,5 +31,6 @@ "oc_web_service": "local", "data_web_service_max_upload_file_size": 10000000000, "oc_web_service_max_upload_file_size": 10000000000, - "oc_web_service_chunks_folder": "/tmp/chunks" + "oc_web_service_chunks_folder": "/tmp/chunks", + "registry_driver": "etcd" } diff --git a/root/authenticationwebserviceclient/authenticationwebserviceclient.go b/root/authenticationwebserviceclient/authenticationwebserviceclient.go index 64a4cc3..1201c12 100644 --- a/root/authenticationwebserviceclient/authenticationwebserviceclient.go +++ b/root/authenticationwebserviceclient/authenticationwebserviceclient.go @@ -7,6 +7,7 @@ import ( "fmt" "github.com/clawio/clawiod/root" "github.com/go-kit/kit/log/levels" + "github.com/patrickmn/go-cache" "io/ioutil" "math/rand" "net/http" @@ -18,15 +19,22 @@ type webServiceClient struct { cm root.ContextManager client *http.Client registryDriver root.RegistryDriver + cache *cache.Cache } // New returns an implementation of DataDriver. -func New(logger levels.Levels, cm root.ContextManager, registryDriver root.RegistryDriver) root.AuthenticationWebServiceClient{ - rand.Seed(time.Now().Unix()) // initialize global pseudorandom generator - return &webServiceClient{logger: logger, cm: cm, client: http.DefaultClient, registryDriver: registryDriver} +func New(logger levels.Levels, cm root.ContextManager, registryDriver root.RegistryDriver) root.AuthenticationWebServiceClient { + cache := cache.New(time.Second*10, time.Second*10) + rand.Seed(time.Now().Unix()) // initialize global pseudo-random generator + return &webServiceClient{logger: logger, cm: cm, client: http.DefaultClient, registryDriver: registryDriver, cache: cache} } func (c *webServiceClient) getAuthenticationURL(ctx context.Context) (string, error) { + u, ok := c.cache.Get("url") + if ok { + return u.(string), nil + } + // TODO(labkode) the logic for choosing a node is very rudimentary. // In the future would be nice to have at least RoundRobin. // Thanks that clients are registry aware we an use our own algorithms @@ -42,7 +50,9 @@ func (c *webServiceClient) getAuthenticationURL(ctx context.Context) (string, er c.logger.Info().Log("msg", "got authentication-nodes", "numnodes", len(nodes)) chosenNode := nodes[rand.Intn(len(nodes))] c.logger.Info().Log("msg", "authentication-node chosen", "authentication-node-url", chosenNode.URL()) - return chosenNode.URL() + "/auth", nil + chosenURL := chosenNode.URL() + "/auth" + c.cache.Set("url", chosenURL, cache.NoExpiration) + return chosenURL, nil } func (c *webServiceClient) Token(ctx context.Context, username, password string) (string, error) { diff --git a/root/datawebserviceclient/datawebserviceclient.go b/root/datawebserviceclient/datawebserviceclient.go index ab84b33..5af0672 100644 --- a/root/datawebserviceclient/datawebserviceclient.go +++ b/root/datawebserviceclient/datawebserviceclient.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/clawio/clawiod/root" "github.com/go-kit/kit/log/levels" + "github.com/patrickmn/go-cache" "io" "math/rand" "net/http" @@ -17,15 +18,22 @@ type webServiceClient struct { cm root.ContextManager client *http.Client registryDriver root.RegistryDriver + cache *cache.Cache } // New returns an implementation of DataDriver. func New(logger levels.Levels, cm root.ContextManager, registryDriver root.RegistryDriver) root.DataWebServiceClient { + cache := cache.New(time.Second*10, time.Second*10) rand.Seed(time.Now().Unix()) // initialize global pseudorandom generator - return &webServiceClient{logger: logger, cm: cm, client: http.DefaultClient, registryDriver: registryDriver} + return &webServiceClient{logger: logger, cm: cm, client: http.DefaultClient, registryDriver: registryDriver, cache: cache} } func (c *webServiceClient) getDataURL(ctx context.Context) (string, error) { + u, ok := c.cache.Get("url") + if ok { + return u.(string), nil + } + // TODO(labkode) the logic for choosing a node is very rudimentary. // In the future would be nice to have at least RoundRobin. // Thanks that clients are registry aware we an use our own algorithms @@ -41,7 +49,9 @@ func (c *webServiceClient) getDataURL(ctx context.Context) (string, error) { c.logger.Info().Log("msg", "got data-nodes", "numnodes", len(nodes)) chosenNode := nodes[rand.Intn(len(nodes))] c.logger.Info().Log("msg", "data-node chosen", "data-node-url", chosenNode.URL()) - return chosenNode.URL() + "/data", nil + chosenURL := chosenNode.URL() + "/data" + c.cache.Set("url", chosenURL, cache.NoExpiration) + return chosenURL, nil } func (c *webServiceClient) UploadFile(ctx context.Context, user root.User, path string, r io.ReadCloser, clientChecksum string) error { traceID := c.cm.MustGetTraceID(ctx) diff --git a/root/metadatawebserviceclient/metadatawebserviceclient.go b/root/metadatawebserviceclient/metadatawebserviceclient.go index c9102a4..7238696 100644 --- a/root/metadatawebserviceclient/metadatawebserviceclient.go +++ b/root/metadatawebserviceclient/metadatawebserviceclient.go @@ -7,9 +7,11 @@ import ( "fmt" "github.com/clawio/clawiod/root" "github.com/go-kit/kit/log/levels" + "github.com/patrickmn/go-cache" "io/ioutil" "math/rand" "net/http" + "time" ) type webServiceClient struct { @@ -17,13 +19,20 @@ type webServiceClient struct { cm root.ContextManager client *http.Client registryDriver root.RegistryDriver + cache *cache.Cache } func New(logger levels.Levels, cm root.ContextManager, registryDriver root.RegistryDriver) root.MetaDataWebServiceClient { - return &webServiceClient{logger: logger, cm: cm, registryDriver: registryDriver, client: http.DefaultClient} + cache := cache.New(time.Second*10, time.Second*10) + return &webServiceClient{logger: logger, cm: cm, registryDriver: registryDriver, client: http.DefaultClient, cache: cache} } func (c *webServiceClient) getMetaDataURL(ctx context.Context) (string, error) { + u, ok := c.cache.Get("url") + if ok { + return u.(string), nil + } + // TODO(labkode) the logic for choosing a node is very rudimentary. // In the future would be nice to have at least RoundRobin. // Thanks that clients are registry aware we an use our own algorithms @@ -39,7 +48,8 @@ func (c *webServiceClient) getMetaDataURL(ctx context.Context) (string, error) { c.logger.Info().Log("msg", "got metadata-nodes", "numnodes", len(nodes)) chosenNode := nodes[rand.Intn(len(nodes))] c.logger.Info().Log("msg", "metadata-node chosen", "metadata-node-url", chosenNode.URL()) - return chosenNode.URL() + "/meta", nil + chosenURL := chosenNode.URL() + "/meta" + return chosenURL, nil } func (c *webServiceClient) Examine(ctx context.Context, user root.User, path string) (root.FileInfo, error) { diff --git a/root/proxiedauthenticationwebservice/proxiedauthenticationwebservice.go b/root/proxiedauthenticationwebservice/proxiedauthenticationwebservice.go index a2ddb1e..a293fb1 100644 --- a/root/proxiedauthenticationwebservice/proxiedauthenticationwebservice.go +++ b/root/proxiedauthenticationwebservice/proxiedauthenticationwebservice.go @@ -5,25 +5,35 @@ import ( "fmt" "github.com/clawio/clawiod/root" "github.com/go-kit/kit/log/levels" + "github.com/patrickmn/go-cache" "math/rand" "net/http" "net/http/httputil" "net/url" + "time" ) type service struct { logger levels.Levels registryDriver root.RegistryDriver + cache *cache.Cache } func New(logger levels.Levels, registryDriver root.RegistryDriver) (root.WebService, error) { + cache := cache.New(time.Second*10, time.Second*10) return &service{ logger: logger, registryDriver: registryDriver, + cache: cache, }, nil } func (s *service) getProxy(ctx context.Context) (*httputil.ReverseProxy, error) { + p, ok := s.cache.Get("proxy") + if ok { + return p.(*httputil.ReverseProxy), nil + } + // TODO(labkode) the logic for choosing a node is very rudimentary. // In the future would be nice to have at least RoundRobin. // Thanks that clients are registry aware we an use our own algorithms @@ -43,7 +53,9 @@ func (s *service) getProxy(ctx context.Context) (*httputil.ReverseProxy, error) if err != nil { return nil, err } - return httputil.NewSingleHostReverseProxy(u), nil + proxy := httputil.NewSingleHostReverseProxy(u) + s.cache.Set("proxy", proxy, cache.NoExpiration) + return proxy, nil } func (s *service) IsProxy() bool { diff --git a/root/proxieddatawebservice/proxieddatawebservice.go b/root/proxieddatawebservice/proxieddatawebservice.go index 6d8be80..48d0f65 100644 --- a/root/proxieddatawebservice/proxieddatawebservice.go +++ b/root/proxieddatawebservice/proxieddatawebservice.go @@ -5,25 +5,35 @@ import ( "fmt" "github.com/clawio/clawiod/root" "github.com/go-kit/kit/log/levels" + "github.com/patrickmn/go-cache" "math/rand" "net/http" "net/http/httputil" "net/url" + "time" ) type service struct { logger levels.Levels registryDriver root.RegistryDriver + cache *cache.Cache } func New(logger levels.Levels, registryDriver root.RegistryDriver) (root.WebService, error) { + cache := cache.New(time.Second * 10, time.Second * 10) return &service{ logger: logger, registryDriver: registryDriver, + cache: cache, }, nil } func (s *service) getProxy(ctx context.Context) (*httputil.ReverseProxy, error) { + p, ok := s.cache.Get("proxy") + if ok { + return p.(*httputil.ReverseProxy), nil + } + // TODO(labkode) the logic for choosing a node is very rudimentary. // In the future would be nice to have at least RoundRobin. // Thanks that clients are registry aware we an use our own algorithms @@ -43,7 +53,9 @@ func (s *service) getProxy(ctx context.Context) (*httputil.ReverseProxy, error) if err != nil { return nil, err } - return httputil.NewSingleHostReverseProxy(u), nil + proxy := httputil.NewSingleHostReverseProxy(u) + s.cache.Set("proxy", proxy, cache.NoExpiration) + return proxy, nil } func (s *service) IsProxy() bool { diff --git a/root/proxiedmetadatawebservice/proxiedmetadatawebservice.go b/root/proxiedmetadatawebservice/proxiedmetadatawebservice.go index 888ac0e..f5b90c7 100644 --- a/root/proxiedmetadatawebservice/proxiedmetadatawebservice.go +++ b/root/proxiedmetadatawebservice/proxiedmetadatawebservice.go @@ -5,25 +5,35 @@ import ( "fmt" "github.com/clawio/clawiod/root" "github.com/go-kit/kit/log/levels" + "github.com/patrickmn/go-cache" "math/rand" "net/http" "net/http/httputil" "net/url" + "time" ) type service struct { logger levels.Levels registryDriver root.RegistryDriver + cache *cache.Cache } func New(logger levels.Levels, registryDriver root.RegistryDriver) (root.WebService, error) { + cache := cache.New(time.Second*10, time.Second*10) return &service{ logger: logger, registryDriver: registryDriver, + cache: cache, }, nil } func (s *service) getProxy(ctx context.Context) (*httputil.ReverseProxy, error) { + p, ok := s.cache.Get("proxy") + if ok { + return p.(*httputil.ReverseProxy), nil + } + // TODO(labkode) the logic for choosing a node is very rudimentary. // In the future would be nice to have at least RoundRobin. // Thanks that clients are registry aware we an use our own algorithms @@ -43,7 +53,9 @@ func (s *service) getProxy(ctx context.Context) (*httputil.ReverseProxy, error) if err != nil { return nil, err } - return httputil.NewSingleHostReverseProxy(u), nil + proxy := httputil.NewSingleHostReverseProxy(u) + s.cache.Set("proxy", proxy, cache.NoExpiration) + return proxy, nil } func (s *service) IsProxy() bool { diff --git a/root/proxiedocwebservice/proxiedocwebservice.go b/root/proxiedocwebservice/proxiedocwebservice.go index 128f9b0..86156e5 100644 --- a/root/proxiedocwebservice/proxiedocwebservice.go +++ b/root/proxiedocwebservice/proxiedocwebservice.go @@ -5,25 +5,35 @@ import ( "fmt" "github.com/clawio/clawiod/root" "github.com/go-kit/kit/log/levels" + "github.com/patrickmn/go-cache" "math/rand" "net/http" "net/http/httputil" "net/url" + "time" ) type service struct { logger levels.Levels registryDriver root.RegistryDriver + cache *cache.Cache } func New(logger levels.Levels, registryDriver root.RegistryDriver) (root.WebService, error) { + cache := cache.New(time.Second*10, time.Second*10) return &service{ logger: logger, registryDriver: registryDriver, + cache: cache, }, nil } func (s *service) getProxy(ctx context.Context) (*httputil.ReverseProxy, error) { + p, ok := s.cache.Get("proxy") + if ok { + return p.(*httputil.ReverseProxy), nil + } + // TODO(labkode) the logic for choosing a node is very rudimentary. // In the future would be nice to have at least RoundRobin. // Thanks that clients are registry aware we an use our own algorithms @@ -43,7 +53,9 @@ func (s *service) getProxy(ctx context.Context) (*httputil.ReverseProxy, error) if err != nil { return nil, err } - return httputil.NewSingleHostReverseProxy(u), nil + proxy := httputil.NewSingleHostReverseProxy(u) + s.cache.Set("proxy", proxy, cache.NoExpiration) + return proxy, nil } func (s *service) IsProxy() bool { diff --git a/root/remotebasicauthmiddleware/remotebasicauthmiddleware.go b/root/remotebasicauthmiddleware/remotebasicauthmiddleware.go index e36f97a..d1f49d2 100644 --- a/root/remotebasicauthmiddleware/remotebasicauthmiddleware.go +++ b/root/remotebasicauthmiddleware/remotebasicauthmiddleware.go @@ -6,10 +6,10 @@ import ( ) type middleware struct { - cookieName string - cm root.ContextManager + cookieName string + cm root.ContextManager authenticationWebServiceClient root.AuthenticationWebServiceClient - tokenDriver root.TokenDriver + tokenDriver root.TokenDriver } func New(cm root.ContextManager, authenticationWebServiceClient root.AuthenticationWebServiceClient, tokenDriver root.TokenDriver, cookieName string) root.BasicAuthMiddleware {