From 4328bc4a7ef8669b6ba86008fb891e22c559e626 Mon Sep 17 00:00:00 2001 From: schmidtw Date: Wed, 4 Oct 2023 01:22:58 -0700 Subject: [PATCH 1/2] Connect to the test cluster using configuration. --- .gitignore | 5 ++- config.go | 36 ++++++++++++++-- credentials.go | 97 ++++++++++++++++++++++++++++++++++++++++++++ fs.go | 34 ++++++++++++++++ internal/fs/os/os.go | 36 +++++++++++----- main.go | 8 ++++ 6 files changed, 200 insertions(+), 16 deletions(-) create mode 100644 credentials.go create mode 100644 fs.go diff --git a/.gitignore b/.gitignore index 07cdbb9..2ef410b 100644 --- a/.gitignore +++ b/.gitignore @@ -26,4 +26,7 @@ internal/jwtxt/cmd/example/* internal/credentials/cmd/example/* *.dot -*.pem \ No newline at end of file +*.pem +*.yml + +.storage diff --git a/config.go b/config.go index 9b49917..f341e33 100644 --- a/config.go +++ b/config.go @@ -5,6 +5,7 @@ package main import ( "fmt" + "io/fs" "os" "time" @@ -19,7 +20,9 @@ type Config struct { Identity Identity OperationalState OperationalState XmidtCredentials XmidtCredentials + XmidtService XmidtService Logger sallust.Config + Storage Storage } type Identity struct { @@ -37,9 +40,11 @@ type OperationalState struct { } type XmidtCredentials struct { - URL string - HTTPClient arrangehttp.ClientConfig - RefetchPercent float64 + URL string + HTTPClient arrangehttp.ClientConfig + RefetchPercent float64 + FileName string + FilePermissions fs.FileMode } type XmidtService struct { @@ -64,6 +69,11 @@ type Backoff struct { MaxDelay time.Duration } +type Storage struct { + Temporary string + Durable string +} + // Collect and process the configuration files and env vars and // produce a configuration object. func provideConfig(cli *CLI) (*goschtalt.Config, error) { @@ -125,4 +135,22 @@ func provideConfig(cli *CLI) (*goschtalt.Config, error) { // see what the default configuration is. // ----------------------------------------------------------------------------- -var defaultConfig = Config{} +var defaultConfig = Config{ + XmidtCredentials: XmidtCredentials{ + RefetchPercent: 90.0, + FileName: "credentials.msgpack", + FilePermissions: fs.FileMode(0600), + }, + XmidtService: XmidtService{ + JwtTxtRedirector: JwtTxtRedirector{ + Required: true, + Timeout: 10 * time.Second, + AllowedAlgorithms: []string{ + "EdDSA", + "ES256", "ES384", "ES512", + "PS256", "PS384", "PS512", + "RS256", "RS384", "RS512", + }, + }, + }, +} diff --git a/credentials.go b/credentials.go new file mode 100644 index 0000000..5e69172 --- /dev/null +++ b/credentials.go @@ -0,0 +1,97 @@ +// SPDX-FileCopyrightText: 2023 Comcast Cable Communications Management, LLC +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "time" + + "github.com/xmidt-org/xmidt-agent/internal/credentials" + "github.com/xmidt-org/xmidt-agent/internal/credentials/event" + "github.com/xmidt-org/xmidt-agent/internal/fs" + "go.uber.org/fx" + "go.uber.org/zap" +) + +const ( + xmidtProtocol = "protocol" +) + +type credsIn struct { + fx.In + Creds XmidtCredentials + ID Identity + Ops OperationalState + Durable fs.FS `name:"durable_fs" optional:"true"` + LC fx.Lifecycle + Logger *zap.Logger +} + +func credentialsModule() fx.Option { + return fx.Module( + "credentials", + fx.Provide( + func(in credsIn) (*credentials.Credentials, error) { + client, err := in.Creds.HTTPClient.NewClient() + if err != nil { + return nil, err + } + + logger := in.Logger.Named("credentials") + + opts := []credentials.Option{ + credentials.URL(in.Creds.URL), + credentials.HTTPClient(client), + credentials.MacAddress(in.ID.DeviceID), + credentials.SerialNumber(in.ID.SerialNumber), + credentials.HardwareModel(in.ID.HardwareModel), + credentials.HardwareManufacturer(in.ID.HardwareManufacturer), + credentials.FirmwareVersion(in.ID.FirmwareVersion), + credentials.PartnerID(func() string { return in.ID.PartnerID }), + credentials.LastRebootReason(in.Ops.LastRebootReason), + credentials.XmidtProtocol(xmidtProtocol), + credentials.BootRetryWait(time.Second), + credentials.RefetchPercent(in.Creds.RefetchPercent), + credentials.AddFetchListener(event.FetchListenerFunc( + func(e event.Fetch) { + logger.Info("fetch", + zap.String("origin", e.Origin), + zap.Time("at", e.At), + zap.Duration("duration", e.Duration), + zap.String("uuid", e.UUID.String()), + zap.Int("status_code", e.StatusCode), + zap.Duration("retry_in", e.RetryIn), + zap.Time("expiration", e.Expiration), + zap.Error(e.Err), + ) + })), + } + + if in.Durable != nil { + opts = append(opts, + credentials.LocalStorage(in.Durable, in.Creds.FileName, in.Creds.FilePermissions), + ) + } + + creds, err := credentials.New(opts...) + + in.LC.Append(fx.Hook{ + OnStart: func(context.Context) error { + creds.Start() + return nil + }, + OnStop: func(context.Context) error { + creds.Stop() + return nil + }, + }) + + return creds, err + }, + ), + fx.Invoke( + func(*credentials.Credentials) error { return nil }, + ), + ) +} diff --git a/fs.go b/fs.go new file mode 100644 index 0000000..e6652c3 --- /dev/null +++ b/fs.go @@ -0,0 +1,34 @@ +// SPDX-FileCopyrightText: 2023 Comcast Cable Communications Management, LLC +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "errors" + + "github.com/xmidt-org/xmidt-agent/internal/fs" + "github.com/xmidt-org/xmidt-agent/internal/fs/os" + "go.uber.org/fx" +) + +func fsProvide() fx.Option { + return fx.Provide( + fx.Annotate( + func(s Storage) (fs.FS, fs.FS, error) { + var tmp, durable fs.FS + var err, errs error + + if s.Temporary != "" { + tmp, err = os.New(s.Temporary) + errs = errors.Join(errs, err) + } + if s.Durable != "" { + durable, err = os.New(s.Durable) + errs = errors.Join(errs, err) + } + return tmp, durable, errs + }, + fx.ResultTags(`name:"temporary_fs"`, `name:"durable_fs"`), + ), + ) +} diff --git a/internal/fs/os/os.go b/internal/fs/os/os.go index 902556f..47e674e 100644 --- a/internal/fs/os/os.go +++ b/internal/fs/os/os.go @@ -6,33 +6,47 @@ package os import ( iofs "io/fs" "os" + "path/filepath" xafs "github.com/xmidt-org/xmidt-agent/internal/fs" ) // Provide the os implementation of the internal FS interface. -type fs struct{} +type fs struct { + base string +} + +// New creates a new fs struct with the specified base directory. If the +// directory does not exist, it is created with 0755 permissions. +func New(base string) (*fs, error) { + tmp := fs{} + + err := xafs.Operate(&tmp, xafs.WithDirs(base, 0755)) + if err != nil { + return nil, err + } -func New() *fs { - return &fs{} + return &fs{ + base: base, + }, nil } // Ensure that the fs struct implements the internal and fs.FS interfaces. var _ iofs.FS = (*fs)(nil) var _ xafs.FS = (*fs)(nil) -func (*fs) Open(name string) (iofs.File, error) { - return os.Open(name) +func (f *fs) Open(name string) (iofs.File, error) { + return os.Open(filepath.Join(f.base, name)) } -func (*fs) Mkdir(path string, perm iofs.FileMode) error { - return os.Mkdir(path, perm) +func (f *fs) Mkdir(path string, perm iofs.FileMode) error { + return os.Mkdir(filepath.Join(f.base, path), perm) } -func (*fs) ReadFile(name string) ([]byte, error) { - return os.ReadFile(name) +func (f *fs) ReadFile(name string) ([]byte, error) { + return os.ReadFile(filepath.Join(f.base, name)) } -func (*fs) WriteFile(name string, data []byte, perm iofs.FileMode) error { - return os.WriteFile(name, data, perm) +func (f *fs) WriteFile(name string, data []byte, perm iofs.FileMode) error { + return os.WriteFile(filepath.Join(f.base, name), data, perm) } diff --git a/main.go b/main.go index d8cba33..192a32d 100644 --- a/main.go +++ b/main.go @@ -69,8 +69,16 @@ func xmidtAgent(args []string) (*fx.App, error) { provideConfig, goschtalt.UnmarshalFunc[sallust.Config]("logger", goschtalt.Optional()), + goschtalt.UnmarshalFunc[Identity]("identity"), + goschtalt.UnmarshalFunc[OperationalState]("operational_state"), + goschtalt.UnmarshalFunc[XmidtCredentials]("xmidt_credentials"), + goschtalt.UnmarshalFunc[XmidtService]("xmidt_service"), + goschtalt.UnmarshalFunc[Storage]("storage"), ), + credentialsModule(), + fsProvide(), + fx.Invoke(), ) From a2abb79d4d7e656f4d2accbccd93fa1c628141b6 Mon Sep 17 00:00:00 2001 From: schmidtw Date: Wed, 4 Oct 2023 01:42:02 -0700 Subject: [PATCH 2/2] Enable the credentials to be optional. --- credentials.go | 115 +++++++++++++++++++++++-------------------------- main.go | 10 ++++- 2 files changed, 63 insertions(+), 62 deletions(-) diff --git a/credentials.go b/credentials.go index 5e69172..3f0a6e4 100644 --- a/credentials.go +++ b/credentials.go @@ -28,70 +28,65 @@ type credsIn struct { Logger *zap.Logger } -func credentialsModule() fx.Option { - return fx.Module( - "credentials", - fx.Provide( - func(in credsIn) (*credentials.Credentials, error) { - client, err := in.Creds.HTTPClient.NewClient() - if err != nil { - return nil, err - } +func provideCredentials(in credsIn) (*credentials.Credentials, error) { + // If the URL is empty, then there is no credentials service to use. + if in.Creds.URL == "" { + return nil, nil + } - logger := in.Logger.Named("credentials") + client, err := in.Creds.HTTPClient.NewClient() + if err != nil { + return nil, err + } - opts := []credentials.Option{ - credentials.URL(in.Creds.URL), - credentials.HTTPClient(client), - credentials.MacAddress(in.ID.DeviceID), - credentials.SerialNumber(in.ID.SerialNumber), - credentials.HardwareModel(in.ID.HardwareModel), - credentials.HardwareManufacturer(in.ID.HardwareManufacturer), - credentials.FirmwareVersion(in.ID.FirmwareVersion), - credentials.PartnerID(func() string { return in.ID.PartnerID }), - credentials.LastRebootReason(in.Ops.LastRebootReason), - credentials.XmidtProtocol(xmidtProtocol), - credentials.BootRetryWait(time.Second), - credentials.RefetchPercent(in.Creds.RefetchPercent), - credentials.AddFetchListener(event.FetchListenerFunc( - func(e event.Fetch) { - logger.Info("fetch", - zap.String("origin", e.Origin), - zap.Time("at", e.At), - zap.Duration("duration", e.Duration), - zap.String("uuid", e.UUID.String()), - zap.Int("status_code", e.StatusCode), - zap.Duration("retry_in", e.RetryIn), - zap.Time("expiration", e.Expiration), - zap.Error(e.Err), - ) - })), - } + logger := in.Logger.Named("credentials") - if in.Durable != nil { - opts = append(opts, - credentials.LocalStorage(in.Durable, in.Creds.FileName, in.Creds.FilePermissions), - ) - } + opts := []credentials.Option{ + credentials.URL(in.Creds.URL), + credentials.HTTPClient(client), + credentials.MacAddress(in.ID.DeviceID), + credentials.SerialNumber(in.ID.SerialNumber), + credentials.HardwareModel(in.ID.HardwareModel), + credentials.HardwareManufacturer(in.ID.HardwareManufacturer), + credentials.FirmwareVersion(in.ID.FirmwareVersion), + credentials.PartnerID(func() string { return in.ID.PartnerID }), + credentials.LastRebootReason(in.Ops.LastRebootReason), + credentials.XmidtProtocol(xmidtProtocol), + credentials.BootRetryWait(time.Second), + credentials.RefetchPercent(in.Creds.RefetchPercent), + credentials.AddFetchListener(event.FetchListenerFunc( + func(e event.Fetch) { + logger.Info("fetch", + zap.String("origin", e.Origin), + zap.Time("at", e.At), + zap.Duration("duration", e.Duration), + zap.String("uuid", e.UUID.String()), + zap.Int("status_code", e.StatusCode), + zap.Duration("retry_in", e.RetryIn), + zap.Time("expiration", e.Expiration), + zap.Error(e.Err), + ) + })), + } - creds, err := credentials.New(opts...) + if in.Durable != nil { + opts = append(opts, + credentials.LocalStorage(in.Durable, in.Creds.FileName, in.Creds.FilePermissions), + ) + } - in.LC.Append(fx.Hook{ - OnStart: func(context.Context) error { - creds.Start() - return nil - }, - OnStop: func(context.Context) error { - creds.Stop() - return nil - }, - }) + creds, err := credentials.New(opts...) - return creds, err - }, - ), - fx.Invoke( - func(*credentials.Credentials) error { return nil }, - ), - ) + in.LC.Append(fx.Hook{ + OnStart: func(context.Context) error { + creds.Start() + return nil + }, + OnStop: func(context.Context) error { + creds.Stop() + return nil + }, + }) + + return creds, err } diff --git a/main.go b/main.go index 192a32d..5496fdb 100644 --- a/main.go +++ b/main.go @@ -13,6 +13,7 @@ import ( _ "github.com/goschtalt/yaml-decoder" _ "github.com/goschtalt/yaml-encoder" "github.com/xmidt-org/sallust" + "github.com/xmidt-org/xmidt-agent/internal/credentials" "go.uber.org/fx" "go.uber.org/fx/fxevent" @@ -67,6 +68,7 @@ func xmidtAgent(args []string) (*fx.App, error) { provideCLI, provideLogger, provideConfig, + provideCredentials, goschtalt.UnmarshalFunc[sallust.Config]("logger", goschtalt.Optional()), goschtalt.UnmarshalFunc[Identity]("identity"), @@ -76,10 +78,14 @@ func xmidtAgent(args []string) (*fx.App, error) { goschtalt.UnmarshalFunc[Storage]("storage"), ), - credentialsModule(), fsProvide(), - fx.Invoke(), + fx.Invoke( + // TODO: Remove this. + // For now require the credentials to be fetched this way. Later + // Other services will depend on this. + func(*credentials.Credentials) {}, + ), ) if cli != nil && cli.Graph != "" {