From d252638f94bddc9214801341f36c1ea5c5a56e87 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Tue, 31 Dec 2024 17:38:05 +0000 Subject: [PATCH 01/36] B-22056 - work files copied over for backend api call. --- cmd/milmove/serve.go | 7 + go.mod | 4 +- go.sum | 4 + pkg/gen/internalapi/configure_mymove.go | 9 + pkg/gen/internalapi/doc.go | 1 + pkg/gen/internalapi/embedded_spec.go | 104 ++++++++++ .../internaloperations/mymove_api.go | 24 +++ .../uploads/get_upload_status.go | 58 ++++++ .../uploads/get_upload_status_parameters.go | 91 +++++++++ .../uploads/get_upload_status_responses.go | 177 ++++++++++++++++ .../uploads/get_upload_status_urlbuilder.go | 101 +++++++++ pkg/handlers/authentication/auth.go | 1 + pkg/handlers/config.go | 14 ++ pkg/handlers/config_test.go | 2 +- pkg/handlers/internalapi/api.go | 2 + .../internal/payloads/model_to_payload.go | 15 +- pkg/handlers/internalapi/uploads.go | 124 +++++++++++ pkg/handlers/internalapi/uploads_test.go | 44 ++++ pkg/models/upload.go | 35 +++- pkg/notifications/notification_receiver.go | 192 ++++++++++++++++++ swagger-def/internal.yaml | 37 ++++ swagger/internal.yaml | 36 ++++ 22 files changed, 1065 insertions(+), 17 deletions(-) create mode 100644 pkg/gen/internalapi/internaloperations/uploads/get_upload_status.go create mode 100644 pkg/gen/internalapi/internaloperations/uploads/get_upload_status_parameters.go create mode 100644 pkg/gen/internalapi/internaloperations/uploads/get_upload_status_responses.go create mode 100644 pkg/gen/internalapi/internaloperations/uploads/get_upload_status_urlbuilder.go create mode 100644 pkg/notifications/notification_receiver.go diff --git a/cmd/milmove/serve.go b/cmd/milmove/serve.go index 505936d3868..7168bf87acd 100644 --- a/cmd/milmove/serve.go +++ b/cmd/milmove/serve.go @@ -478,6 +478,12 @@ func buildRoutingConfig(appCtx appcontext.AppContext, v *viper.Viper, redisPool appCtx.Logger().Fatal("notification sender sending not enabled", zap.Error(err)) } + // Event Receiver + notificationReceiver, err := notifications.InitReceiver(v, appCtx.Logger()) + if err != nil { + appCtx.Logger().Fatal("notification receiver listening not enabled") + } + routingConfig.BuildRoot = v.GetString(cli.BuildRootFlag) sendProductionInvoice := v.GetBool(cli.GEXSendProdInvoiceFlag) @@ -567,6 +573,7 @@ func buildRoutingConfig(appCtx appcontext.AppContext, v *viper.Viper, redisPool dtodRoutePlanner, fileStorer, notificationSender, + notificationReceiver, iwsPersonLookup, sendProductionInvoice, gexSender, diff --git a/go.mod b/go.mod index e528f684f9d..74bbb9f4e0c 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,8 @@ require ( github.com/aws/aws-sdk-go-v2/service/rds v1.78.2 github.com/aws/aws-sdk-go-v2/service/s3 v1.59.0 github.com/aws/aws-sdk-go-v2/service/ses v1.25.3 + github.com/aws/aws-sdk-go-v2/service/sns v1.31.8 + github.com/aws/aws-sdk-go-v2/service/sqs v1.34.6 github.com/aws/aws-sdk-go-v2/service/ssm v1.52.8 github.com/aws/aws-sdk-go-v2/service/sts v1.30.7 github.com/aws/smithy-go v1.20.4 @@ -264,7 +266,7 @@ require ( golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.25.0 // indirect golang.org/x/term v0.24.0 // indirect - google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda // indirect + google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect google.golang.org/protobuf v1.34.2 // indirect diff --git a/go.sum b/go.sum index edfdd1c49f0..c272e10eca1 100644 --- a/go.sum +++ b/go.sum @@ -82,6 +82,10 @@ github.com/aws/aws-sdk-go-v2/service/s3 v1.59.0 h1:Cso4Ev/XauMVsbwdhYEoxg8rxZWw4 github.com/aws/aws-sdk-go-v2/service/s3 v1.59.0/go.mod h1:BSPI0EfnYUuNHPS0uqIo5VrRwzie+Fp+YhQOUs16sKI= github.com/aws/aws-sdk-go-v2/service/ses v1.25.3 h1:wcfUsE2nqsXhEj68gxr7MnGXNPcBPKx0RW2DzBVgVlM= github.com/aws/aws-sdk-go-v2/service/ses v1.25.3/go.mod h1:6Ul/Ir8oOCsI3dFN0prULK9fvpxP+WTYmlHDkFzaAVA= +github.com/aws/aws-sdk-go-v2/service/sns v1.31.8 h1:vRSk062d1SmaEVbiqFePkvYuhCTnW2JnPkUdt19nqeY= +github.com/aws/aws-sdk-go-v2/service/sns v1.31.8/go.mod h1:wjhxA9hlVu75dCL/5Wcx8Cwmszvu6t0i8WEDypcB4+s= +github.com/aws/aws-sdk-go-v2/service/sqs v1.34.6 h1:DbjODDHumQBdJ3T+EO7AXVoFUeUhAsJYOdjStH5Ws4A= +github.com/aws/aws-sdk-go-v2/service/sqs v1.34.6/go.mod h1:7idt3XszF6sE9WPS1GqZRiDJOxw4oPtlRBXodWnCGjU= github.com/aws/aws-sdk-go-v2/service/ssm v1.52.8 h1:7cjN4Wp3U3cud17TsnUxSomTwKzKQGUWdq/N1aWqgMk= github.com/aws/aws-sdk-go-v2/service/ssm v1.52.8/go.mod h1:nUSNPaG8mv5rIu7EclHnFqZOjhreEUwRKENtKTtJ9aw= github.com/aws/aws-sdk-go-v2/service/sso v1.22.7 h1:pIaGg+08llrP7Q5aiz9ICWbY8cqhTkyy+0SHvfzQpTc= diff --git a/pkg/gen/internalapi/configure_mymove.go b/pkg/gen/internalapi/configure_mymove.go index 3b277e0037c..d1fa1bc3756 100644 --- a/pkg/gen/internalapi/configure_mymove.go +++ b/pkg/gen/internalapi/configure_mymove.go @@ -4,6 +4,7 @@ package internalapi import ( "crypto/tls" + "io" "net/http" "github.com/go-openapi/errors" @@ -60,6 +61,9 @@ func configureAPI(api *internaloperations.MymoveAPI) http.Handler { api.BinProducer = runtime.ByteStreamProducer() api.JSONProducer = runtime.JSONProducer() + api.TextEventStreamProducer = runtime.ProducerFunc(func(w io.Writer, data interface{}) error { + return errors.NotImplemented("textEventStream producer has not yet been implemented") + }) // You may change here the memory limit for this multipart form parser. Below is the default (32 MB). // ppm.CreatePPMUploadMaxParseMemory = 32 << 20 @@ -205,6 +209,11 @@ func configureAPI(api *internaloperations.MymoveAPI) http.Handler { return middleware.NotImplemented("operation transportation_offices.GetTransportationOffices has not yet been implemented") }) } + if api.UploadsGetUploadStatusHandler == nil { + api.UploadsGetUploadStatusHandler = uploads.GetUploadStatusHandlerFunc(func(params uploads.GetUploadStatusParams) middleware.Responder { + return middleware.NotImplemented("operation uploads.GetUploadStatus has not yet been implemented") + }) + } if api.EntitlementsIndexEntitlementsHandler == nil { api.EntitlementsIndexEntitlementsHandler = entitlements.IndexEntitlementsHandlerFunc(func(params entitlements.IndexEntitlementsParams) middleware.Responder { return middleware.NotImplemented("operation entitlements.IndexEntitlements has not yet been implemented") diff --git a/pkg/gen/internalapi/doc.go b/pkg/gen/internalapi/doc.go index 463e7be3e81..f8040028e22 100644 --- a/pkg/gen/internalapi/doc.go +++ b/pkg/gen/internalapi/doc.go @@ -22,6 +22,7 @@ // Produces: // - application/pdf // - application/json +// - text/event-stream // // swagger:meta package internalapi diff --git a/pkg/gen/internalapi/embedded_spec.go b/pkg/gen/internalapi/embedded_spec.go index c1351734062..f30aca3b049 100644 --- a/pkg/gen/internalapi/embedded_spec.go +++ b/pkg/gen/internalapi/embedded_spec.go @@ -3272,6 +3272,58 @@ func init() { } } }, + "/uploads/{uploadId}/status": { + "get": { + "description": "Returns status of an upload based on antivirus run", + "produces": [ + "text/event-stream" + ], + "tags": [ + "uploads" + ], + "summary": "Returns status of an upload", + "operationId": "getUploadStatus", + "parameters": [ + { + "type": "string", + "format": "uuid", + "description": "UUID of the upload to return status of", + "name": "uploadId", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "the requested upload status", + "schema": { + "type": "string", + "enum": [ + "INFECTED", + "CLEAN", + "PROCESSING" + ], + "readOnly": true + } + }, + "400": { + "description": "invalid request", + "schema": { + "$ref": "#/definitions/InvalidRequestResponsePayload" + } + }, + "403": { + "description": "not authorized" + }, + "404": { + "description": "not found" + }, + "500": { + "description": "server error" + } + } + } + }, "/users/is_logged_in": { "get": { "description": "Returns boolean as to whether the user is logged in", @@ -12391,6 +12443,58 @@ func init() { } } }, + "/uploads/{uploadId}/status": { + "get": { + "description": "Returns status of an upload based on antivirus run", + "produces": [ + "text/event-stream" + ], + "tags": [ + "uploads" + ], + "summary": "Returns status of an upload", + "operationId": "getUploadStatus", + "parameters": [ + { + "type": "string", + "format": "uuid", + "description": "UUID of the upload to return status of", + "name": "uploadId", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "the requested upload status", + "schema": { + "type": "string", + "enum": [ + "INFECTED", + "CLEAN", + "PROCESSING" + ], + "readOnly": true + } + }, + "400": { + "description": "invalid request", + "schema": { + "$ref": "#/definitions/InvalidRequestResponsePayload" + } + }, + "403": { + "description": "not authorized" + }, + "404": { + "description": "not found" + }, + "500": { + "description": "server error" + } + } + } + }, "/users/is_logged_in": { "get": { "description": "Returns boolean as to whether the user is logged in", diff --git a/pkg/gen/internalapi/internaloperations/mymove_api.go b/pkg/gen/internalapi/internaloperations/mymove_api.go index b1ba4e1ac47..f061964c6f5 100644 --- a/pkg/gen/internalapi/internaloperations/mymove_api.go +++ b/pkg/gen/internalapi/internaloperations/mymove_api.go @@ -7,6 +7,7 @@ package internaloperations import ( "fmt" + "io" "net/http" "strings" @@ -66,6 +67,9 @@ func NewMymoveAPI(spec *loads.Document) *MymoveAPI { BinProducer: runtime.ByteStreamProducer(), JSONProducer: runtime.JSONProducer(), + TextEventStreamProducer: runtime.ProducerFunc(func(w io.Writer, data interface{}) error { + return errors.NotImplemented("textEventStream producer has not yet been implemented") + }), OfficeApproveMoveHandler: office.ApproveMoveHandlerFunc(func(params office.ApproveMoveParams) middleware.Responder { return middleware.NotImplemented("operation office.ApproveMove has not yet been implemented") @@ -148,6 +152,9 @@ func NewMymoveAPI(spec *loads.Document) *MymoveAPI { TransportationOfficesGetTransportationOfficesHandler: transportation_offices.GetTransportationOfficesHandlerFunc(func(params transportation_offices.GetTransportationOfficesParams) middleware.Responder { return middleware.NotImplemented("operation transportation_offices.GetTransportationOffices has not yet been implemented") }), + UploadsGetUploadStatusHandler: uploads.GetUploadStatusHandlerFunc(func(params uploads.GetUploadStatusParams) middleware.Responder { + return middleware.NotImplemented("operation uploads.GetUploadStatus has not yet been implemented") + }), EntitlementsIndexEntitlementsHandler: entitlements.IndexEntitlementsHandlerFunc(func(params entitlements.IndexEntitlementsParams) middleware.Responder { return middleware.NotImplemented("operation entitlements.IndexEntitlements has not yet been implemented") }), @@ -323,6 +330,9 @@ type MymoveAPI struct { // JSONProducer registers a producer for the following mime types: // - application/json JSONProducer runtime.Producer + // TextEventStreamProducer registers a producer for the following mime types: + // - text/event-stream + TextEventStreamProducer runtime.Producer // OfficeApproveMoveHandler sets the operation handler for the approve move operation OfficeApproveMoveHandler office.ApproveMoveHandler @@ -378,6 +388,8 @@ type MymoveAPI struct { AddressesGetLocationByZipCityStateHandler addresses.GetLocationByZipCityStateHandler // TransportationOfficesGetTransportationOfficesHandler sets the operation handler for the get transportation offices operation TransportationOfficesGetTransportationOfficesHandler transportation_offices.GetTransportationOfficesHandler + // UploadsGetUploadStatusHandler sets the operation handler for the get upload status operation + UploadsGetUploadStatusHandler uploads.GetUploadStatusHandler // EntitlementsIndexEntitlementsHandler sets the operation handler for the index entitlements operation EntitlementsIndexEntitlementsHandler entitlements.IndexEntitlementsHandler // MoveDocsIndexMoveDocumentsHandler sets the operation handler for the index move documents operation @@ -546,6 +558,9 @@ func (o *MymoveAPI) Validate() error { if o.JSONProducer == nil { unregistered = append(unregistered, "JSONProducer") } + if o.TextEventStreamProducer == nil { + unregistered = append(unregistered, "TextEventStreamProducer") + } if o.OfficeApproveMoveHandler == nil { unregistered = append(unregistered, "office.ApproveMoveHandler") @@ -628,6 +643,9 @@ func (o *MymoveAPI) Validate() error { if o.TransportationOfficesGetTransportationOfficesHandler == nil { unregistered = append(unregistered, "transportation_offices.GetTransportationOfficesHandler") } + if o.UploadsGetUploadStatusHandler == nil { + unregistered = append(unregistered, "uploads.GetUploadStatusHandler") + } if o.EntitlementsIndexEntitlementsHandler == nil { unregistered = append(unregistered, "entitlements.IndexEntitlementsHandler") } @@ -809,6 +827,8 @@ func (o *MymoveAPI) ProducersFor(mediaTypes []string) map[string]runtime.Produce result["application/pdf"] = o.BinProducer case "application/json": result["application/json"] = o.JSONProducer + case "text/event-stream": + result["text/event-stream"] = o.TextEventStreamProducer } if p, ok := o.customProducers[mt]; ok { @@ -960,6 +980,10 @@ func (o *MymoveAPI) initHandlerCache() { if o.handlers["GET"] == nil { o.handlers["GET"] = make(map[string]http.Handler) } + o.handlers["GET"]["/uploads/{uploadId}/status"] = uploads.NewGetUploadStatus(o.context, o.UploadsGetUploadStatusHandler) + if o.handlers["GET"] == nil { + o.handlers["GET"] = make(map[string]http.Handler) + } o.handlers["GET"]["/entitlements"] = entitlements.NewIndexEntitlements(o.context, o.EntitlementsIndexEntitlementsHandler) if o.handlers["GET"] == nil { o.handlers["GET"] = make(map[string]http.Handler) diff --git a/pkg/gen/internalapi/internaloperations/uploads/get_upload_status.go b/pkg/gen/internalapi/internaloperations/uploads/get_upload_status.go new file mode 100644 index 00000000000..dc2c021f021 --- /dev/null +++ b/pkg/gen/internalapi/internaloperations/uploads/get_upload_status.go @@ -0,0 +1,58 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package uploads + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the generate command + +import ( + "net/http" + + "github.com/go-openapi/runtime/middleware" +) + +// GetUploadStatusHandlerFunc turns a function with the right signature into a get upload status handler +type GetUploadStatusHandlerFunc func(GetUploadStatusParams) middleware.Responder + +// Handle executing the request and returning a response +func (fn GetUploadStatusHandlerFunc) Handle(params GetUploadStatusParams) middleware.Responder { + return fn(params) +} + +// GetUploadStatusHandler interface for that can handle valid get upload status params +type GetUploadStatusHandler interface { + Handle(GetUploadStatusParams) middleware.Responder +} + +// NewGetUploadStatus creates a new http.Handler for the get upload status operation +func NewGetUploadStatus(ctx *middleware.Context, handler GetUploadStatusHandler) *GetUploadStatus { + return &GetUploadStatus{Context: ctx, Handler: handler} +} + +/* + GetUploadStatus swagger:route GET /uploads/{uploadId}/status uploads getUploadStatus + +# Returns status of an upload + +Returns status of an upload based on antivirus run +*/ +type GetUploadStatus struct { + Context *middleware.Context + Handler GetUploadStatusHandler +} + +func (o *GetUploadStatus) ServeHTTP(rw http.ResponseWriter, r *http.Request) { + route, rCtx, _ := o.Context.RouteInfo(r) + if rCtx != nil { + *r = *rCtx + } + var Params = NewGetUploadStatusParams() + if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params + o.Context.Respond(rw, r, route.Produces, route, err) + return + } + + res := o.Handler.Handle(Params) // actually handle the request + o.Context.Respond(rw, r, route.Produces, route, res) + +} diff --git a/pkg/gen/internalapi/internaloperations/uploads/get_upload_status_parameters.go b/pkg/gen/internalapi/internaloperations/uploads/get_upload_status_parameters.go new file mode 100644 index 00000000000..1770aa8ca6b --- /dev/null +++ b/pkg/gen/internalapi/internaloperations/uploads/get_upload_status_parameters.go @@ -0,0 +1,91 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package uploads + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "net/http" + + "github.com/go-openapi/errors" + "github.com/go-openapi/runtime/middleware" + "github.com/go-openapi/strfmt" + "github.com/go-openapi/validate" +) + +// NewGetUploadStatusParams creates a new GetUploadStatusParams object +// +// There are no default values defined in the spec. +func NewGetUploadStatusParams() GetUploadStatusParams { + + return GetUploadStatusParams{} +} + +// GetUploadStatusParams contains all the bound params for the get upload status operation +// typically these are obtained from a http.Request +// +// swagger:parameters getUploadStatus +type GetUploadStatusParams struct { + + // HTTP Request Object + HTTPRequest *http.Request `json:"-"` + + /*UUID of the upload to return status of + Required: true + In: path + */ + UploadID strfmt.UUID +} + +// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface +// for simple values it will use straight method calls. +// +// To ensure default values, the struct must have been initialized with NewGetUploadStatusParams() beforehand. +func (o *GetUploadStatusParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { + var res []error + + o.HTTPRequest = r + + rUploadID, rhkUploadID, _ := route.Params.GetOK("uploadId") + if err := o.bindUploadID(rUploadID, rhkUploadID, route.Formats); err != nil { + res = append(res, err) + } + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} + +// bindUploadID binds and validates parameter UploadID from path. +func (o *GetUploadStatusParams) bindUploadID(rawData []string, hasKey bool, formats strfmt.Registry) error { + var raw string + if len(rawData) > 0 { + raw = rawData[len(rawData)-1] + } + + // Required: true + // Parameter is provided by construction from the route + + // Format: uuid + value, err := formats.Parse("uuid", raw) + if err != nil { + return errors.InvalidType("uploadId", "path", "strfmt.UUID", raw) + } + o.UploadID = *(value.(*strfmt.UUID)) + + if err := o.validateUploadID(formats); err != nil { + return err + } + + return nil +} + +// validateUploadID carries on validations for parameter UploadID +func (o *GetUploadStatusParams) validateUploadID(formats strfmt.Registry) error { + + if err := validate.FormatOf("uploadId", "path", "uuid", o.UploadID.String(), formats); err != nil { + return err + } + return nil +} diff --git a/pkg/gen/internalapi/internaloperations/uploads/get_upload_status_responses.go b/pkg/gen/internalapi/internaloperations/uploads/get_upload_status_responses.go new file mode 100644 index 00000000000..7b6b4b15b7d --- /dev/null +++ b/pkg/gen/internalapi/internaloperations/uploads/get_upload_status_responses.go @@ -0,0 +1,177 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package uploads + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "net/http" + + "github.com/go-openapi/runtime" + + "github.com/transcom/mymove/pkg/gen/internalmessages" +) + +// GetUploadStatusOKCode is the HTTP code returned for type GetUploadStatusOK +const GetUploadStatusOKCode int = 200 + +/* +GetUploadStatusOK the requested upload status + +swagger:response getUploadStatusOK +*/ +type GetUploadStatusOK struct { + + /* + In: Body + */ + Payload string `json:"body,omitempty"` +} + +// NewGetUploadStatusOK creates GetUploadStatusOK with default headers values +func NewGetUploadStatusOK() *GetUploadStatusOK { + + return &GetUploadStatusOK{} +} + +// WithPayload adds the payload to the get upload status o k response +func (o *GetUploadStatusOK) WithPayload(payload string) *GetUploadStatusOK { + o.Payload = payload + return o +} + +// SetPayload sets the payload to the get upload status o k response +func (o *GetUploadStatusOK) SetPayload(payload string) { + o.Payload = payload +} + +// WriteResponse to the client +func (o *GetUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { + + rw.WriteHeader(200) + payload := o.Payload + if err := producer.Produce(rw, payload); err != nil { + panic(err) // let the recovery middleware deal with this + } +} + +// GetUploadStatusBadRequestCode is the HTTP code returned for type GetUploadStatusBadRequest +const GetUploadStatusBadRequestCode int = 400 + +/* +GetUploadStatusBadRequest invalid request + +swagger:response getUploadStatusBadRequest +*/ +type GetUploadStatusBadRequest struct { + + /* + In: Body + */ + Payload *internalmessages.InvalidRequestResponsePayload `json:"body,omitempty"` +} + +// NewGetUploadStatusBadRequest creates GetUploadStatusBadRequest with default headers values +func NewGetUploadStatusBadRequest() *GetUploadStatusBadRequest { + + return &GetUploadStatusBadRequest{} +} + +// WithPayload adds the payload to the get upload status bad request response +func (o *GetUploadStatusBadRequest) WithPayload(payload *internalmessages.InvalidRequestResponsePayload) *GetUploadStatusBadRequest { + o.Payload = payload + return o +} + +// SetPayload sets the payload to the get upload status bad request response +func (o *GetUploadStatusBadRequest) SetPayload(payload *internalmessages.InvalidRequestResponsePayload) { + o.Payload = payload +} + +// WriteResponse to the client +func (o *GetUploadStatusBadRequest) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { + + rw.WriteHeader(400) + if o.Payload != nil { + payload := o.Payload + if err := producer.Produce(rw, payload); err != nil { + panic(err) // let the recovery middleware deal with this + } + } +} + +// GetUploadStatusForbiddenCode is the HTTP code returned for type GetUploadStatusForbidden +const GetUploadStatusForbiddenCode int = 403 + +/* +GetUploadStatusForbidden not authorized + +swagger:response getUploadStatusForbidden +*/ +type GetUploadStatusForbidden struct { +} + +// NewGetUploadStatusForbidden creates GetUploadStatusForbidden with default headers values +func NewGetUploadStatusForbidden() *GetUploadStatusForbidden { + + return &GetUploadStatusForbidden{} +} + +// WriteResponse to the client +func (o *GetUploadStatusForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { + + rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses + + rw.WriteHeader(403) +} + +// GetUploadStatusNotFoundCode is the HTTP code returned for type GetUploadStatusNotFound +const GetUploadStatusNotFoundCode int = 404 + +/* +GetUploadStatusNotFound not found + +swagger:response getUploadStatusNotFound +*/ +type GetUploadStatusNotFound struct { +} + +// NewGetUploadStatusNotFound creates GetUploadStatusNotFound with default headers values +func NewGetUploadStatusNotFound() *GetUploadStatusNotFound { + + return &GetUploadStatusNotFound{} +} + +// WriteResponse to the client +func (o *GetUploadStatusNotFound) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { + + rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses + + rw.WriteHeader(404) +} + +// GetUploadStatusInternalServerErrorCode is the HTTP code returned for type GetUploadStatusInternalServerError +const GetUploadStatusInternalServerErrorCode int = 500 + +/* +GetUploadStatusInternalServerError server error + +swagger:response getUploadStatusInternalServerError +*/ +type GetUploadStatusInternalServerError struct { +} + +// NewGetUploadStatusInternalServerError creates GetUploadStatusInternalServerError with default headers values +func NewGetUploadStatusInternalServerError() *GetUploadStatusInternalServerError { + + return &GetUploadStatusInternalServerError{} +} + +// WriteResponse to the client +func (o *GetUploadStatusInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { + + rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses + + rw.WriteHeader(500) +} diff --git a/pkg/gen/internalapi/internaloperations/uploads/get_upload_status_urlbuilder.go b/pkg/gen/internalapi/internaloperations/uploads/get_upload_status_urlbuilder.go new file mode 100644 index 00000000000..276a011d780 --- /dev/null +++ b/pkg/gen/internalapi/internaloperations/uploads/get_upload_status_urlbuilder.go @@ -0,0 +1,101 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package uploads + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the generate command + +import ( + "errors" + "net/url" + golangswaggerpaths "path" + "strings" + + "github.com/go-openapi/strfmt" +) + +// GetUploadStatusURL generates an URL for the get upload status operation +type GetUploadStatusURL struct { + UploadID strfmt.UUID + + _basePath string + // avoid unkeyed usage + _ struct{} +} + +// WithBasePath sets the base path for this url builder, only required when it's different from the +// base path specified in the swagger spec. +// When the value of the base path is an empty string +func (o *GetUploadStatusURL) WithBasePath(bp string) *GetUploadStatusURL { + o.SetBasePath(bp) + return o +} + +// SetBasePath sets the base path for this url builder, only required when it's different from the +// base path specified in the swagger spec. +// When the value of the base path is an empty string +func (o *GetUploadStatusURL) SetBasePath(bp string) { + o._basePath = bp +} + +// Build a url path and query string +func (o *GetUploadStatusURL) Build() (*url.URL, error) { + var _result url.URL + + var _path = "/uploads/{uploadId}/status" + + uploadID := o.UploadID.String() + if uploadID != "" { + _path = strings.Replace(_path, "{uploadId}", uploadID, -1) + } else { + return nil, errors.New("uploadId is required on GetUploadStatusURL") + } + + _basePath := o._basePath + if _basePath == "" { + _basePath = "/internal" + } + _result.Path = golangswaggerpaths.Join(_basePath, _path) + + return &_result, nil +} + +// Must is a helper function to panic when the url builder returns an error +func (o *GetUploadStatusURL) Must(u *url.URL, err error) *url.URL { + if err != nil { + panic(err) + } + if u == nil { + panic("url can't be nil") + } + return u +} + +// String returns the string representation of the path with query string +func (o *GetUploadStatusURL) String() string { + return o.Must(o.Build()).String() +} + +// BuildFull builds a full url with scheme, host, path and query string +func (o *GetUploadStatusURL) BuildFull(scheme, host string) (*url.URL, error) { + if scheme == "" { + return nil, errors.New("scheme is required for a full url on GetUploadStatusURL") + } + if host == "" { + return nil, errors.New("host is required for a full url on GetUploadStatusURL") + } + + base, err := o.Build() + if err != nil { + return nil, err + } + + base.Scheme = scheme + base.Host = host + return base, nil +} + +// StringFull returns the string representation of a complete url +func (o *GetUploadStatusURL) StringFull(scheme, host string) string { + return o.Must(o.BuildFull(scheme, host)).String() +} diff --git a/pkg/handlers/authentication/auth.go b/pkg/handlers/authentication/auth.go index a01f499de5e..8e59132c750 100644 --- a/pkg/handlers/authentication/auth.go +++ b/pkg/handlers/authentication/auth.go @@ -221,6 +221,7 @@ var allowedRoutes = map[string]bool{ "uploads.deleteUpload": true, "users.showLoggedInUser": true, "okta_profile.showOktaInfo": true, + "uploads.getUploadStatus": true, } // checkIfRouteIsAllowed checks to see if the route is one of the ones that should be allowed through without stricter diff --git a/pkg/handlers/config.go b/pkg/handlers/config.go index b4bb2026915..50d45ee1978 100644 --- a/pkg/handlers/config.go +++ b/pkg/handlers/config.go @@ -39,6 +39,7 @@ type HandlerConfig interface { ) http.Handler FileStorer() storage.FileStorer NotificationSender() notifications.NotificationSender + NotificationReceiver() notifications.NotificationReceiver HHGPlanner() route.Planner DTODPlanner() route.Planner CookieSecret() string @@ -66,6 +67,7 @@ type Config struct { dtodPlanner route.Planner storage storage.FileStorer notificationSender notifications.NotificationSender + notificationReceiver notifications.NotificationReceiver iwsPersonLookup iws.PersonLookup sendProductionInvoice bool senderToGex services.GexSender @@ -86,6 +88,7 @@ func NewHandlerConfig( dtodPlanner route.Planner, storage storage.FileStorer, notificationSender notifications.NotificationSender, + notificationReceiver notifications.NotificationReceiver, iwsPersonLookup iws.PersonLookup, sendProductionInvoice bool, senderToGex services.GexSender, @@ -103,6 +106,7 @@ func NewHandlerConfig( dtodPlanner: dtodPlanner, storage: storage, notificationSender: notificationSender, + notificationReceiver: notificationReceiver, iwsPersonLookup: iwsPersonLookup, sendProductionInvoice: sendProductionInvoice, senderToGex: senderToGex, @@ -247,6 +251,16 @@ func (c *Config) SetNotificationSender(sender notifications.NotificationSender) c.notificationSender = sender } +// NotificationReceiver returns the sender to use in the current context +func (c *Config) NotificationReceiver() notifications.NotificationReceiver { + return c.notificationReceiver +} + +// SetNotificationSender is a simple setter for AWS SQS private field +func (c *Config) SetNotificationReceiver(receiver notifications.NotificationReceiver) { + c.notificationReceiver = receiver +} + // SetPlanner is a simple setter for the route.Planner private field func (c *Config) SetPlanner(planner route.Planner) { c.planner = planner diff --git a/pkg/handlers/config_test.go b/pkg/handlers/config_test.go index 26595daea29..85c9ccbff7c 100644 --- a/pkg/handlers/config_test.go +++ b/pkg/handlers/config_test.go @@ -30,7 +30,7 @@ func (suite *ConfigSuite) TestConfigHandler() { appCtx := suite.AppContextForTest() sessionManagers := auth.SetupSessionManagers(nil, false, time.Duration(180*time.Second), time.Duration(180*time.Second)) - handler := NewHandlerConfig(appCtx.DB(), nil, "", nil, nil, nil, nil, nil, false, nil, nil, false, ApplicationTestServername(), sessionManagers, nil) + handler := NewHandlerConfig(appCtx.DB(), nil, "", nil, nil, nil, nil, nil, nil, false, nil, nil, false, ApplicationTestServername(), sessionManagers, nil) req, err := http.NewRequest("GET", "/", nil) suite.NoError(err) myMethodCalled := false diff --git a/pkg/handlers/internalapi/api.go b/pkg/handlers/internalapi/api.go index 8aff3ee28e4..2d3c3c38f35 100644 --- a/pkg/handlers/internalapi/api.go +++ b/pkg/handlers/internalapi/api.go @@ -173,6 +173,7 @@ func NewInternalAPI(handlerConfig handlers.HandlerConfig) *internalops.MymoveAPI internalAPI.UploadsCreateUploadHandler = CreateUploadHandler{handlerConfig} internalAPI.UploadsDeleteUploadHandler = DeleteUploadHandler{handlerConfig, upload.NewUploadInformationFetcher()} internalAPI.UploadsDeleteUploadsHandler = DeleteUploadsHandler{handlerConfig} + internalAPI.UploadsGetUploadStatusHandler = GetUploadStatusHandler{handlerConfig, upload.NewUploadInformationFetcher()} internalAPI.QueuesShowQueueHandler = ShowQueueHandler{handlerConfig} internalAPI.OfficeApproveMoveHandler = ApproveMoveHandler{handlerConfig, moveRouter} @@ -186,6 +187,7 @@ func NewInternalAPI(handlerConfig handlers.HandlerConfig) *internalops.MymoveAPI internalAPI.PpmShowAOAPacketHandler = showAOAPacketHandler{handlerConfig, SSWPPMComputer, SSWPPMGenerator, AOAPacketCreator} internalAPI.RegisterProducer(uploader.FileTypePDF, PDFProducer()) + internalAPI.TextEventStreamProducer = runtime.ByteStreamProducer() internalAPI.PostalCodesValidatePostalCodeWithRateDataHandler = ValidatePostalCodeWithRateDataHandler{ handlerConfig, diff --git a/pkg/handlers/internalapi/internal/payloads/model_to_payload.go b/pkg/handlers/internalapi/internal/payloads/model_to_payload.go index 68e9cd5b576..f24d3ea21fd 100644 --- a/pkg/handlers/internalapi/internal/payloads/model_to_payload.go +++ b/pkg/handlers/internalapi/internal/payloads/model_to_payload.go @@ -453,12 +453,19 @@ func PayloadForUploadModel( CreatedAt: strfmt.DateTime(upload.CreatedAt), UpdatedAt: strfmt.DateTime(upload.UpdatedAt), } - tags, err := storer.Tags(upload.StorageKey) - if err != nil || len(tags) == 0 { - uploadPayload.Status = "PROCESSING" + + if upload.AVStatus == nil { + tags, err := storer.Tags(upload.StorageKey) + if err != nil || len(tags) == 0 { + uploadPayload.Status = "PROCESSING" + } else { + uploadPayload.Status = tags["av-status"] + // TODO: update db with the tags + } } else { - uploadPayload.Status = tags["av-status"] + uploadPayload.Status = string(*upload.AVStatus) } + return uploadPayload } diff --git a/pkg/handlers/internalapi/uploads.go b/pkg/handlers/internalapi/uploads.go index 4167d7ed2b8..d541a550073 100644 --- a/pkg/handlers/internalapi/uploads.go +++ b/pkg/handlers/internalapi/uploads.go @@ -3,8 +3,10 @@ package internalapi import ( "fmt" "io" + "net/http" "path/filepath" "regexp" + "strconv" "strings" "github.com/go-openapi/runtime" @@ -19,9 +21,11 @@ import ( "github.com/transcom/mymove/pkg/handlers" "github.com/transcom/mymove/pkg/handlers/internalapi/internal/payloads" "github.com/transcom/mymove/pkg/models" + "github.com/transcom/mymove/pkg/notifications" "github.com/transcom/mymove/pkg/services" "github.com/transcom/mymove/pkg/services/ppmshipment" weightticketparser "github.com/transcom/mymove/pkg/services/weight_ticket_parser" + "github.com/transcom/mymove/pkg/storage" "github.com/transcom/mymove/pkg/uploader" uploaderpkg "github.com/transcom/mymove/pkg/uploader" ) @@ -246,6 +250,126 @@ func (h DeleteUploadsHandler) Handle(params uploadop.DeleteUploadsParams) middle }) } +// UploadStatusHandler returns status of an upload +type GetUploadStatusHandler struct { + handlers.HandlerConfig + services.UploadInformationFetcher +} + +type CustomNewUploadStatusOK struct { + params uploadop.GetUploadStatusParams + appCtx appcontext.AppContext + receiver notifications.NotificationReceiver + storer storage.FileStorer +} + +func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { + + // TODO: add check for permissions to view upload + + uploadId := o.params.UploadID.String() + + uploadUUID, err := uuid.FromString(uploadId) + if err != nil { + panic(err) + } + + // Check current tag before event-driven wait for anti-virus + + uploaded, err := models.FetchUserUploadFromUploadID(o.appCtx.DB(), o.appCtx.Session(), uploadUUID) + if err != nil { + o.appCtx.Logger().Error(err.Error()) + } + + tags, err := o.storer.Tags(uploaded.Upload.StorageKey) + var uploadStatus models.AVStatusType + if err != nil || len(tags) == 0 { + uploadStatus = models.AVStatusTypePROCESSING + } else { + uploadStatus = models.AVStatusType(tags["av-status"]) + } + + resProcess := []byte("id: 0\nevent: message\ndata: " + string(uploadStatus) + "\n\n") + if produceErr := producer.Produce(rw, resProcess); produceErr != nil { + panic(produceErr) + } + + if f, ok := rw.(http.Flusher); ok { + f.Flush() + } + + if uploadStatus == models.AVStatusTypeCLEAN || uploadStatus == models.AVStatusTypeINFECTED { + return + } + + // Start waiting for tag updates + + topicName := "app_s3_tag_events" + notificationParams := notifications.NotificationQueueParams{ + Action: "ObjectTagsAdded", + ObjectId: uploadId, + } + + queueUrl, err := o.receiver.CreateQueueWithSubscription(o.appCtx, topicName, notificationParams) + if err != nil { + o.appCtx.Logger().Error(err.Error()) + } + + id_counter := 0 + // Run for 120 seconds, 20 second long polling 6 times + for range 6 { + o.appCtx.Logger().Info("Receiving...") + messages, errs := o.receiver.ReceiveMessages(o.appCtx, queueUrl) + if errs != nil { + o.appCtx.Logger().Error(errs.Error()) + } + + if len(messages) != 0 { + errTransaction := o.appCtx.NewTransaction(func(txnAppCtx appcontext.AppContext) error { + + tags, err := o.storer.Tags(uploaded.Upload.StorageKey) + + if err != nil || len(tags) == 0 { + uploadStatus = models.AVStatusTypePROCESSING + } else { + uploadStatus = models.AVStatusType(tags["av-status"]) + } + + resProcess := []byte("id: " + strconv.Itoa(id_counter) + "\nevent: message\ndata: " + string(uploadStatus) + "\n\n") + if produceErr := producer.Produce(rw, resProcess); produceErr != nil { + panic(produceErr) // let the recovery middleware deal with this + } + + return nil + }) + + if errTransaction != nil { + o.appCtx.Logger().Error(err.Error()) + } + } + + if f, ok := rw.(http.Flusher); ok { + f.Flush() + } + id_counter++ + } + + // TODO: add a close here after ends +} + +// Handle returns status of an upload +func (h GetUploadStatusHandler) Handle(params uploadop.GetUploadStatusParams) middleware.Responder { + return h.AuditableAppContextFromRequestWithErrors(params.HTTPRequest, + func(appCtx appcontext.AppContext) (middleware.Responder, error) { + return &CustomNewUploadStatusOK{ + params: params, + appCtx: h.AppContextFromRequest(params.HTTPRequest), + receiver: h.NotificationReceiver(), + storer: h.FileStorer(), + }, nil + }) +} + func (h CreatePPMUploadHandler) Handle(params ppmop.CreatePPMUploadParams) middleware.Responder { return h.AuditableAppContextFromRequestWithErrors(params.HTTPRequest, func(appCtx appcontext.AppContext) (middleware.Responder, error) { diff --git a/pkg/handlers/internalapi/uploads_test.go b/pkg/handlers/internalapi/uploads_test.go index 36119617912..ebc6eb0373c 100644 --- a/pkg/handlers/internalapi/uploads_test.go +++ b/pkg/handlers/internalapi/uploads_test.go @@ -447,6 +447,50 @@ func (suite *HandlerSuite) TestDeleteUploadHandlerSuccessEvenWithS3Failure() { suite.NotNil(queriedUpload.DeletedAt) } +// TODO: functioning test +func (suite *HandlerSuite) TestGetUploadStatusHandlerSuccess() { + fakeS3 := storageTest.NewFakeS3Storage(true) + + move := factory.BuildMove(suite.DB(), nil, nil) + uploadUser1 := factory.BuildUserUpload(suite.DB(), []factory.Customization{ + { + Model: move.Orders.UploadedOrders, + LinkOnly: true, + }, + { + Model: models.Upload{ + Filename: "FileName", + Bytes: int64(15), + ContentType: uploader.FileTypePDF, + }, + }, + }, nil) + + file := suite.Fixture(FixturePDF) + fakeS3.Store(uploadUser1.Upload.StorageKey, file.Data, "somehash", nil) + + params := uploadop.NewGetUploadStatusParams() + params.UploadID = strfmt.UUID(uploadUser1.ID.String()) + + req := &http.Request{} + req = suite.AuthenticateRequest(req, uploadUser1.Document.ServiceMember) + params.HTTPRequest = req + + handlerConfig := suite.HandlerConfig() + handlerConfig.SetFileStorer(fakeS3) + uploadInformationFetcher := upload.NewUploadInformationFetcher() + handler := GetUploadStatusHandler{handlerConfig, uploadInformationFetcher} + + response := handler.Handle(params) + + _, ok := response.(*CustomNewUploadStatusOK) + suite.True(ok) + + queriedUpload := models.Upload{} + err := suite.DB().Find(&queriedUpload, uploadUser1.Upload.ID) + suite.Nil(err) +} + func (suite *HandlerSuite) TestCreatePPMUploadsHandlerSuccess() { suite.Run("uploads .xls file", func() { fakeS3 := storageTest.NewFakeS3Storage(true) diff --git a/pkg/models/upload.go b/pkg/models/upload.go index d6afc2d0d4a..0703dff29ca 100644 --- a/pkg/models/upload.go +++ b/pkg/models/upload.go @@ -25,19 +25,32 @@ const ( UploadTypeOFFICE UploadType = "OFFICE" ) +// AVStatusType represents the type of the anti-virus status, whether it is still processing, clean or infected +type AVStatusType string + +const ( + // AVStatusTypePROCESSING string PROCESSING + AVStatusTypePROCESSING AVStatusType = "PROCESSING" + // AVStatusTypeCLEAN string CLEAN + AVStatusTypeCLEAN AVStatusType = "CLEAN" + // AVStatusTypeINFECTED string INFECTED + AVStatusTypeINFECTED AVStatusType = "INFECTED" +) + // An Upload represents an uploaded file, such as an image or PDF. type Upload struct { - ID uuid.UUID `db:"id"` - Filename string `db:"filename"` - Bytes int64 `db:"bytes"` - Rotation *int64 `db:"rotation"` - ContentType string `db:"content_type"` - Checksum string `db:"checksum"` - StorageKey string `db:"storage_key"` - UploadType UploadType `db:"upload_type"` - CreatedAt time.Time `db:"created_at"` - UpdatedAt time.Time `db:"updated_at"` - DeletedAt *time.Time `db:"deleted_at"` + ID uuid.UUID `db:"id"` + Filename string `db:"filename"` + Bytes int64 `db:"bytes"` + Rotation *int64 `db:"rotation"` + ContentType string `db:"content_type"` + Checksum string `db:"checksum"` + StorageKey string `db:"storage_key"` + AVStatus *AVStatusType `db:"av_status"` + UploadType UploadType `db:"upload_type"` + CreatedAt time.Time `db:"created_at"` + UpdatedAt time.Time `db:"updated_at"` + DeletedAt *time.Time `db:"deleted_at"` } // TableName overrides the table name used by Pop. diff --git a/pkg/notifications/notification_receiver.go b/pkg/notifications/notification_receiver.go new file mode 100644 index 00000000000..7ec0ec655ac --- /dev/null +++ b/pkg/notifications/notification_receiver.go @@ -0,0 +1,192 @@ +package notifications + +import ( + "context" + "fmt" + "log" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/sns" + "github.com/aws/aws-sdk-go-v2/service/sqs" + "github.com/aws/aws-sdk-go-v2/service/sqs/types" + "github.com/spf13/viper" + "go.uber.org/zap" + + "github.com/transcom/mymove/pkg/appcontext" + "github.com/transcom/mymove/pkg/cli" +) + +// Notification is an interface for creating emails +type NotificationQueueParams struct { + // TODO: change to enum + Action string + ObjectId string +} + +// NotificationSender is an interface for sending notifications +// +//go:generate mockery --name NotificationSender +type NotificationReceiver interface { + CreateQueueWithSubscription(appCtx appcontext.AppContext, topicArn string, params NotificationQueueParams) (string, error) + ReceiveMessages(appCtx appcontext.AppContext, queueUrl string) ([]types.Message, error) +} + +// NotificationSendingContext provides context to a notification sender +type NotificationReceiverContext struct { + snsService *sns.Client + sqsService *sqs.Client + awsRegion string + awsAccountId string +} + +// NewNotificationSender returns a new NotificationSendingContext +func NewNotificationReceiver(snsService *sns.Client, sqsService *sqs.Client, awsRegion string, awsAccountId string) NotificationReceiverContext { + return NotificationReceiverContext{ + snsService: snsService, + sqsService: sqsService, + awsRegion: awsRegion, + awsAccountId: awsAccountId, + } +} + +func (n NotificationReceiverContext) CreateQueueWithSubscription(appCtx appcontext.AppContext, topicName string, params NotificationQueueParams) (string, error) { + + queueName := fmt.Sprintf("%s_%s", params.Action, params.ObjectId) + queueArn := n.constructArn("sqs", queueName) + topicArn := n.constructArn("sns", topicName) + + // Create queue + + accessPolicy := fmt.Sprintf(`{ + "Version": "2012-10-17", + "Statement": [{ + "Sid": "AllowSNSPublish", + "Effect": "Allow", + "Principal": { + "Service": "sns.amazonaws.com" + }, + "Action": ["sqs:SendMessage"], + "Resource": "%s", + "Condition": { + "ArnEquals": { + "aws:SourceArn": "%s" + } + } + }] + }`, queueArn, topicArn) + + input := &sqs.CreateQueueInput{ + QueueName: &queueName, + Attributes: map[string]string{ + "MessageRetentionPeriod": "120", + "Policy": accessPolicy, + }, + } + + result, err := n.sqsService.CreateQueue(context.Background(), input) + if err != nil { + log.Fatalf("Failed to create SQS queue, %v", err) + } + + // Create subscription + + filterPolicy := fmt.Sprintf(`{ + "detail": { + "object": { + "key": [ + {"suffix": "%s"} + ] + } + } + }`, params.ObjectId) + + subscribeInput := &sns.SubscribeInput{ + TopicArn: &topicArn, + Protocol: aws.String("sqs"), + Endpoint: &queueArn, + Attributes: map[string]string{ + "FilterPolicy": filterPolicy, + "FilterPolicyScope": "MessageBody", + }, + } + _, err = n.snsService.Subscribe(context.Background(), subscribeInput) + if err != nil { + log.Fatalf("Failed to create subscription, %v", err) + } + + return *result.QueueUrl, err +} + +// SendNotification sends a one or more notifications for all supported mediums +func (n NotificationReceiverContext) ReceiveMessages(appCtx appcontext.AppContext, queueUrl string) ([]types.Message, error) { + result, err := n.sqsService.ReceiveMessage(context.Background(), &sqs.ReceiveMessageInput{ + QueueUrl: &queueUrl, + MaxNumberOfMessages: 1, + WaitTimeSeconds: 20, + }) + if err != nil { + appCtx.Logger().Fatal("Couldn't get messages from queue. Here's why: %v\n", zap.Error(err)) + } + return result.Messages, err +} + +// InitEmail initializes the email backend +func InitReceiver(v *viper.Viper, logger *zap.Logger) (NotificationReceiver, error) { + // if v.GetString(cli.EmailBackendFlag) == "ses" { + // // Setup Amazon SES (email) service TODO: This might be able + // // to be combined with the AWS Session that we're using for S3 + // // down below. + + // awsSESRegion := v.GetString(cli.AWSSESRegionFlag) + // awsSESDomain := v.GetString(cli.AWSSESDomainFlag) + // sysAdminEmail := v.GetString(cli.SysAdminEmail) + // logger.Info("Using ses email backend", + // zap.String("region", awsSESRegion), + // zap.String("domain", awsSESDomain)) + // cfg, err := config.LoadDefaultConfig(context.Background(), + // config.WithRegion(awsSESRegion), + // ) + // if err != nil { + // logger.Fatal("error loading ses aws config", zap.Error(err)) + // } + + // sesService := ses.NewFromConfig(cfg) + // input := &ses.GetAccountSendingEnabledInput{} + // result, err := sesService.GetAccountSendingEnabled(context.Background(), input) + // if err != nil || result == nil || !result.Enabled { + // logger.Error("email sending not enabled", zap.Error(err)) + // return NewNotificationSender(nil, awsSESDomain, sysAdminEmail), err + // } + // return NewNotificationSender(sesService, awsSESDomain, sysAdminEmail), nil + // } + + // domain := "milmovelocal" + // logger.Info("Using local email backend", zap.String("domain", domain)) + // return NewStubNotificationSender(domain), nil + + // Setup Amazon SES (email) service TODO: This might be able + // to be combined with the AWS Session that we're using for S3 + // down below. + + // TODO: verify if we should change this param name to awsNotificationRegion + awsSESRegion := v.GetString(cli.AWSSESRegionFlag) + awsAccountId := v.GetString("aws-account-id") + + cfg, err := config.LoadDefaultConfig(context.Background(), + config.WithRegion(awsSESRegion), + ) + if err != nil { + logger.Fatal("error loading ses aws config", zap.Error(err)) + return nil, err + } + + snsService := sns.NewFromConfig(cfg) + sqsService := sqs.NewFromConfig(cfg) + + return NewNotificationReceiver(snsService, sqsService, awsSESRegion, awsAccountId), nil +} + +func (n NotificationReceiverContext) constructArn(awsService string, endpointName string) string { + return fmt.Sprintf("arn:aws-us-gov:%s:%s:%s:%s", awsService, n.awsRegion, n.awsAccountId, endpointName) +} diff --git a/swagger-def/internal.yaml b/swagger-def/internal.yaml index a8c8dfb732c..305d4b845b9 100644 --- a/swagger-def/internal.yaml +++ b/swagger-def/internal.yaml @@ -3426,6 +3426,43 @@ paths: description: not found '500': description: server error + + /uploads/{uploadId}/status: + get: + summary: Returns status of an upload + description: Returns status of an upload based on antivirus run + operationId: getUploadStatus + produces: + - text/event-stream + tags: + - uploads + parameters: + - in: path + name: uploadId + type: string + format: uuid + required: true + description: UUID of the upload to return status of + responses: + '200': + description: the requested upload status + schema: + type: string + enum: + - INFECTED + - CLEAN + - PROCESSING + readOnly: true + '400': + description: invalid request + schema: + $ref: '#/definitions/InvalidRequestResponsePayload' + '403': + description: not authorized + '404': + description: not found + '500': + description: server error /service_members: post: summary: Creates service member for a logged-in user diff --git a/swagger/internal.yaml b/swagger/internal.yaml index 84097cd100a..21483825daa 100644 --- a/swagger/internal.yaml +++ b/swagger/internal.yaml @@ -5335,6 +5335,42 @@ paths: description: not found '500': description: server error + /uploads/{uploadId}/status: + get: + summary: Returns status of an upload + description: Returns status of an upload based on antivirus run + operationId: getUploadStatus + produces: + - text/event-stream + tags: + - uploads + parameters: + - in: path + name: uploadId + type: string + format: uuid + required: true + description: UUID of the upload to return status of + responses: + '200': + description: the requested upload status + schema: + type: string + enum: + - INFECTED + - CLEAN + - PROCESSING + readOnly: true + '400': + description: invalid request + schema: + $ref: '#/definitions/InvalidRequestResponsePayload' + '403': + description: not authorized + '404': + description: not found + '500': + description: server error /service_members: post: summary: Creates service member for a logged-in user From 24daa1a0163a4728d492315a9baa1220a2a63bd0 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Wed, 8 Jan 2025 19:22:11 +0000 Subject: [PATCH 02/36] B-22056 - checkin tests and updates from other branch. --- cmd/milmove/serve.go | 7 +- pkg/cli/receiver.go | 53 ++++ pkg/cli/receiver_test.go | 6 + pkg/handlers/apitests.go | 12 +- .../internal/payloads/model_to_payload.go | 14 +- pkg/handlers/internalapi/uploads.go | 163 ++++++++---- pkg/handlers/internalapi/uploads_test.go | 53 +++- pkg/handlers/routing/base_routing_suite.go | 1 + .../routing/internalapi_test/uploads_test.go | 41 +++ pkg/models/upload.go | 35 +-- .../mocks/NotificationReceiver.go | 133 ++++++++++ pkg/notifications/notification_receiver.go | 248 +++++++++++------- .../notification_receiver_stub.go | 49 ++++ .../notification_receiver_test.go | 158 +++++++++++ ...on_stub.go => notification_sender_stub.go} | 0 ...on_test.go => notification_sender_test.go} | 0 pkg/storage/filesystem.go | 2 + pkg/storage/memory.go | 2 + pkg/storage/test/s3.go | 3 +- 19 files changed, 781 insertions(+), 199 deletions(-) create mode 100644 pkg/cli/receiver.go create mode 100644 pkg/cli/receiver_test.go create mode 100644 pkg/handlers/routing/internalapi_test/uploads_test.go create mode 100644 pkg/notifications/mocks/NotificationReceiver.go create mode 100644 pkg/notifications/notification_receiver_stub.go create mode 100644 pkg/notifications/notification_receiver_test.go rename pkg/notifications/{notification_stub.go => notification_sender_stub.go} (100%) rename pkg/notifications/{notification_test.go => notification_sender_test.go} (100%) diff --git a/cmd/milmove/serve.go b/cmd/milmove/serve.go index 7168bf87acd..8e9d8878d82 100644 --- a/cmd/milmove/serve.go +++ b/cmd/milmove/serve.go @@ -478,11 +478,8 @@ func buildRoutingConfig(appCtx appcontext.AppContext, v *viper.Viper, redisPool appCtx.Logger().Fatal("notification sender sending not enabled", zap.Error(err)) } - // Event Receiver - notificationReceiver, err := notifications.InitReceiver(v, appCtx.Logger()) - if err != nil { - appCtx.Logger().Fatal("notification receiver listening not enabled") - } + // Email + notificationReceiver, _ := notifications.InitReceiver(v, appCtx.Logger()) routingConfig.BuildRoot = v.GetString(cli.BuildRootFlag) sendProductionInvoice := v.GetBool(cli.GEXSendProdInvoiceFlag) diff --git a/pkg/cli/receiver.go b/pkg/cli/receiver.go new file mode 100644 index 00000000000..91f6f30f872 --- /dev/null +++ b/pkg/cli/receiver.go @@ -0,0 +1,53 @@ +package cli + +import ( + "fmt" + + "github.com/spf13/pflag" + "github.com/spf13/viper" +) + +const ( + // ReceiverBackend is the Receiver Backend Flag + ReceiverBackendFlag string = "receiver-backend" + // AWSSNSObjectTagsAddedTopic is the AWS SNS Object Tags Added Topic Flag + AWSSNSObjectTagsAddedTopicFlag string = "aws-sns-object-tags-added-topic" + // AWSS3RegionFlag is the AWS SNS Region Flag + AWSSNSRegionFlag string = "aws-sns-region" + // AWSSNSAccountId is the application's AWS account id + AWSSNSAccountId string = "aws-account-id" +) + +// InitReceiverFlags initializes Storage command line flags +func InitReceiverFlags(flag *pflag.FlagSet) { + flag.String(ReceiverBackendFlag, "local", "Receiver backend to use, either local or sns&sqs.") + flag.String(AWSSNSObjectTagsAddedTopicFlag, "", "SNS Topic for receiving event messages") + flag.String(AWSSNSRegionFlag, "", "AWS region used for SNS and SQS") + flag.String(AWSSNSAccountId, "", "AWS account Id") +} + +// CheckReceiver validates Storage command line flags +func CheckReceiver(v *viper.Viper) error { + + receiverBackend := v.GetString(ReceiverBackendFlag) + if !stringSliceContains([]string{"local", "sns&sqs"}, receiverBackend) { + return fmt.Errorf("invalid receiver-backend %s, expecting local or sns&sqs", receiverBackend) + } + + if receiverBackend == "sns&sqs" { + r := v.GetString(AWSSNSRegionFlag) + if r == "" { + return fmt.Errorf("invalid value for %s: %s", AWSSNSRegionFlag, r) + } + topic := v.GetString(AWSSNSObjectTagsAddedTopicFlag) + if topic == "" { + return fmt.Errorf("invalid value for %s: %s", AWSSNSObjectTagsAddedTopicFlag, topic) + } + accountId := v.GetString(AWSSNSAccountId) + if topic == "" { + return fmt.Errorf("invalid value for %s: %s", AWSSNSAccountId, accountId) + } + } + + return nil +} diff --git a/pkg/cli/receiver_test.go b/pkg/cli/receiver_test.go new file mode 100644 index 00000000000..7095a672f5f --- /dev/null +++ b/pkg/cli/receiver_test.go @@ -0,0 +1,6 @@ +package cli + +func (suite *cliTestSuite) TestConfigReceiver() { + suite.Setup(InitReceiverFlags, []string{}) + suite.NoError(CheckReceiver(suite.viper)) +} diff --git a/pkg/handlers/apitests.go b/pkg/handlers/apitests.go index a84a6627f2c..a540d37e1f3 100644 --- a/pkg/handlers/apitests.go +++ b/pkg/handlers/apitests.go @@ -9,6 +9,7 @@ import ( "path" "path/filepath" "runtime/debug" + "strings" "time" "github.com/go-openapi/runtime" @@ -148,6 +149,11 @@ func (suite *BaseHandlerTestSuite) TestNotificationSender() notifications.Notifi return suite.notificationSender } +// TestNotificationReceiver returns the notification sender to use in the suite +func (suite *BaseHandlerTestSuite) TestNotificationReceiver() notifications.NotificationReceiver { + return notifications.NewStubNotificationReceiver() +} + // HasWebhookNotification checks that there's a record on the WebhookNotifications table for the object and trace IDs func (suite *BaseHandlerTestSuite) HasWebhookNotification(objectID uuid.UUID, traceID uuid.UUID) { notification := &models.WebhookNotification{} @@ -277,8 +283,12 @@ func (suite *BaseHandlerTestSuite) Fixture(name string) *runtime.File { if err != nil { suite.T().Error(err) } + cdRouting := "" + if strings.Contains(cwd, "routing") { + cdRouting = ".." + } - fixturePath := path.Join(cwd, "..", "..", fixtureDir, name) + fixturePath := path.Join(cwd, "..", "..", cdRouting, fixtureDir, name) file, err := os.Open(filepath.Clean(fixturePath)) if err != nil { diff --git a/pkg/handlers/internalapi/internal/payloads/model_to_payload.go b/pkg/handlers/internalapi/internal/payloads/model_to_payload.go index f24d3ea21fd..9550b4a11f9 100644 --- a/pkg/handlers/internalapi/internal/payloads/model_to_payload.go +++ b/pkg/handlers/internalapi/internal/payloads/model_to_payload.go @@ -454,16 +454,12 @@ func PayloadForUploadModel( UpdatedAt: strfmt.DateTime(upload.UpdatedAt), } - if upload.AVStatus == nil { - tags, err := storer.Tags(upload.StorageKey) - if err != nil || len(tags) == 0 { - uploadPayload.Status = "PROCESSING" - } else { - uploadPayload.Status = tags["av-status"] - // TODO: update db with the tags - } + tags, err := storer.Tags(upload.StorageKey) + if err != nil || len(tags) == 0 { + uploadPayload.Status = "PROCESSING" } else { - uploadPayload.Status = string(*upload.AVStatus) + uploadPayload.Status = tags["av-status"] + // TODO: update db with the tags } return uploadPayload diff --git a/pkg/handlers/internalapi/uploads.go b/pkg/handlers/internalapi/uploads.go index d541a550073..834d2124d43 100644 --- a/pkg/handlers/internalapi/uploads.go +++ b/pkg/handlers/internalapi/uploads.go @@ -1,6 +1,7 @@ package internalapi import ( + "context" "fmt" "io" "net/http" @@ -13,6 +14,7 @@ import ( "github.com/go-openapi/runtime/middleware" "github.com/gobuffalo/validate/v3" "github.com/gofrs/uuid" + "github.com/pkg/errors" "go.uber.org/zap" "github.com/transcom/mymove/pkg/appcontext" @@ -257,115 +259,168 @@ type GetUploadStatusHandler struct { } type CustomNewUploadStatusOK struct { - params uploadop.GetUploadStatusParams - appCtx appcontext.AppContext - receiver notifications.NotificationReceiver - storer storage.FileStorer + params uploadop.GetUploadStatusParams + storageKey string + appCtx appcontext.AppContext + receiver notifications.NotificationReceiver + storer storage.FileStorer } -func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - // TODO: add check for permissions to view upload +// AVStatusType represents the type of the anti-virus status, whether it is still processing, clean or infected +type AVStatusType string - uploadId := o.params.UploadID.String() +const ( + // AVStatusTypePROCESSING string PROCESSING + AVStatusTypePROCESSING AVStatusType = "PROCESSING" + // AVStatusTypeCLEAN string CLEAN + AVStatusTypeCLEAN AVStatusType = "CLEAN" + // AVStatusTypeINFECTED string INFECTED + AVStatusTypeINFECTED AVStatusType = "INFECTED" +) - uploadUUID, err := uuid.FromString(uploadId) - if err != nil { - panic(err) +func writeEventStreamMessage(rw http.ResponseWriter, producer runtime.Producer, id int, event string, data string) { + resProcess := []byte(fmt.Sprintf("id: %s\nevent: %s\ndata: %s\n\n", strconv.Itoa(id), event, data)) + if produceErr := producer.Produce(rw, resProcess); produceErr != nil { + panic(produceErr) } - - // Check current tag before event-driven wait for anti-virus - - uploaded, err := models.FetchUserUploadFromUploadID(o.appCtx.DB(), o.appCtx.Session(), uploadUUID) - if err != nil { - o.appCtx.Logger().Error(err.Error()) + if f, ok := rw.(http.Flusher); ok { + f.Flush() } +} + +func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - tags, err := o.storer.Tags(uploaded.Upload.StorageKey) - var uploadStatus models.AVStatusType + // Check current tag before event-driven wait for anti-virus + tags, err := o.storer.Tags(o.storageKey) + var uploadStatus AVStatusType if err != nil || len(tags) == 0 { - uploadStatus = models.AVStatusTypePROCESSING + uploadStatus = AVStatusTypePROCESSING } else { - uploadStatus = models.AVStatusType(tags["av-status"]) + uploadStatus = AVStatusType(tags["av-status"]) } - resProcess := []byte("id: 0\nevent: message\ndata: " + string(uploadStatus) + "\n\n") - if produceErr := producer.Produce(rw, resProcess); produceErr != nil { - panic(produceErr) - } + writeEventStreamMessage(rw, producer, 0, "message", string(uploadStatus)) - if f, ok := rw.(http.Flusher); ok { - f.Flush() + if uploadStatus == AVStatusTypeCLEAN || uploadStatus == AVStatusTypeINFECTED { + writeEventStreamMessage(rw, producer, 1, "close", "Connection closed") + return // skip notification loop since object already tagged from anti-virus } - if uploadStatus == models.AVStatusTypeCLEAN || uploadStatus == models.AVStatusTypeINFECTED { + // Start waiting for tag updates + topicName, err := o.receiver.GetDefaultTopic() + if err != nil { + o.appCtx.Logger().Error("aws_sns_object_tags_added_topic key not available.") return } - // Start waiting for tag updates + filterPolicy := fmt.Sprintf(`{ + "detail": { + "object": { + "key": [ + {"suffix": "%s"} + ] + } + } + }`, o.params.UploadID) - topicName := "app_s3_tag_events" notificationParams := notifications.NotificationQueueParams{ - Action: "ObjectTagsAdded", - ObjectId: uploadId, + SubscriptionTopicName: topicName, + NamePrefix: "ObjectTagsAdded", + FilterPolicy: filterPolicy, } - queueUrl, err := o.receiver.CreateQueueWithSubscription(o.appCtx, topicName, notificationParams) + queueUrl, err := o.receiver.CreateQueueWithSubscription(o.appCtx, notificationParams) if err != nil { o.appCtx.Logger().Error(err.Error()) } - id_counter := 0 - // Run for 120 seconds, 20 second long polling 6 times + // Cleanup + go func() { + <-o.params.HTTPRequest.Context().Done() + _ = o.receiver.CloseoutQueue(o.appCtx, queueUrl) + }() + + id_counter := 1 + // Run for 120 seconds, 20 second long polling for receiver, 6 times for range 6 { o.appCtx.Logger().Info("Receiving...") messages, errs := o.receiver.ReceiveMessages(o.appCtx, queueUrl) - if errs != nil { + if errs != nil && errs != context.Canceled { o.appCtx.Logger().Error(errs.Error()) } + if errs == context.Canceled { + break + } + if len(messages) != 0 { errTransaction := o.appCtx.NewTransaction(func(txnAppCtx appcontext.AppContext) error { - tags, err := o.storer.Tags(uploaded.Upload.StorageKey) + tags, err := o.storer.Tags(o.storageKey) if err != nil || len(tags) == 0 { - uploadStatus = models.AVStatusTypePROCESSING + uploadStatus = AVStatusTypePROCESSING } else { - uploadStatus = models.AVStatusType(tags["av-status"]) + uploadStatus = AVStatusType(tags["av-status"]) } - resProcess := []byte("id: " + strconv.Itoa(id_counter) + "\nevent: message\ndata: " + string(uploadStatus) + "\n\n") - if produceErr := producer.Produce(rw, resProcess); produceErr != nil { - panic(produceErr) // let the recovery middleware deal with this + writeEventStreamMessage(rw, producer, id_counter, "message", string(uploadStatus)) + + if uploadStatus == AVStatusTypeCLEAN || uploadStatus == AVStatusTypeINFECTED { + return errors.New("connection_closed") } - return nil + return err }) - if errTransaction != nil { - o.appCtx.Logger().Error(err.Error()) + if errTransaction != nil && errTransaction.Error() == "connection_closed" { + id_counter++ + writeEventStreamMessage(rw, producer, id_counter, "close", "Connection closed") + break } - } - if f, ok := rw.(http.Flusher); ok { - f.Flush() + if errTransaction != nil { + panic(errTransaction) // let the recovery middleware deal with this + } } id_counter++ } - - // TODO: add a close here after ends } // Handle returns status of an upload func (h GetUploadStatusHandler) Handle(params uploadop.GetUploadStatusParams) middleware.Responder { return h.AuditableAppContextFromRequestWithErrors(params.HTTPRequest, func(appCtx appcontext.AppContext) (middleware.Responder, error) { + + handleError := func(err error) (middleware.Responder, error) { + appCtx.Logger().Error("GetUploadStatusHandler error", zap.Error(err)) + switch errors.Cause(err) { + case models.ErrFetchForbidden: + return uploadop.NewGetUploadStatusForbidden(), err + case models.ErrFetchNotFound: + return uploadop.NewGetUploadStatusNotFound(), err + default: + return uploadop.NewGetUploadStatusInternalServerError(), err + } + } + + uploadId := params.UploadID.String() + uploadUUID, err := uuid.FromString(uploadId) + if err != nil { + return handleError(err) + } + + uploaded, err := models.FetchUserUploadFromUploadID(appCtx.DB(), appCtx.Session(), uploadUUID) + if err != nil { + return handleError(err) + } + return &CustomNewUploadStatusOK{ - params: params, - appCtx: h.AppContextFromRequest(params.HTTPRequest), - receiver: h.NotificationReceiver(), - storer: h.FileStorer(), + params: params, + storageKey: uploaded.Upload.StorageKey, + appCtx: h.AppContextFromRequest(params.HTTPRequest), + receiver: h.NotificationReceiver(), + storer: h.FileStorer(), }, nil }) } diff --git a/pkg/handlers/internalapi/uploads_test.go b/pkg/handlers/internalapi/uploads_test.go index ebc6eb0373c..143dfa465eb 100644 --- a/pkg/handlers/internalapi/uploads_test.go +++ b/pkg/handlers/internalapi/uploads_test.go @@ -24,6 +24,7 @@ import ( uploadop "github.com/transcom/mymove/pkg/gen/internalapi/internaloperations/uploads" "github.com/transcom/mymove/pkg/handlers" "github.com/transcom/mymove/pkg/models" + "github.com/transcom/mymove/pkg/notifications" paperworkgenerator "github.com/transcom/mymove/pkg/paperwork" "github.com/transcom/mymove/pkg/services/upload" weightticketparser "github.com/transcom/mymove/pkg/services/weight_ticket_parser" @@ -109,6 +110,7 @@ func createPPMExpensePrereqs(suite *HandlerSuite, fixtureFile string) (models.Do func makeRequest(suite *HandlerSuite, params uploadop.CreateUploadParams, serviceMember models.ServiceMember, fakeS3 *storageTest.FakeS3Storage) middleware.Responder { req := &http.Request{} + req = suite.AuthenticateRequest(req, serviceMember) params.HTTPRequest = req @@ -447,14 +449,14 @@ func (suite *HandlerSuite) TestDeleteUploadHandlerSuccessEvenWithS3Failure() { suite.NotNil(queriedUpload.DeletedAt) } -// TODO: functioning test func (suite *HandlerSuite) TestGetUploadStatusHandlerSuccess() { fakeS3 := storageTest.NewFakeS3Storage(true) + localReceiver := notifications.StubNotificationReceiver{} - move := factory.BuildMove(suite.DB(), nil, nil) + orders := factory.BuildOrder(suite.DB(), nil, nil) uploadUser1 := factory.BuildUserUpload(suite.DB(), []factory.Customization{ { - Model: move.Orders.UploadedOrders, + Model: orders.UploadedOrders, LinkOnly: true, }, { @@ -467,10 +469,11 @@ func (suite *HandlerSuite) TestGetUploadStatusHandlerSuccess() { }, nil) file := suite.Fixture(FixturePDF) - fakeS3.Store(uploadUser1.Upload.StorageKey, file.Data, "somehash", nil) + _, err := fakeS3.Store(uploadUser1.Upload.StorageKey, file.Data, "somehash", nil) + suite.NoError(err) params := uploadop.NewGetUploadStatusParams() - params.UploadID = strfmt.UUID(uploadUser1.ID.String()) + params.UploadID = strfmt.UUID(uploadUser1.Upload.ID.String()) req := &http.Request{} req = suite.AuthenticateRequest(req, uploadUser1.Document.ServiceMember) @@ -478,17 +481,51 @@ func (suite *HandlerSuite) TestGetUploadStatusHandlerSuccess() { handlerConfig := suite.HandlerConfig() handlerConfig.SetFileStorer(fakeS3) + handlerConfig.SetNotificationReceiver(localReceiver) uploadInformationFetcher := upload.NewUploadInformationFetcher() handler := GetUploadStatusHandler{handlerConfig, uploadInformationFetcher} response := handler.Handle(params) - _, ok := response.(*CustomNewUploadStatusOK) suite.True(ok) queriedUpload := models.Upload{} - err := suite.DB().Find(&queriedUpload, uploadUser1.Upload.ID) - suite.Nil(err) + err = suite.DB().Find(&queriedUpload, uploadUser1.Upload.ID) + suite.NoError(err) +} + +func (suite *HandlerSuite) TestGetUploadStatusHandlerFailure() { + suite.Run("Error on no match for uploadId", func() { + orders := factory.BuildOrder(suite.DB(), factory.GetTraitActiveServiceMemberUser(), nil) + + uploadUUID := uuid.Must(uuid.NewV4()) + + params := uploadop.NewGetUploadStatusParams() + params.UploadID = strfmt.UUID(uploadUUID.String()) + + req := &http.Request{} + req = suite.AuthenticateRequest(req, orders.ServiceMember) + params.HTTPRequest = req + + fakeS3 := storageTest.NewFakeS3Storage(true) + localReceiver := notifications.StubNotificationReceiver{} + + handlerConfig := suite.HandlerConfig() + handlerConfig.SetFileStorer(fakeS3) + handlerConfig.SetNotificationReceiver(localReceiver) + uploadInformationFetcher := upload.NewUploadInformationFetcher() + handler := GetUploadStatusHandler{handlerConfig, uploadInformationFetcher} + + response := handler.Handle(params) + _, ok := response.(*uploadop.GetUploadStatusNotFound) + suite.True(ok) + + queriedUpload := models.Upload{} + err := suite.DB().Find(&queriedUpload, uploadUUID) + suite.Error(err) + }) + + // TODO: ADD A FORBIDDEN TEST } func (suite *HandlerSuite) TestCreatePPMUploadsHandlerSuccess() { diff --git a/pkg/handlers/routing/base_routing_suite.go b/pkg/handlers/routing/base_routing_suite.go index 23e538792b7..77049e33664 100644 --- a/pkg/handlers/routing/base_routing_suite.go +++ b/pkg/handlers/routing/base_routing_suite.go @@ -85,6 +85,7 @@ func (suite *BaseRoutingSuite) RoutingConfig() *Config { handlerConfig := suite.BaseHandlerTestSuite.HandlerConfig() handlerConfig.SetAppNames(handlers.ApplicationTestServername()) handlerConfig.SetNotificationSender(suite.TestNotificationSender()) + handlerConfig.SetNotificationReceiver(suite.TestNotificationReceiver()) // Need this for any requests that will either retrieve or save files or their info. fakeS3 := storageTest.NewFakeS3Storage(true) diff --git a/pkg/handlers/routing/internalapi_test/uploads_test.go b/pkg/handlers/routing/internalapi_test/uploads_test.go new file mode 100644 index 00000000000..3fe89e8927d --- /dev/null +++ b/pkg/handlers/routing/internalapi_test/uploads_test.go @@ -0,0 +1,41 @@ +package internalapi_test + +import ( + "net/http" + "net/http/httptest" + + "github.com/transcom/mymove/pkg/factory" + "github.com/transcom/mymove/pkg/models" + "github.com/transcom/mymove/pkg/uploader" +) + +func (suite *InternalAPISuite) TestUploads() { + suite.Run("Received message for upload", func() { + orders := factory.BuildOrder(suite.DB(), factory.GetTraitActiveServiceMemberUser(), nil) + uploadUser1 := factory.BuildUserUpload(suite.DB(), []factory.Customization{ + { + Model: orders.UploadedOrders, + LinkOnly: true, + }, + { + Model: models.Upload{ + Filename: "FileName", + Bytes: int64(15), + ContentType: uploader.FileTypePDF, + }, + }, + }, nil) + file := suite.Fixture("test.pdf") + _, err := suite.HandlerConfig().FileStorer().Store(uploadUser1.Upload.StorageKey, file.Data, "somehash", nil) + suite.NoError(err) + + req := suite.NewAuthenticatedMilRequest("GET", "/internal/uploads/"+uploadUser1.Upload.ID.String()+"/status", nil, orders.ServiceMember) + rr := httptest.NewRecorder() + + suite.SetupSiteHandler().ServeHTTP(rr, req) + + suite.Equal(http.StatusOK, rr.Code) + suite.Equal("text/event-stream", rr.Header().Get("content-type")) + suite.Equal("id: 0\nevent: message\ndata: CLEAN\n\nid: 1\nevent: close\ndata: Connection closed\n\n", rr.Body.String()) + }) +} diff --git a/pkg/models/upload.go b/pkg/models/upload.go index 0703dff29ca..d6afc2d0d4a 100644 --- a/pkg/models/upload.go +++ b/pkg/models/upload.go @@ -25,32 +25,19 @@ const ( UploadTypeOFFICE UploadType = "OFFICE" ) -// AVStatusType represents the type of the anti-virus status, whether it is still processing, clean or infected -type AVStatusType string - -const ( - // AVStatusTypePROCESSING string PROCESSING - AVStatusTypePROCESSING AVStatusType = "PROCESSING" - // AVStatusTypeCLEAN string CLEAN - AVStatusTypeCLEAN AVStatusType = "CLEAN" - // AVStatusTypeINFECTED string INFECTED - AVStatusTypeINFECTED AVStatusType = "INFECTED" -) - // An Upload represents an uploaded file, such as an image or PDF. type Upload struct { - ID uuid.UUID `db:"id"` - Filename string `db:"filename"` - Bytes int64 `db:"bytes"` - Rotation *int64 `db:"rotation"` - ContentType string `db:"content_type"` - Checksum string `db:"checksum"` - StorageKey string `db:"storage_key"` - AVStatus *AVStatusType `db:"av_status"` - UploadType UploadType `db:"upload_type"` - CreatedAt time.Time `db:"created_at"` - UpdatedAt time.Time `db:"updated_at"` - DeletedAt *time.Time `db:"deleted_at"` + ID uuid.UUID `db:"id"` + Filename string `db:"filename"` + Bytes int64 `db:"bytes"` + Rotation *int64 `db:"rotation"` + ContentType string `db:"content_type"` + Checksum string `db:"checksum"` + StorageKey string `db:"storage_key"` + UploadType UploadType `db:"upload_type"` + CreatedAt time.Time `db:"created_at"` + UpdatedAt time.Time `db:"updated_at"` + DeletedAt *time.Time `db:"deleted_at"` } // TableName overrides the table name used by Pop. diff --git a/pkg/notifications/mocks/NotificationReceiver.go b/pkg/notifications/mocks/NotificationReceiver.go new file mode 100644 index 00000000000..df8329e5f60 --- /dev/null +++ b/pkg/notifications/mocks/NotificationReceiver.go @@ -0,0 +1,133 @@ +// Code generated by mockery. DO NOT EDIT. + +package mocks + +import ( + mock "github.com/stretchr/testify/mock" + appcontext "github.com/transcom/mymove/pkg/appcontext" + + notifications "github.com/transcom/mymove/pkg/notifications" +) + +// NotificationReceiver is an autogenerated mock type for the NotificationReceiver type +type NotificationReceiver struct { + mock.Mock +} + +// CloseoutQueue provides a mock function with given fields: appCtx, queueUrl +func (_m *NotificationReceiver) CloseoutQueue(appCtx appcontext.AppContext, queueUrl string) error { + ret := _m.Called(appCtx, queueUrl) + + if len(ret) == 0 { + panic("no return value specified for CloseoutQueue") + } + + var r0 error + if rf, ok := ret.Get(0).(func(appcontext.AppContext, string) error); ok { + r0 = rf(appCtx, queueUrl) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// CreateQueueWithSubscription provides a mock function with given fields: appCtx, params +func (_m *NotificationReceiver) CreateQueueWithSubscription(appCtx appcontext.AppContext, params notifications.NotificationQueueParams) (string, error) { + ret := _m.Called(appCtx, params) + + if len(ret) == 0 { + panic("no return value specified for CreateQueueWithSubscription") + } + + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func(appcontext.AppContext, notifications.NotificationQueueParams) (string, error)); ok { + return rf(appCtx, params) + } + if rf, ok := ret.Get(0).(func(appcontext.AppContext, notifications.NotificationQueueParams) string); ok { + r0 = rf(appCtx, params) + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func(appcontext.AppContext, notifications.NotificationQueueParams) error); ok { + r1 = rf(appCtx, params) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetDefaultTopic provides a mock function with given fields: +func (_m *NotificationReceiver) GetDefaultTopic() (string, error) { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for GetDefaultTopic") + } + + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func() (string, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ReceiveMessages provides a mock function with given fields: appCtx, queueUrl +func (_m *NotificationReceiver) ReceiveMessages(appCtx appcontext.AppContext, queueUrl string) ([]notifications.ReceivedMessage, error) { + ret := _m.Called(appCtx, queueUrl) + + if len(ret) == 0 { + panic("no return value specified for ReceiveMessages") + } + + var r0 []notifications.ReceivedMessage + var r1 error + if rf, ok := ret.Get(0).(func(appcontext.AppContext, string) ([]notifications.ReceivedMessage, error)); ok { + return rf(appCtx, queueUrl) + } + if rf, ok := ret.Get(0).(func(appcontext.AppContext, string) []notifications.ReceivedMessage); ok { + r0 = rf(appCtx, queueUrl) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]notifications.ReceivedMessage) + } + } + + if rf, ok := ret.Get(1).(func(appcontext.AppContext, string) error); ok { + r1 = rf(appCtx, queueUrl) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// NewNotificationReceiver creates a new instance of NotificationReceiver. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewNotificationReceiver(t interface { + mock.TestingT + Cleanup(func()) +}) *NotificationReceiver { + mock := &NotificationReceiver{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/pkg/notifications/notification_receiver.go b/pkg/notifications/notification_receiver.go index 7ec0ec655ac..b1c95495bc7 100644 --- a/pkg/notifications/notification_receiver.go +++ b/pkg/notifications/notification_receiver.go @@ -2,61 +2,93 @@ package notifications import ( "context" + "errors" "fmt" "log" + "strings" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/sns" "github.com/aws/aws-sdk-go-v2/service/sqs" - "github.com/aws/aws-sdk-go-v2/service/sqs/types" - "github.com/spf13/viper" + "github.com/gofrs/uuid" "go.uber.org/zap" "github.com/transcom/mymove/pkg/appcontext" "github.com/transcom/mymove/pkg/cli" ) -// Notification is an interface for creating emails +// NotificationQueueParams stores the params for queue creation type NotificationQueueParams struct { - // TODO: change to enum - Action string - ObjectId string + SubscriptionTopicName string + NamePrefix string + FilterPolicy string } -// NotificationSender is an interface for sending notifications +// NotificationReceiver is an interface for receiving notifications // -//go:generate mockery --name NotificationSender +//go:generate mockery --name NotificationReceiver type NotificationReceiver interface { - CreateQueueWithSubscription(appCtx appcontext.AppContext, topicArn string, params NotificationQueueParams) (string, error) - ReceiveMessages(appCtx appcontext.AppContext, queueUrl string) ([]types.Message, error) + CreateQueueWithSubscription(appCtx appcontext.AppContext, params NotificationQueueParams) (string, error) + ReceiveMessages(appCtx appcontext.AppContext, queueUrl string) ([]ReceivedMessage, error) + CloseoutQueue(appCtx appcontext.AppContext, queueUrl string) error + GetDefaultTopic() (string, error) } -// NotificationSendingContext provides context to a notification sender +// NotificationReceiverConext provides context to a notification Receiver. Maps use queueUrl for key type NotificationReceiverContext struct { - snsService *sns.Client - sqsService *sqs.Client - awsRegion string - awsAccountId string + viper ViperType + snsService SnsClient + sqsService SqsClient + awsRegion string + awsAccountId string + queueSubscriptionMap map[string]string + receiverCancelMap map[string]context.CancelFunc } -// NewNotificationSender returns a new NotificationSendingContext -func NewNotificationReceiver(snsService *sns.Client, sqsService *sqs.Client, awsRegion string, awsAccountId string) NotificationReceiverContext { +type SnsClient interface { + Subscribe(ctx context.Context, params *sns.SubscribeInput, optFns ...func(*sns.Options)) (*sns.SubscribeOutput, error) + Unsubscribe(ctx context.Context, params *sns.UnsubscribeInput, optFns ...func(*sns.Options)) (*sns.UnsubscribeOutput, error) +} + +type SqsClient interface { + CreateQueue(ctx context.Context, params *sqs.CreateQueueInput, optFns ...func(*sqs.Options)) (*sqs.CreateQueueOutput, error) + ReceiveMessage(ctx context.Context, params *sqs.ReceiveMessageInput, optFns ...func(*sqs.Options)) (*sqs.ReceiveMessageOutput, error) + DeleteQueue(ctx context.Context, params *sqs.DeleteQueueInput, optFns ...func(*sqs.Options)) (*sqs.DeleteQueueOutput, error) +} + +type ViperType interface { + GetString(string) string + SetEnvKeyReplacer(*strings.Replacer) +} + +// ReceivedMessage standardizes the format of the received message +type ReceivedMessage struct { + MessageId string + Body *string +} + +// NewNotificationReceiver returns a new NotificationReceiverContext +func NewNotificationReceiver(v ViperType, snsService SnsClient, sqsService SqsClient, awsRegion string, awsAccountId string) NotificationReceiverContext { return NotificationReceiverContext{ - snsService: snsService, - sqsService: sqsService, - awsRegion: awsRegion, - awsAccountId: awsAccountId, + viper: v, + snsService: snsService, + sqsService: sqsService, + awsRegion: awsRegion, + awsAccountId: awsAccountId, + queueSubscriptionMap: make(map[string]string), + receiverCancelMap: make(map[string]context.CancelFunc), } } -func (n NotificationReceiverContext) CreateQueueWithSubscription(appCtx appcontext.AppContext, topicName string, params NotificationQueueParams) (string, error) { +// CreateQueueWithSubscription first creates a new queue, then subscribes an AWS topic to it +func (n NotificationReceiverContext) CreateQueueWithSubscription(appCtx appcontext.AppContext, params NotificationQueueParams) (string, error) { - queueName := fmt.Sprintf("%s_%s", params.Action, params.ObjectId) - queueArn := n.constructArn("sqs", queueName) - topicArn := n.constructArn("sns", topicName) + queueUUID := uuid.Must(uuid.NewV4()) - // Create queue + queueName := fmt.Sprintf("%s_%s", params.NamePrefix, queueUUID) + queueArn := n.constructArn("sqs", queueName) + topicArn := n.constructArn("sns", params.SubscriptionTopicName) accessPolicy := fmt.Sprintf(`{ "Version": "2012-10-17", @@ -89,102 +121,124 @@ func (n NotificationReceiverContext) CreateQueueWithSubscription(appCtx appconte log.Fatalf("Failed to create SQS queue, %v", err) } - // Create subscription - - filterPolicy := fmt.Sprintf(`{ - "detail": { - "object": { - "key": [ - {"suffix": "%s"} - ] - } - } - }`, params.ObjectId) - subscribeInput := &sns.SubscribeInput{ TopicArn: &topicArn, Protocol: aws.String("sqs"), Endpoint: &queueArn, Attributes: map[string]string{ - "FilterPolicy": filterPolicy, + "FilterPolicy": params.FilterPolicy, "FilterPolicyScope": "MessageBody", }, } - _, err = n.snsService.Subscribe(context.Background(), subscribeInput) + subscribeOutput, err := n.snsService.Subscribe(context.Background(), subscribeInput) if err != nil { log.Fatalf("Failed to create subscription, %v", err) } + n.queueSubscriptionMap[*result.QueueUrl] = *subscribeOutput.SubscriptionArn + return *result.QueueUrl, err } -// SendNotification sends a one or more notifications for all supported mediums -func (n NotificationReceiverContext) ReceiveMessages(appCtx appcontext.AppContext, queueUrl string) ([]types.Message, error) { - result, err := n.sqsService.ReceiveMessage(context.Background(), &sqs.ReceiveMessageInput{ +// ReceiveMessages polls given queue continuously for messages for up to 20 seconds +func (n NotificationReceiverContext) ReceiveMessages(appCtx appcontext.AppContext, queueUrl string) ([]ReceivedMessage, error) { + recCtx, cancelRecCtx := context.WithCancel(context.Background()) + defer cancelRecCtx() + n.receiverCancelMap[queueUrl] = cancelRecCtx + + result, err := n.sqsService.ReceiveMessage(recCtx, &sqs.ReceiveMessageInput{ QueueUrl: &queueUrl, MaxNumberOfMessages: 1, WaitTimeSeconds: 20, }) - if err != nil { - appCtx.Logger().Fatal("Couldn't get messages from queue. Here's why: %v\n", zap.Error(err)) + if err != nil && recCtx.Err() != context.Canceled { + appCtx.Logger().Info("Couldn't get messages from queue. Error: %v\n", zap.Error(err)) + return nil, err + } + + if recCtx.Err() == context.Canceled { + return nil, recCtx.Err() + } + + receivedMessages := make([]ReceivedMessage, len(result.Messages)) + for index, value := range result.Messages { + receivedMessages[index] = ReceivedMessage{ + MessageId: *value.MessageId, + Body: value.Body, + } } - return result.Messages, err + + return receivedMessages, recCtx.Err() } -// InitEmail initializes the email backend -func InitReceiver(v *viper.Viper, logger *zap.Logger) (NotificationReceiver, error) { - // if v.GetString(cli.EmailBackendFlag) == "ses" { - // // Setup Amazon SES (email) service TODO: This might be able - // // to be combined with the AWS Session that we're using for S3 - // // down below. - - // awsSESRegion := v.GetString(cli.AWSSESRegionFlag) - // awsSESDomain := v.GetString(cli.AWSSESDomainFlag) - // sysAdminEmail := v.GetString(cli.SysAdminEmail) - // logger.Info("Using ses email backend", - // zap.String("region", awsSESRegion), - // zap.String("domain", awsSESDomain)) - // cfg, err := config.LoadDefaultConfig(context.Background(), - // config.WithRegion(awsSESRegion), - // ) - // if err != nil { - // logger.Fatal("error loading ses aws config", zap.Error(err)) - // } - - // sesService := ses.NewFromConfig(cfg) - // input := &ses.GetAccountSendingEnabledInput{} - // result, err := sesService.GetAccountSendingEnabled(context.Background(), input) - // if err != nil || result == nil || !result.Enabled { - // logger.Error("email sending not enabled", zap.Error(err)) - // return NewNotificationSender(nil, awsSESDomain, sysAdminEmail), err - // } - // return NewNotificationSender(sesService, awsSESDomain, sysAdminEmail), nil - // } - - // domain := "milmovelocal" - // logger.Info("Using local email backend", zap.String("domain", domain)) - // return NewStubNotificationSender(domain), nil - - // Setup Amazon SES (email) service TODO: This might be able - // to be combined with the AWS Session that we're using for S3 - // down below. - - // TODO: verify if we should change this param name to awsNotificationRegion - awsSESRegion := v.GetString(cli.AWSSESRegionFlag) - awsAccountId := v.GetString("aws-account-id") - - cfg, err := config.LoadDefaultConfig(context.Background(), - config.WithRegion(awsSESRegion), - ) - if err != nil { - logger.Fatal("error loading ses aws config", zap.Error(err)) - return nil, err +// CloseoutQueue stops receiving messages and cleans up the queue and its subscriptions +func (n NotificationReceiverContext) CloseoutQueue(appCtx appcontext.AppContext, queueUrl string) error { + appCtx.Logger().Info("Closing out queue: %v", zap.String("queueUrl", queueUrl)) + + if cancelFunc, exists := n.receiverCancelMap[queueUrl]; exists { + cancelFunc() + delete(n.receiverCancelMap, queueUrl) + } + + if subscriptionArn, exists := n.queueSubscriptionMap[queueUrl]; exists { + _, err := n.snsService.Unsubscribe(context.Background(), &sns.UnsubscribeInput{ + SubscriptionArn: &subscriptionArn, + }) + if err != nil { + return err + } + delete(n.queueSubscriptionMap, queueUrl) + } + + _, err := n.sqsService.DeleteQueue(context.Background(), &sqs.DeleteQueueInput{ + QueueUrl: &queueUrl, + }) + + return err +} + +// GetDefaultTopic returns the topic value set within the environment +func (n NotificationReceiverContext) GetDefaultTopic() (string, error) { + // v := viper.New() + n.viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) + // v.AutomaticEnv() + topicName := n.viper.GetString(cli.AWSSNSObjectTagsAddedTopicFlag) + receiverBackend := n.viper.GetString(cli.ReceiverBackendFlag) + if topicName == "" && receiverBackend == "sns&sqs" { + return "", errors.New("aws_sns_object_tags_added_topic key not available") } + return topicName, nil +} + +// InitReceiver initializes the receiver backend +func InitReceiver(v ViperType, logger *zap.Logger) (NotificationReceiver, error) { + + // v := viper.New() + // v.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) + // v.AutomaticEnv() - snsService := sns.NewFromConfig(cfg) - sqsService := sqs.NewFromConfig(cfg) + if v.GetString(cli.ReceiverBackendFlag) == "sns&sqs" { + // Setup notification receiver service with SNS & SQS backend dependencies + awsSNSRegion := v.GetString(cli.AWSSNSRegionFlag) + awsAccountId := v.GetString(cli.AWSSNSAccountId) + + logger.Info("Using aws sns&sqs receiver backend", zap.String("region", awsSNSRegion)) + + cfg, err := config.LoadDefaultConfig(context.Background(), + config.WithRegion(awsSNSRegion), + ) + if err != nil { + logger.Fatal("error loading sns aws config", zap.Error(err)) + return nil, err + } + + snsService := sns.NewFromConfig(cfg) + sqsService := sqs.NewFromConfig(cfg) + + return NewNotificationReceiver(v, snsService, sqsService, awsSNSRegion, awsAccountId), nil + } - return NewNotificationReceiver(snsService, sqsService, awsSESRegion, awsAccountId), nil + return NewStubNotificationReceiver(), nil } func (n NotificationReceiverContext) constructArn(awsService string, endpointName string) string { diff --git a/pkg/notifications/notification_receiver_stub.go b/pkg/notifications/notification_receiver_stub.go new file mode 100644 index 00000000000..f87806b9451 --- /dev/null +++ b/pkg/notifications/notification_receiver_stub.go @@ -0,0 +1,49 @@ +package notifications + +import ( + "context" + + "go.uber.org/zap" + + "github.com/transcom/mymove/pkg/appcontext" +) + +// StubNotificationReceiver mocks an SNS & SQS client for local usage +type StubNotificationReceiver NotificationReceiverContext + +// NewStubNotificationReceiver returns a new StubNotificationReceiver +func NewStubNotificationReceiver() StubNotificationReceiver { + return StubNotificationReceiver{ + snsService: nil, + sqsService: nil, + awsRegion: "", + awsAccountId: "", + queueSubscriptionMap: make(map[string]string), + receiverCancelMap: make(map[string]context.CancelFunc), + } +} + +func (n StubNotificationReceiver) CreateQueueWithSubscription(appCtx appcontext.AppContext, params NotificationQueueParams) (string, error) { + return "stubQueueName", nil +} + +func (n StubNotificationReceiver) ReceiveMessages(appCtx appcontext.AppContext, queueUrl string) ([]ReceivedMessage, error) { + messageId := "stubMessageId" + body := queueUrl + ":stubMessageBody" + mockMessages := make([]ReceivedMessage, 1) + mockMessages[0] = ReceivedMessage{ + MessageId: messageId, + Body: &body, + } + appCtx.Logger().Debug("Receiving a stubbed message for queue: %v", zap.String("queueUrl", queueUrl)) + return mockMessages, nil +} + +func (n StubNotificationReceiver) CloseoutQueue(appCtx appcontext.AppContext, queueUrl string) error { + appCtx.Logger().Debug("Closing out the stubbed queue.") + return nil +} + +func (n StubNotificationReceiver) GetDefaultTopic() (string, error) { + return "stubDefaultTopic", nil +} diff --git a/pkg/notifications/notification_receiver_test.go b/pkg/notifications/notification_receiver_test.go new file mode 100644 index 00000000000..836ed986c19 --- /dev/null +++ b/pkg/notifications/notification_receiver_test.go @@ -0,0 +1,158 @@ +package notifications + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/sns" + "github.com/aws/aws-sdk-go-v2/service/sqs" + "github.com/aws/aws-sdk-go-v2/service/sqs/types" + "github.com/spf13/viper" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" + + "github.com/transcom/mymove/pkg/cli" + "github.com/transcom/mymove/pkg/testingsuite" +) + +type notificationReceiverSuite struct { + *testingsuite.PopTestSuite +} + +func TestNotificationReceiverSuite(t *testing.T) { + + hs := ¬ificationReceiverSuite{ + PopTestSuite: testingsuite.NewPopTestSuite(testingsuite.CurrentPackage(), + testingsuite.WithPerTestTransaction()), + } + suite.Run(t, hs) + hs.PopTestSuite.TearDown() +} + +// mock - Viper +type Viper struct { + mock.Mock +} + +func (_m *Viper) GetString(key string) string { + switch key { + case cli.ReceiverBackendFlag: + return "sns&sqs" + case cli.AWSRegionFlag: + return "us-gov-west-1" + case cli.AWSSNSAccountId: + return "12345" + case cli.AWSSNSObjectTagsAddedTopicFlag: + return "fake_sns_topic" + } + return "" +} +func (_m *Viper) SetEnvKeyReplacer(_ *strings.Replacer) {} + +// mock - SNS +type MockSnsClient struct { + mock.Mock +} + +func (_m *MockSnsClient) Subscribe(ctx context.Context, params *sns.SubscribeInput, optFns ...func(*sns.Options)) (*sns.SubscribeOutput, error) { + return &sns.SubscribeOutput{SubscriptionArn: aws.String("FakeSubscriptionArn")}, nil +} + +func (_m *MockSnsClient) Unsubscribe(ctx context.Context, params *sns.UnsubscribeInput, optFns ...func(*sns.Options)) (*sns.UnsubscribeOutput, error) { + return &sns.UnsubscribeOutput{}, nil +} + +// mock - SQS +type MockSqsClient struct { + mock.Mock +} + +func (_m *MockSqsClient) CreateQueue(ctx context.Context, params *sqs.CreateQueueInput, optFns ...func(*sqs.Options)) (*sqs.CreateQueueOutput, error) { + return &sqs.CreateQueueOutput{ + QueueUrl: aws.String("FakeQueueUrl"), + }, nil +} +func (_m *MockSqsClient) ReceiveMessage(ctx context.Context, params *sqs.ReceiveMessageInput, optFns ...func(*sqs.Options)) (*sqs.ReceiveMessageOutput, error) { + messages := make([]types.Message, 0) + messages = append(messages, types.Message{ + MessageId: aws.String("fakeMessageId"), + Body: aws.String(*params.QueueUrl + ":fakeMessageBody"), + }) + return &sqs.ReceiveMessageOutput{ + Messages: messages, + }, nil +} +func (_m *MockSqsClient) DeleteQueue(ctx context.Context, params *sqs.DeleteQueueInput, optFns ...func(*sqs.Options)) (*sqs.DeleteQueueOutput, error) { + return &sqs.DeleteQueueOutput{}, nil +} + +func (suite *notificationReceiverSuite) TestSuccessPath() { + + suite.Run("local backend - notification receiver stub", func() { + v := viper.New() + localReceiver, err := InitReceiver(v, suite.Logger()) + + suite.NoError(err) + suite.IsType(StubNotificationReceiver{}, localReceiver) + + defaultTopic, err := localReceiver.GetDefaultTopic() + suite.Equal("stubDefaultTopic", defaultTopic) + suite.NoError(err) + + queueParams := NotificationQueueParams{ + NamePrefix: "testPrefix", + } + createdQueueUrl, err := localReceiver.CreateQueueWithSubscription(suite.AppContextForTest(), queueParams) + suite.NoError(err) + suite.NotContains(createdQueueUrl, queueParams.NamePrefix) + suite.Equal(createdQueueUrl, "stubQueueName") + + receivedMessages, err := localReceiver.ReceiveMessages(suite.AppContextForTest(), createdQueueUrl) + suite.NoError(err) + suite.Len(receivedMessages, 1) + suite.Equal(receivedMessages[0].MessageId, "stubMessageId") + suite.Equal(*receivedMessages[0].Body, fmt.Sprintf("%s:stubMessageBody", createdQueueUrl)) + }) + + suite.Run("aws backend - notification receiver init", func() { + v := Viper{} + + receiver, _ := InitReceiver(&v, suite.Logger()) + suite.IsType(NotificationReceiverContext{}, receiver) + defaultTopic, err := receiver.GetDefaultTopic() + suite.Equal("fake_sns_topic", defaultTopic) + suite.NoError(err) + }) + + suite.Run("aws backend - notification receiver with mock services", func() { + v := Viper{} + snsService := MockSnsClient{} + sqsService := MockSqsClient{} + + receiver := NewNotificationReceiver(&v, &snsService, &sqsService, "", "") + suite.IsType(NotificationReceiverContext{}, receiver) + + defaultTopic, err := receiver.GetDefaultTopic() + suite.Equal("fake_sns_topic", defaultTopic) + suite.NoError(err) + + queueParams := NotificationQueueParams{ + NamePrefix: "testPrefix", + } + createdQueueUrl, err := receiver.CreateQueueWithSubscription(suite.AppContextForTest(), queueParams) + suite.NoError(err) + suite.Equal("FakeQueueUrl", createdQueueUrl) + + receivedMessages, err := receiver.ReceiveMessages(suite.AppContextForTest(), createdQueueUrl) + suite.NoError(err) + suite.Len(receivedMessages, 1) + suite.Equal(receivedMessages[0].MessageId, "fakeMessageId") + suite.Equal(*receivedMessages[0].Body, fmt.Sprintf("%s:fakeMessageBody", createdQueueUrl)) + + err = receiver.CloseoutQueue(suite.AppContextForTest(), createdQueueUrl) + suite.NoError(err) + }) +} diff --git a/pkg/notifications/notification_stub.go b/pkg/notifications/notification_sender_stub.go similarity index 100% rename from pkg/notifications/notification_stub.go rename to pkg/notifications/notification_sender_stub.go diff --git a/pkg/notifications/notification_test.go b/pkg/notifications/notification_sender_test.go similarity index 100% rename from pkg/notifications/notification_test.go rename to pkg/notifications/notification_sender_test.go diff --git a/pkg/storage/filesystem.go b/pkg/storage/filesystem.go index 259fd4ee8ab..f6e43583420 100644 --- a/pkg/storage/filesystem.go +++ b/pkg/storage/filesystem.go @@ -116,6 +116,8 @@ func (fs *Filesystem) Fetch(key string) (io.ReadCloser, error) { // Tags returns the tags for a specified key func (fs *Filesystem) Tags(_ string) (map[string]string, error) { tags := make(map[string]string) + // Assume anti-virus complete + tags["av-status"] = "CLEAN" return tags, nil } diff --git a/pkg/storage/memory.go b/pkg/storage/memory.go index 2f06ed6b96e..4e171e40e9d 100644 --- a/pkg/storage/memory.go +++ b/pkg/storage/memory.go @@ -116,6 +116,8 @@ func (fs *Memory) Fetch(key string) (io.ReadCloser, error) { // Tags returns the tags for a specified key func (fs *Memory) Tags(_ string) (map[string]string, error) { tags := make(map[string]string) + // Assume anti-virus complete + tags["av-status"] = "CLEAN" return tags, nil } diff --git a/pkg/storage/test/s3.go b/pkg/storage/test/s3.go index da076681dfe..5f738e7b088 100644 --- a/pkg/storage/test/s3.go +++ b/pkg/storage/test/s3.go @@ -90,7 +90,8 @@ func (fake *FakeS3Storage) TempFileSystem() *afero.Afero { // Tags returns the tags for a specified key func (fake *FakeS3Storage) Tags(_ string) (map[string]string, error) { tags := map[string]string{ - "tagName": "tagValue", + "tagName": "tagValue", + "av-status": "CLEAN", // Assume anti-virus run } return tags, nil } From 1729cbde3a2e902e069dbd412de9f5114630afb8 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Wed, 8 Jan 2025 21:23:10 +0000 Subject: [PATCH 03/36] B-22056 - env var updates to match planned in param store. --- .envrc | 10 ++++++- pkg/cli/receiver.go | 30 +++++++++---------- pkg/notifications/notification_receiver.go | 16 +++------- .../notification_receiver_test.go | 6 ++-- 4 files changed, 31 insertions(+), 31 deletions(-) diff --git a/.envrc b/.envrc index 3891c5b8d85..64ef9f3c646 100644 --- a/.envrc +++ b/.envrc @@ -234,14 +234,17 @@ export TZ="UTC" # # export STORAGE_BACKEND=s3 # export EMAIL_BACKEND=ses +# export RECEIVER_BACKEND="sns&sqs" # # Instructions for using S3 storage backend here: https://dp3.atlassian.net/wiki/spaces/MT/pages/1470955567/How+to+test+storing+data+in+S3+locally # Instructions for using SES email backend here: https://dp3.atlassian.net/wiki/spaces/MT/pages/1467973894/How+to+test+sending+email+locally +# Instructions for using SNS&SQS backend here: ... # # The default and equivalent to not being set is: # # export STORAGE_BACKEND=local # export EMAIL_BACKEND=local +# export RECEIVER_BACKEND=local # # Setting region and profile conditionally while we migrate from com to govcloud. if [ "$STORAGE_BACKEND" == "s3" ]; then @@ -255,6 +258,11 @@ export AWS_S3_KEY_NAMESPACE=$USER export AWS_SES_DOMAIN="devlocal.dp3.us" export AWS_SES_REGION="us-gov-west-1" +if [ "$RECEIVER_BACKEND" == "sns&sqs" ]; then + export SNS_TAGS_UPDATED_TOPIC="app_s3_tag_events" + export SNS_REGION="us-gov-west-1" +fi + # To use s3 links aws-bucketname/xx/user/ for local builds, # you'll need to add the following to your .envrc.local: # @@ -441,4 +449,4 @@ then fi # Check that all required environment variables are set -check_required_variables \ No newline at end of file +check_required_variables diff --git a/pkg/cli/receiver.go b/pkg/cli/receiver.go index 91f6f30f872..be30daf135d 100644 --- a/pkg/cli/receiver.go +++ b/pkg/cli/receiver.go @@ -10,20 +10,20 @@ import ( const ( // ReceiverBackend is the Receiver Backend Flag ReceiverBackendFlag string = "receiver-backend" - // AWSSNSObjectTagsAddedTopic is the AWS SNS Object Tags Added Topic Flag - AWSSNSObjectTagsAddedTopicFlag string = "aws-sns-object-tags-added-topic" - // AWSS3RegionFlag is the AWS SNS Region Flag - AWSSNSRegionFlag string = "aws-sns-region" - // AWSSNSAccountId is the application's AWS account id - AWSSNSAccountId string = "aws-account-id" + // SNSTagsUpdatedTopicFlag is the SNS Tags Updated Topic Flag + SNSTagsUpdatedTopicFlag string = "sns-tags-updated-topic" + // SNSRegionFlag is the SNS Region flag + SNSRegionFlag string = "sns-region" + // SNSAccountId is the application's AWS account id + SNSAccountId string = "aws-account-id" ) // InitReceiverFlags initializes Storage command line flags func InitReceiverFlags(flag *pflag.FlagSet) { flag.String(ReceiverBackendFlag, "local", "Receiver backend to use, either local or sns&sqs.") - flag.String(AWSSNSObjectTagsAddedTopicFlag, "", "SNS Topic for receiving event messages") - flag.String(AWSSNSRegionFlag, "", "AWS region used for SNS and SQS") - flag.String(AWSSNSAccountId, "", "AWS account Id") + flag.String(SNSTagsUpdatedTopicFlag, "", "SNS Topic for receiving event messages") + flag.String(SNSRegionFlag, "", "Region used for SNS and SQS") + flag.String(SNSAccountId, "", "SNS account Id") } // CheckReceiver validates Storage command line flags @@ -35,17 +35,17 @@ func CheckReceiver(v *viper.Viper) error { } if receiverBackend == "sns&sqs" { - r := v.GetString(AWSSNSRegionFlag) + r := v.GetString(SNSRegionFlag) if r == "" { - return fmt.Errorf("invalid value for %s: %s", AWSSNSRegionFlag, r) + return fmt.Errorf("invalid value for %s: %s", SNSRegionFlag, r) } - topic := v.GetString(AWSSNSObjectTagsAddedTopicFlag) + topic := v.GetString(SNSTagsUpdatedTopicFlag) if topic == "" { - return fmt.Errorf("invalid value for %s: %s", AWSSNSObjectTagsAddedTopicFlag, topic) + return fmt.Errorf("invalid value for %s: %s", SNSTagsUpdatedTopicFlag, topic) } - accountId := v.GetString(AWSSNSAccountId) + accountId := v.GetString(SNSAccountId) if topic == "" { - return fmt.Errorf("invalid value for %s: %s", AWSSNSAccountId, accountId) + return fmt.Errorf("invalid value for %s: %s", SNSAccountId, accountId) } } diff --git a/pkg/notifications/notification_receiver.go b/pkg/notifications/notification_receiver.go index b1c95495bc7..e6eba10a5e7 100644 --- a/pkg/notifications/notification_receiver.go +++ b/pkg/notifications/notification_receiver.go @@ -199,28 +199,20 @@ func (n NotificationReceiverContext) CloseoutQueue(appCtx appcontext.AppContext, // GetDefaultTopic returns the topic value set within the environment func (n NotificationReceiverContext) GetDefaultTopic() (string, error) { - // v := viper.New() - n.viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) - // v.AutomaticEnv() - topicName := n.viper.GetString(cli.AWSSNSObjectTagsAddedTopicFlag) + topicName := n.viper.GetString(cli.SNSTagsUpdatedTopicFlag) receiverBackend := n.viper.GetString(cli.ReceiverBackendFlag) if topicName == "" && receiverBackend == "sns&sqs" { - return "", errors.New("aws_sns_object_tags_added_topic key not available") + return "", errors.New("sns_tags_updated_topic key not available") } return topicName, nil } // InitReceiver initializes the receiver backend func InitReceiver(v ViperType, logger *zap.Logger) (NotificationReceiver, error) { - - // v := viper.New() - // v.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) - // v.AutomaticEnv() - if v.GetString(cli.ReceiverBackendFlag) == "sns&sqs" { // Setup notification receiver service with SNS & SQS backend dependencies - awsSNSRegion := v.GetString(cli.AWSSNSRegionFlag) - awsAccountId := v.GetString(cli.AWSSNSAccountId) + awsSNSRegion := v.GetString(cli.SNSRegionFlag) + awsAccountId := v.GetString(cli.SNSAccountId) logger.Info("Using aws sns&sqs receiver backend", zap.String("region", awsSNSRegion)) diff --git a/pkg/notifications/notification_receiver_test.go b/pkg/notifications/notification_receiver_test.go index 836ed986c19..e5f0bc8ee38 100644 --- a/pkg/notifications/notification_receiver_test.go +++ b/pkg/notifications/notification_receiver_test.go @@ -41,11 +41,11 @@ func (_m *Viper) GetString(key string) string { switch key { case cli.ReceiverBackendFlag: return "sns&sqs" - case cli.AWSRegionFlag: + case cli.SNSRegionFlag: return "us-gov-west-1" - case cli.AWSSNSAccountId: + case cli.SNSAccountId: return "12345" - case cli.AWSSNSObjectTagsAddedTopicFlag: + case cli.SNSTagsUpdatedTopicFlag: return "fake_sns_topic" } return "" From 75deebbeb68c31d4c651aac33aecefca52cab499 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Thu, 9 Jan 2025 18:06:36 +0000 Subject: [PATCH 04/36] B-22056 - additional test and .envrc cleanup. --- .envrc | 4 +- pkg/handlers/internalapi/uploads_test.go | 47 +++++++++++++++++++++++- 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/.envrc b/.envrc index 64ef9f3c646..7eb37fa168f 100644 --- a/.envrc +++ b/.envrc @@ -229,7 +229,7 @@ export TZ="UTC" # AWS development access # -# To use S3/SES for local builds, you'll need to uncomment the following. +# To use S3/SES or SNS&SQS for local builds, you'll need to uncomment the following. # Do not commit the change: # # export STORAGE_BACKEND=s3 @@ -238,7 +238,7 @@ export TZ="UTC" # # Instructions for using S3 storage backend here: https://dp3.atlassian.net/wiki/spaces/MT/pages/1470955567/How+to+test+storing+data+in+S3+locally # Instructions for using SES email backend here: https://dp3.atlassian.net/wiki/spaces/MT/pages/1467973894/How+to+test+sending+email+locally -# Instructions for using SNS&SQS backend here: ... +# Instructions for using SNS&SQS backend here: https://dp3.atlassian.net/wiki/spaces/MT/pages/2793242625/How+to+test+notifications+receiver+locally # # The default and equivalent to not being set is: # diff --git a/pkg/handlers/internalapi/uploads_test.go b/pkg/handlers/internalapi/uploads_test.go index 143dfa465eb..271495e2991 100644 --- a/pkg/handlers/internalapi/uploads_test.go +++ b/pkg/handlers/internalapi/uploads_test.go @@ -525,7 +525,52 @@ func (suite *HandlerSuite) TestGetUploadStatusHandlerFailure() { suite.Error(err) }) - // TODO: ADD A FORBIDDEN TEST + suite.Run("Error when attempting access to another service member's upload", func() { + fakeS3 := storageTest.NewFakeS3Storage(true) + localReceiver := notifications.StubNotificationReceiver{} + + otherServiceMember := factory.BuildServiceMember(suite.DB(), nil, nil) + + orders := factory.BuildOrder(suite.DB(), nil, nil) + uploadUser1 := factory.BuildUserUpload(suite.DB(), []factory.Customization{ + { + Model: orders.UploadedOrders, + LinkOnly: true, + }, + { + Model: models.Upload{ + Filename: "FileName", + Bytes: int64(15), + ContentType: uploader.FileTypePDF, + }, + }, + }, nil) + + file := suite.Fixture(FixturePDF) + _, err := fakeS3.Store(uploadUser1.Upload.StorageKey, file.Data, "somehash", nil) + suite.NoError(err) + + params := uploadop.NewGetUploadStatusParams() + params.UploadID = strfmt.UUID(uploadUser1.Upload.ID.String()) + + req := &http.Request{} + req = suite.AuthenticateRequest(req, otherServiceMember) + params.HTTPRequest = req + + handlerConfig := suite.HandlerConfig() + handlerConfig.SetFileStorer(fakeS3) + handlerConfig.SetNotificationReceiver(localReceiver) + uploadInformationFetcher := upload.NewUploadInformationFetcher() + handler := GetUploadStatusHandler{handlerConfig, uploadInformationFetcher} + + response := handler.Handle(params) + _, ok := response.(*uploadop.GetUploadStatusForbidden) + suite.True(ok) + + queriedUpload := models.Upload{} + err = suite.DB().Find(&queriedUpload, uploadUser1.Upload.ID) + suite.NoError(err) + }) } func (suite *HandlerSuite) TestCreatePPMUploadsHandlerSuccess() { From 399bfb4d99139b0085acde0943bd67413e0f7b57 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Thu, 9 Jan 2025 19:28:15 +0000 Subject: [PATCH 05/36] B-22056 - setup for exp testing. --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b8d3c39da69..b5bd5920986 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,30 +40,30 @@ references: # In addition, it's common practice to disable acceptance tests and # ignore tests for dp3 deploys. See the branch settings below. - dp3-branch: &dp3-branch placeholder_branch_name + dp3-branch: &dp3-branch MAIN-B-22056_sns_sqs_deps_w_endpoint # MUST BE ONE OF: loadtest, demo, exp. # These are used to pull in env vars so the spelling matters! - dp3-env: &dp3-env placeholder_env + dp3-env: &dp3-env exp # set integration-ignore-branch to the branch if you want to IGNORE # integration tests, or `placeholder_branch_name` if you do want to # run them - integration-ignore-branch: &integration-ignore-branch placeholder_branch_name + integration-ignore-branch: &integration-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint # set integration-mtls-ignore-branch to the branch if you want to # IGNORE mtls integration tests, or `placeholder_branch_name` if you # do want to run them - integration-mtls-ignore-branch: &integration-mtls-ignore-branch placeholder_branch_name + integration-mtls-ignore-branch: &integration-mtls-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint # set client-ignore-branch to the branch if you want to IGNORE # client tests, or `placeholder_branch_name` if you do want to run # them - client-ignore-branch: &client-ignore-branch placeholder_branch_name + client-ignore-branch: &client-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint # set server-ignore-branch to the branch if you want to IGNORE # server tests, or `placeholder_branch_name` if you do want to run # them - server-ignore-branch: &server-ignore-branch placeholder_branch_name + server-ignore-branch: &server-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint executors: base_small: From 2d77f6dcc4fd226285ae48b9997917a42c2d9a45 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Thu, 9 Jan 2025 19:36:42 +0000 Subject: [PATCH 06/36] B-22056 - fix previous merge. --- pkg/gen/internalapi/embedded_spec.go | 10 ++++++++++ swagger/internal.yaml | 4 ++++ 2 files changed, 14 insertions(+) diff --git a/pkg/gen/internalapi/embedded_spec.go b/pkg/gen/internalapi/embedded_spec.go index f30aca3b049..639b229a03a 100644 --- a/pkg/gen/internalapi/embedded_spec.go +++ b/pkg/gen/internalapi/embedded_spec.go @@ -3415,6 +3415,11 @@ func init() { "x-nullable": true, "example": "LOS ANGELES" }, + "destinationGbloc": { + "type": "string", + "pattern": "^[A-Z]{4}$", + "x-nullable": true + }, "eTag": { "type": "string", "readOnly": true @@ -12586,6 +12591,11 @@ func init() { "x-nullable": true, "example": "LOS ANGELES" }, + "destinationGbloc": { + "type": "string", + "pattern": "^[A-Z]{4}$", + "x-nullable": true + }, "eTag": { "type": "string", "readOnly": true diff --git a/swagger/internal.yaml b/swagger/internal.yaml index 21483825daa..15499febdd9 100644 --- a/swagger/internal.yaml +++ b/swagger/internal.yaml @@ -2758,6 +2758,10 @@ definitions: type: string format: uuid example: c56a4180-65aa-42ec-a945-5fd21dec0538 + destinationGbloc: + type: string + pattern: ^[A-Z]{4}$ + x-nullable: true required: - streetAddress1 - city From ed66a1642fe55a16eb26632cdf8c732c68702115 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Thu, 9 Jan 2025 20:35:47 +0000 Subject: [PATCH 07/36] B-22056 - restore exp env. --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b5bd5920986..b8d3c39da69 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,30 +40,30 @@ references: # In addition, it's common practice to disable acceptance tests and # ignore tests for dp3 deploys. See the branch settings below. - dp3-branch: &dp3-branch MAIN-B-22056_sns_sqs_deps_w_endpoint + dp3-branch: &dp3-branch placeholder_branch_name # MUST BE ONE OF: loadtest, demo, exp. # These are used to pull in env vars so the spelling matters! - dp3-env: &dp3-env exp + dp3-env: &dp3-env placeholder_env # set integration-ignore-branch to the branch if you want to IGNORE # integration tests, or `placeholder_branch_name` if you do want to # run them - integration-ignore-branch: &integration-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint + integration-ignore-branch: &integration-ignore-branch placeholder_branch_name # set integration-mtls-ignore-branch to the branch if you want to # IGNORE mtls integration tests, or `placeholder_branch_name` if you # do want to run them - integration-mtls-ignore-branch: &integration-mtls-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint + integration-mtls-ignore-branch: &integration-mtls-ignore-branch placeholder_branch_name # set client-ignore-branch to the branch if you want to IGNORE # client tests, or `placeholder_branch_name` if you do want to run # them - client-ignore-branch: &client-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint + client-ignore-branch: &client-ignore-branch placeholder_branch_name # set server-ignore-branch to the branch if you want to IGNORE # server tests, or `placeholder_branch_name` if you do want to run # them - server-ignore-branch: &server-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint + server-ignore-branch: &server-ignore-branch placeholder_branch_name executors: base_small: From bf315ee3df79ef23f9bb1c2f6685dcbc18244a3f Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Fri, 10 Jan 2025 21:39:34 +0000 Subject: [PATCH 08/36] B-22056 - additional test for receiving messages on routing. --- cmd/milmove/serve.go | 7 +- pkg/handlers/internalapi/uploads.go | 87 ++++++++++------ pkg/handlers/internalapi/uploads_test.go | 2 +- .../routing/internalapi_test/uploads_test.go | 50 +++++++++- pkg/notifications/notification_receiver.go | 98 ++++++++++++++++--- .../notification_receiver_stub.go | 4 +- .../notification_receiver_test.go | 11 ++- pkg/storage/test/s3.go | 4 + 8 files changed, 212 insertions(+), 51 deletions(-) diff --git a/cmd/milmove/serve.go b/cmd/milmove/serve.go index 8e9d8878d82..7d4e28a9918 100644 --- a/cmd/milmove/serve.go +++ b/cmd/milmove/serve.go @@ -478,8 +478,11 @@ func buildRoutingConfig(appCtx appcontext.AppContext, v *viper.Viper, redisPool appCtx.Logger().Fatal("notification sender sending not enabled", zap.Error(err)) } - // Email - notificationReceiver, _ := notifications.InitReceiver(v, appCtx.Logger()) + // Notification Receiver + notificationReceiver, err := notifications.InitReceiver(v, appCtx.Logger()) + if err != nil { + appCtx.Logger().Fatal("notification receiver not enabled", zap.Error(err)) + } routingConfig.BuildRoot = v.GetString(cli.BuildRootFlag) sendProductionInvoice := v.GetBool(cli.GEXSendProdInvoiceFlag) diff --git a/pkg/handlers/internalapi/uploads.go b/pkg/handlers/internalapi/uploads.go index 834d2124d43..a1bff90b220 100644 --- a/pkg/handlers/internalapi/uploads.go +++ b/pkg/handlers/internalapi/uploads.go @@ -9,6 +9,7 @@ import ( "regexp" "strconv" "strings" + "time" "github.com/go-openapi/runtime" "github.com/go-openapi/runtime/middleware" @@ -258,7 +259,7 @@ type GetUploadStatusHandler struct { services.UploadInformationFetcher } -type CustomNewUploadStatusOK struct { +type CustomGetUploadStatusResponse struct { params uploadop.GetUploadStatusParams storageKey string appCtx appcontext.AppContext @@ -278,39 +279,45 @@ const ( AVStatusTypeINFECTED AVStatusType = "INFECTED" ) -func writeEventStreamMessage(rw http.ResponseWriter, producer runtime.Producer, id int, event string, data string) { +func (o *CustomGetUploadStatusResponse) writeEventStreamMessage(rw http.ResponseWriter, producer runtime.Producer, id int, event string, data string) { resProcess := []byte(fmt.Sprintf("id: %s\nevent: %s\ndata: %s\n\n", strconv.Itoa(id), event, data)) if produceErr := producer.Produce(rw, resProcess); produceErr != nil { - panic(produceErr) + o.appCtx.Logger().Error(produceErr.Error()) } if f, ok := rw.(http.Flusher); ok { f.Flush() } } -func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { +func (o *CustomGetUploadStatusResponse) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { // Check current tag before event-driven wait for anti-virus tags, err := o.storer.Tags(o.storageKey) var uploadStatus AVStatusType if err != nil || len(tags) == 0 { uploadStatus = AVStatusTypePROCESSING - } else { + } else if _, exists := tags["av-status"]; exists { uploadStatus = AVStatusType(tags["av-status"]) + } else { + uploadStatus = AVStatusTypePROCESSING } - writeEventStreamMessage(rw, producer, 0, "message", string(uploadStatus)) - if uploadStatus == AVStatusTypeCLEAN || uploadStatus == AVStatusTypeINFECTED { - writeEventStreamMessage(rw, producer, 1, "close", "Connection closed") + rw.WriteHeader(http.StatusOK) + o.writeEventStreamMessage(rw, producer, 0, "message", string(uploadStatus)) + o.writeEventStreamMessage(rw, producer, 1, "close", "Connection closed") return // skip notification loop since object already tagged from anti-virus + } else { + // Limitation: once the status code header has been written (first response), we are not able to update the status for subsequent responses. + // StatusAccepted: Standard code 202 for accepted request, but response not yet ready. + rw.WriteHeader(http.StatusAccepted) + o.writeEventStreamMessage(rw, producer, 0, "message", string(uploadStatus)) } // Start waiting for tag updates topicName, err := o.receiver.GetDefaultTopic() if err != nil { - o.appCtx.Logger().Error("aws_sns_object_tags_added_topic key not available.") - return + o.appCtx.Logger().Error(err.Error()) } filterPolicy := fmt.Sprintf(`{ @@ -325,7 +332,7 @@ func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer notificationParams := notifications.NotificationQueueParams{ SubscriptionTopicName: topicName, - NamePrefix: "ObjectTagsAdded", + NamePrefix: notifications.QueuePrefixObjectTagsAdded, FilterPolicy: filterPolicy, } @@ -334,23 +341,36 @@ func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer o.appCtx.Logger().Error(err.Error()) } - // Cleanup + id_counter := 1 + + // For loop over 120 seconds, cancel context when done and it breaks the loop + totalReceiverContext, totalReceiverContextCancelFunc := context.WithTimeout(context.Background(), 120*time.Second) + defer totalReceiverContextCancelFunc() + + // Cleanup if client closes connection go func() { <-o.params.HTTPRequest.Context().Done() + totalReceiverContextCancelFunc() + }() + + // Cleanup at end of work + go func() { + <-totalReceiverContext.Done() + id_counter++ + o.writeEventStreamMessage(rw, producer, id_counter, "close", "Connection closed") _ = o.receiver.CloseoutQueue(o.appCtx, queueUrl) }() - id_counter := 1 - // Run for 120 seconds, 20 second long polling for receiver, 6 times - for range 6 { - o.appCtx.Logger().Info("Receiving...") - messages, errs := o.receiver.ReceiveMessages(o.appCtx, queueUrl) - if errs != nil && errs != context.Canceled { - o.appCtx.Logger().Error(errs.Error()) - } + for { + o.appCtx.Logger().Info("Receiving Messages...") + messages, errs := o.receiver.ReceiveMessages(o.appCtx, queueUrl, totalReceiverContext) - if errs == context.Canceled { - break + if errors.Is(errs, context.Canceled) || errors.Is(errs, context.DeadlineExceeded) { + return + } + if errs != nil { + o.appCtx.Logger().Error(err.Error()) + return } if len(messages) != 0 { @@ -360,11 +380,13 @@ func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer if err != nil || len(tags) == 0 { uploadStatus = AVStatusTypePROCESSING - } else { + } else if _, exists := tags["av-status"]; exists { uploadStatus = AVStatusType(tags["av-status"]) + } else { + uploadStatus = AVStatusTypePROCESSING } - writeEventStreamMessage(rw, producer, id_counter, "message", string(uploadStatus)) + o.writeEventStreamMessage(rw, producer, id_counter, "message", string(uploadStatus)) if uploadStatus == AVStatusTypeCLEAN || uploadStatus == AVStatusTypeINFECTED { return errors.New("connection_closed") @@ -374,16 +396,23 @@ func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer }) if errTransaction != nil && errTransaction.Error() == "connection_closed" { - id_counter++ - writeEventStreamMessage(rw, producer, id_counter, "close", "Connection closed") - break + return } if errTransaction != nil { - panic(errTransaction) // let the recovery middleware deal with this + o.appCtx.Logger().Error(err.Error()) + return } } id_counter++ + + select { + case <-totalReceiverContext.Done(): + return + default: + time.Sleep(1 * time.Second) // Throttle as a precaution against hounding of the SDK + continue + } } } @@ -415,7 +444,7 @@ func (h GetUploadStatusHandler) Handle(params uploadop.GetUploadStatusParams) mi return handleError(err) } - return &CustomNewUploadStatusOK{ + return &CustomGetUploadStatusResponse{ params: params, storageKey: uploaded.Upload.StorageKey, appCtx: h.AppContextFromRequest(params.HTTPRequest), diff --git a/pkg/handlers/internalapi/uploads_test.go b/pkg/handlers/internalapi/uploads_test.go index 271495e2991..ab9c264f77d 100644 --- a/pkg/handlers/internalapi/uploads_test.go +++ b/pkg/handlers/internalapi/uploads_test.go @@ -486,7 +486,7 @@ func (suite *HandlerSuite) TestGetUploadStatusHandlerSuccess() { handler := GetUploadStatusHandler{handlerConfig, uploadInformationFetcher} response := handler.Handle(params) - _, ok := response.(*CustomNewUploadStatusOK) + _, ok := response.(*CustomGetUploadStatusResponse) suite.True(ok) queriedUpload := models.Upload{} diff --git a/pkg/handlers/routing/internalapi_test/uploads_test.go b/pkg/handlers/routing/internalapi_test/uploads_test.go index 3fe89e8927d..5b760f740bc 100644 --- a/pkg/handlers/routing/internalapi_test/uploads_test.go +++ b/pkg/handlers/routing/internalapi_test/uploads_test.go @@ -3,14 +3,17 @@ package internalapi_test import ( "net/http" "net/http/httptest" + "time" "github.com/transcom/mymove/pkg/factory" "github.com/transcom/mymove/pkg/models" + storageTest "github.com/transcom/mymove/pkg/storage/test" "github.com/transcom/mymove/pkg/uploader" ) func (suite *InternalAPISuite) TestUploads() { - suite.Run("Received message for upload", func() { + + suite.Run("Received status for upload, read tag without event queue", func() { orders := factory.BuildOrder(suite.DB(), factory.GetTraitActiveServiceMemberUser(), nil) uploadUser1 := factory.BuildUserUpload(suite.DB(), []factory.Customization{ { @@ -38,4 +41,49 @@ func (suite *InternalAPISuite) TestUploads() { suite.Equal("text/event-stream", rr.Header().Get("content-type")) suite.Equal("id: 0\nevent: message\ndata: CLEAN\n\nid: 1\nevent: close\ndata: Connection closed\n\n", rr.Body.String()) }) + + suite.Run("Received statuses for upload, receiving multiple statuses with event queue", func() { + orders := factory.BuildOrder(suite.DB(), factory.GetTraitActiveServiceMemberUser(), nil) + uploadUser1 := factory.BuildUserUpload(suite.DB(), []factory.Customization{ + { + Model: orders.UploadedOrders, + LinkOnly: true, + }, + { + Model: models.Upload{ + Filename: "FileName", + Bytes: int64(15), + ContentType: uploader.FileTypePDF, + }, + }, + }, nil) + file := suite.Fixture("test.pdf") + _, err := suite.HandlerConfig().FileStorer().Store(uploadUser1.Upload.StorageKey, file.Data, "somehash", nil) + suite.NoError(err) + + req := suite.NewAuthenticatedMilRequest("GET", "/internal/uploads/"+uploadUser1.Upload.ID.String()+"/status", nil, orders.ServiceMember) + rr := httptest.NewRecorder() + + fakeS3, ok := suite.HandlerConfig().FileStorer().(*storageTest.FakeS3Storage) + if ok && fakeS3 != nil { + fakeS3.EmptyTags = true + } + go func() { + time.Sleep(2 * time.Second) + if ok && fakeS3 != nil { + fakeS3.EmptyTags = false + } + }() + + suite.SetupSiteHandler().ServeHTTP(rr, req) + + suite.Equal(http.StatusAccepted, rr.Code) + suite.Equal("text/event-stream", rr.Header().Get("content-type")) + + message1 := "id: 0\nevent: message\ndata: PROCESSING\n\n" + message2 := "id: 1\nevent: message\ndata: CLEAN\n\n" + messageClose := "id: 2\nevent: close\ndata: Connection closed\n\n" + + suite.Equal(message1+message2+messageClose, rr.Body.String()) + }) } diff --git a/pkg/notifications/notification_receiver.go b/pkg/notifications/notification_receiver.go index e6eba10a5e7..82bc32a02a8 100644 --- a/pkg/notifications/notification_receiver.go +++ b/pkg/notifications/notification_receiver.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "log" "strings" "github.com/aws/aws-sdk-go-v2/aws" @@ -21,7 +20,7 @@ import ( // NotificationQueueParams stores the params for queue creation type NotificationQueueParams struct { SubscriptionTopicName string - NamePrefix string + NamePrefix QueuePrefixType FilterPolicy string } @@ -30,7 +29,7 @@ type NotificationQueueParams struct { //go:generate mockery --name NotificationReceiver type NotificationReceiver interface { CreateQueueWithSubscription(appCtx appcontext.AppContext, params NotificationQueueParams) (string, error) - ReceiveMessages(appCtx appcontext.AppContext, queueUrl string) ([]ReceivedMessage, error) + ReceiveMessages(appCtx appcontext.AppContext, queueUrl string, timerContext context.Context) ([]ReceivedMessage, error) CloseoutQueue(appCtx appcontext.AppContext, queueUrl string) error GetDefaultTopic() (string, error) } @@ -46,6 +45,13 @@ type NotificationReceiverContext struct { receiverCancelMap map[string]context.CancelFunc } +// QueuePrefixType represents a prefix identifier given to a name of dynamic notification queues +type QueuePrefixType string + +const ( + QueuePrefixObjectTagsAdded QueuePrefixType = "ObjectTagsAdded" +) + type SnsClient interface { Subscribe(ctx context.Context, params *sns.SubscribeInput, optFns ...func(*sns.Options)) (*sns.SubscribeOutput, error) Unsubscribe(ctx context.Context, params *sns.UnsubscribeInput, optFns ...func(*sns.Options)) (*sns.UnsubscribeOutput, error) @@ -118,7 +124,8 @@ func (n NotificationReceiverContext) CreateQueueWithSubscription(appCtx appconte result, err := n.sqsService.CreateQueue(context.Background(), input) if err != nil { - log.Fatalf("Failed to create SQS queue, %v", err) + appCtx.Logger().Error("Failed to create SQS queue, %v", zap.Error(err)) + return "", err } subscribeInput := &sns.SubscribeInput{ @@ -132,17 +139,18 @@ func (n NotificationReceiverContext) CreateQueueWithSubscription(appCtx appconte } subscribeOutput, err := n.snsService.Subscribe(context.Background(), subscribeInput) if err != nil { - log.Fatalf("Failed to create subscription, %v", err) + appCtx.Logger().Error("Failed to create subscription, %v", zap.Error(err)) + return "", err } n.queueSubscriptionMap[*result.QueueUrl] = *subscribeOutput.SubscriptionArn - return *result.QueueUrl, err + return *result.QueueUrl, nil } // ReceiveMessages polls given queue continuously for messages for up to 20 seconds -func (n NotificationReceiverContext) ReceiveMessages(appCtx appcontext.AppContext, queueUrl string) ([]ReceivedMessage, error) { - recCtx, cancelRecCtx := context.WithCancel(context.Background()) +func (n NotificationReceiverContext) ReceiveMessages(appCtx appcontext.AppContext, queueUrl string, timerContext context.Context) ([]ReceivedMessage, error) { + recCtx, cancelRecCtx := context.WithCancel(timerContext) defer cancelRecCtx() n.receiverCancelMap[queueUrl] = cancelRecCtx @@ -151,13 +159,13 @@ func (n NotificationReceiverContext) ReceiveMessages(appCtx appcontext.AppContex MaxNumberOfMessages: 1, WaitTimeSeconds: 20, }) - if err != nil && recCtx.Err() != context.Canceled { - appCtx.Logger().Info("Couldn't get messages from queue. Error: %v\n", zap.Error(err)) - return nil, err + if errors.Is(recCtx.Err(), context.Canceled) || errors.Is(recCtx.Err(), context.DeadlineExceeded) { + return nil, recCtx.Err() } - if recCtx.Err() == context.Canceled { - return nil, recCtx.Err() + if err != nil { + appCtx.Logger().Info("Couldn't get messages from queue. Error: %v\n", zap.Error(err)) + return nil, err } receivedMessages := make([]ReceivedMessage, len(result.Messages)) @@ -207,8 +215,9 @@ func (n NotificationReceiverContext) GetDefaultTopic() (string, error) { return topicName, nil } -// InitReceiver initializes the receiver backend +// InitReceiver initializes the receiver backend, only call this once func InitReceiver(v ViperType, logger *zap.Logger) (NotificationReceiver, error) { + if v.GetString(cli.ReceiverBackendFlag) == "sns&sqs" { // Setup notification receiver service with SNS & SQS backend dependencies awsSNSRegion := v.GetString(cli.SNSRegionFlag) @@ -227,7 +236,15 @@ func InitReceiver(v ViperType, logger *zap.Logger) (NotificationReceiver, error) snsService := sns.NewFromConfig(cfg) sqsService := sqs.NewFromConfig(cfg) - return NewNotificationReceiver(v, snsService, sqsService, awsSNSRegion, awsAccountId), nil + notificationReceiver := NewNotificationReceiver(v, snsService, sqsService, awsSNSRegion, awsAccountId) + + // Remove any remaining previous notification queues on server start + err = notificationReceiver.wipeAllNotificationQueues(snsService, sqsService, logger) + if err != nil { + return nil, err + } + + return notificationReceiver, nil } return NewStubNotificationReceiver(), nil @@ -236,3 +253,54 @@ func InitReceiver(v ViperType, logger *zap.Logger) (NotificationReceiver, error) func (n NotificationReceiverContext) constructArn(awsService string, endpointName string) string { return fmt.Sprintf("arn:aws-us-gov:%s:%s:%s:%s", awsService, n.awsRegion, n.awsAccountId, endpointName) } + +// Removes ALL previously created notification queues +func (n *NotificationReceiverContext) wipeAllNotificationQueues(snsService *sns.Client, sqsService *sqs.Client, logger *zap.Logger) error { + + defaultTopic, err := n.GetDefaultTopic() + if err != nil { + return err + } + + logger.Info("Removing previous subscriptions...") + paginator := sns.NewListSubscriptionsByTopicPaginator(snsService, &sns.ListSubscriptionsByTopicInput{ + TopicArn: aws.String(n.constructArn("sns", defaultTopic)), + }) + + for paginator.HasMorePages() { + output, err := paginator.NextPage(context.Background()) + if err != nil { + return err + } + for _, subscription := range output.Subscriptions { + if strings.Contains(*subscription.Endpoint, string(QueuePrefixObjectTagsAdded)) { + logger.Info("Subscription ARN: ", zap.String("subscription arn", *subscription.SubscriptionArn)) + logger.Info("Endpoint ARN: ", zap.String("endpoint arn", *subscription.Endpoint)) + _, err = snsService.Unsubscribe(context.Background(), &sns.UnsubscribeInput{ + SubscriptionArn: subscription.SubscriptionArn, + }) + if err != nil { + return err + } + } + } + } + + logger.Info("Removing previous queues...") + result, err := sqsService.ListQueues(context.Background(), &sqs.ListQueuesInput{ + QueueNamePrefix: aws.String(string(QueuePrefixObjectTagsAdded)), + }) + if err != nil { + return err + } + + for _, url := range result.QueueUrls { + _, err = sqsService.DeleteQueue(context.Background(), &sqs.DeleteQueueInput{ + QueueUrl: &url, + }) + if err != nil { + return err + } + } + return nil +} diff --git a/pkg/notifications/notification_receiver_stub.go b/pkg/notifications/notification_receiver_stub.go index f87806b9451..b09b61363fc 100644 --- a/pkg/notifications/notification_receiver_stub.go +++ b/pkg/notifications/notification_receiver_stub.go @@ -2,6 +2,7 @@ package notifications import ( "context" + "time" "go.uber.org/zap" @@ -27,7 +28,8 @@ func (n StubNotificationReceiver) CreateQueueWithSubscription(appCtx appcontext. return "stubQueueName", nil } -func (n StubNotificationReceiver) ReceiveMessages(appCtx appcontext.AppContext, queueUrl string) ([]ReceivedMessage, error) { +func (n StubNotificationReceiver) ReceiveMessages(appCtx appcontext.AppContext, queueUrl string, timerContext context.Context) ([]ReceivedMessage, error) { + time.Sleep(2 * time.Second) messageId := "stubMessageId" body := queueUrl + ":stubMessageBody" mockMessages := make([]ReceivedMessage, 1) diff --git a/pkg/notifications/notification_receiver_test.go b/pkg/notifications/notification_receiver_test.go index e5f0bc8ee38..e3275827e21 100644 --- a/pkg/notifications/notification_receiver_test.go +++ b/pkg/notifications/notification_receiver_test.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" "testing" + "time" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/sns" @@ -110,7 +111,10 @@ func (suite *notificationReceiverSuite) TestSuccessPath() { suite.NotContains(createdQueueUrl, queueParams.NamePrefix) suite.Equal(createdQueueUrl, "stubQueueName") - receivedMessages, err := localReceiver.ReceiveMessages(suite.AppContextForTest(), createdQueueUrl) + timerContext, cancelTimerContext := context.WithTimeout(context.Background(), 2*time.Second) + defer cancelTimerContext() + + receivedMessages, err := localReceiver.ReceiveMessages(suite.AppContextForTest(), createdQueueUrl, timerContext) suite.NoError(err) suite.Len(receivedMessages, 1) suite.Equal(receivedMessages[0].MessageId, "stubMessageId") @@ -146,7 +150,10 @@ func (suite *notificationReceiverSuite) TestSuccessPath() { suite.NoError(err) suite.Equal("FakeQueueUrl", createdQueueUrl) - receivedMessages, err := receiver.ReceiveMessages(suite.AppContextForTest(), createdQueueUrl) + timerContext, cancelTimerContext := context.WithTimeout(context.Background(), 2*time.Second) + defer cancelTimerContext() + + receivedMessages, err := receiver.ReceiveMessages(suite.AppContextForTest(), createdQueueUrl, timerContext) suite.NoError(err) suite.Len(receivedMessages, 1) suite.Equal(receivedMessages[0].MessageId, "fakeMessageId") diff --git a/pkg/storage/test/s3.go b/pkg/storage/test/s3.go index 5f738e7b088..901edf370e5 100644 --- a/pkg/storage/test/s3.go +++ b/pkg/storage/test/s3.go @@ -17,6 +17,7 @@ type FakeS3Storage struct { willSucceed bool fs *afero.Afero tempFs *afero.Afero + EmptyTags bool } // Delete removes a file. @@ -93,6 +94,9 @@ func (fake *FakeS3Storage) Tags(_ string) (map[string]string, error) { "tagName": "tagValue", "av-status": "CLEAN", // Assume anti-virus run } + if fake.EmptyTags { + tags = map[string]string{} + } return tags, nil } From 41dcc68832e5f886c7b1ee746e51111f24ff16c4 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Fri, 10 Jan 2025 22:33:06 +0000 Subject: [PATCH 09/36] B-22056 - deploy to exp. --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b8d3c39da69..b5bd5920986 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,30 +40,30 @@ references: # In addition, it's common practice to disable acceptance tests and # ignore tests for dp3 deploys. See the branch settings below. - dp3-branch: &dp3-branch placeholder_branch_name + dp3-branch: &dp3-branch MAIN-B-22056_sns_sqs_deps_w_endpoint # MUST BE ONE OF: loadtest, demo, exp. # These are used to pull in env vars so the spelling matters! - dp3-env: &dp3-env placeholder_env + dp3-env: &dp3-env exp # set integration-ignore-branch to the branch if you want to IGNORE # integration tests, or `placeholder_branch_name` if you do want to # run them - integration-ignore-branch: &integration-ignore-branch placeholder_branch_name + integration-ignore-branch: &integration-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint # set integration-mtls-ignore-branch to the branch if you want to # IGNORE mtls integration tests, or `placeholder_branch_name` if you # do want to run them - integration-mtls-ignore-branch: &integration-mtls-ignore-branch placeholder_branch_name + integration-mtls-ignore-branch: &integration-mtls-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint # set client-ignore-branch to the branch if you want to IGNORE # client tests, or `placeholder_branch_name` if you do want to run # them - client-ignore-branch: &client-ignore-branch placeholder_branch_name + client-ignore-branch: &client-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint # set server-ignore-branch to the branch if you want to IGNORE # server tests, or `placeholder_branch_name` if you do want to run # them - server-ignore-branch: &server-ignore-branch placeholder_branch_name + server-ignore-branch: &server-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint executors: base_small: From 51740dc03b9ba20ad1dd5d0f669d7cf37df37aaf Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Fri, 10 Jan 2025 23:27:54 +0000 Subject: [PATCH 10/36] B-22056 - restore exp env. --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b5bd5920986..b8d3c39da69 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,30 +40,30 @@ references: # In addition, it's common practice to disable acceptance tests and # ignore tests for dp3 deploys. See the branch settings below. - dp3-branch: &dp3-branch MAIN-B-22056_sns_sqs_deps_w_endpoint + dp3-branch: &dp3-branch placeholder_branch_name # MUST BE ONE OF: loadtest, demo, exp. # These are used to pull in env vars so the spelling matters! - dp3-env: &dp3-env exp + dp3-env: &dp3-env placeholder_env # set integration-ignore-branch to the branch if you want to IGNORE # integration tests, or `placeholder_branch_name` if you do want to # run them - integration-ignore-branch: &integration-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint + integration-ignore-branch: &integration-ignore-branch placeholder_branch_name # set integration-mtls-ignore-branch to the branch if you want to # IGNORE mtls integration tests, or `placeholder_branch_name` if you # do want to run them - integration-mtls-ignore-branch: &integration-mtls-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint + integration-mtls-ignore-branch: &integration-mtls-ignore-branch placeholder_branch_name # set client-ignore-branch to the branch if you want to IGNORE # client tests, or `placeholder_branch_name` if you do want to run # them - client-ignore-branch: &client-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint + client-ignore-branch: &client-ignore-branch placeholder_branch_name # set server-ignore-branch to the branch if you want to IGNORE # server tests, or `placeholder_branch_name` if you do want to run # them - server-ignore-branch: &server-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint + server-ignore-branch: &server-ignore-branch placeholder_branch_name executors: base_small: From 20f887c234d415ac61754f6328118a6caec43021 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Mon, 13 Jan 2025 22:06:49 +0000 Subject: [PATCH 11/36] B-22056 - fix tests. --- cmd/milmove/serve.go | 2 +- pkg/handlers/internalapi/uploads.go | 15 ++++++------ .../routing/internalapi_test/uploads_test.go | 2 +- .../mocks/NotificationReceiver.go | 23 +++++++++++-------- pkg/notifications/notification_receiver.go | 23 +++++++++++-------- .../notification_receiver_test.go | 16 ++++++++++--- 6 files changed, 49 insertions(+), 32 deletions(-) diff --git a/cmd/milmove/serve.go b/cmd/milmove/serve.go index 7d4e28a9918..a19f4b2444f 100644 --- a/cmd/milmove/serve.go +++ b/cmd/milmove/serve.go @@ -479,7 +479,7 @@ func buildRoutingConfig(appCtx appcontext.AppContext, v *viper.Viper, redisPool } // Notification Receiver - notificationReceiver, err := notifications.InitReceiver(v, appCtx.Logger()) + notificationReceiver, err := notifications.InitReceiver(v, appCtx.Logger(), true) if err != nil { appCtx.Logger().Fatal("notification receiver not enabled", zap.Error(err)) } diff --git a/pkg/handlers/internalapi/uploads.go b/pkg/handlers/internalapi/uploads.go index a1bff90b220..e4968707b7b 100644 --- a/pkg/handlers/internalapi/uploads.go +++ b/pkg/handlers/internalapi/uploads.go @@ -302,15 +302,14 @@ func (o *CustomGetUploadStatusResponse) WriteResponse(rw http.ResponseWriter, pr uploadStatus = AVStatusTypePROCESSING } + // Limitation: once the status code header has been written (first response), we are not able to update the status for subsequent responses. + // Standard 200 OK used with common SSE paradigm + rw.WriteHeader(http.StatusOK) if uploadStatus == AVStatusTypeCLEAN || uploadStatus == AVStatusTypeINFECTED { - rw.WriteHeader(http.StatusOK) o.writeEventStreamMessage(rw, producer, 0, "message", string(uploadStatus)) o.writeEventStreamMessage(rw, producer, 1, "close", "Connection closed") return // skip notification loop since object already tagged from anti-virus } else { - // Limitation: once the status code header has been written (first response), we are not able to update the status for subsequent responses. - // StatusAccepted: Standard code 202 for accepted request, but response not yet ready. - rw.WriteHeader(http.StatusAccepted) o.writeEventStreamMessage(rw, producer, 0, "message", string(uploadStatus)) } @@ -345,7 +344,11 @@ func (o *CustomGetUploadStatusResponse) WriteResponse(rw http.ResponseWriter, pr // For loop over 120 seconds, cancel context when done and it breaks the loop totalReceiverContext, totalReceiverContextCancelFunc := context.WithTimeout(context.Background(), 120*time.Second) - defer totalReceiverContextCancelFunc() + defer func() { + id_counter++ + o.writeEventStreamMessage(rw, producer, id_counter, "close", "Connection closed") + totalReceiverContextCancelFunc() + }() // Cleanup if client closes connection go func() { @@ -356,8 +359,6 @@ func (o *CustomGetUploadStatusResponse) WriteResponse(rw http.ResponseWriter, pr // Cleanup at end of work go func() { <-totalReceiverContext.Done() - id_counter++ - o.writeEventStreamMessage(rw, producer, id_counter, "close", "Connection closed") _ = o.receiver.CloseoutQueue(o.appCtx, queueUrl) }() diff --git a/pkg/handlers/routing/internalapi_test/uploads_test.go b/pkg/handlers/routing/internalapi_test/uploads_test.go index 5b760f740bc..382cd74a5bf 100644 --- a/pkg/handlers/routing/internalapi_test/uploads_test.go +++ b/pkg/handlers/routing/internalapi_test/uploads_test.go @@ -77,7 +77,7 @@ func (suite *InternalAPISuite) TestUploads() { suite.SetupSiteHandler().ServeHTTP(rr, req) - suite.Equal(http.StatusAccepted, rr.Code) + suite.Equal(http.StatusOK, rr.Code) suite.Equal("text/event-stream", rr.Header().Get("content-type")) message1 := "id: 0\nevent: message\ndata: PROCESSING\n\n" diff --git a/pkg/notifications/mocks/NotificationReceiver.go b/pkg/notifications/mocks/NotificationReceiver.go index df8329e5f60..04c7d931659 100644 --- a/pkg/notifications/mocks/NotificationReceiver.go +++ b/pkg/notifications/mocks/NotificationReceiver.go @@ -3,9 +3,12 @@ package mocks import ( - mock "github.com/stretchr/testify/mock" + context "context" + appcontext "github.com/transcom/mymove/pkg/appcontext" + mock "github.com/stretchr/testify/mock" + notifications "github.com/transcom/mymove/pkg/notifications" ) @@ -88,9 +91,9 @@ func (_m *NotificationReceiver) GetDefaultTopic() (string, error) { return r0, r1 } -// ReceiveMessages provides a mock function with given fields: appCtx, queueUrl -func (_m *NotificationReceiver) ReceiveMessages(appCtx appcontext.AppContext, queueUrl string) ([]notifications.ReceivedMessage, error) { - ret := _m.Called(appCtx, queueUrl) +// ReceiveMessages provides a mock function with given fields: appCtx, queueUrl, timerContext +func (_m *NotificationReceiver) ReceiveMessages(appCtx appcontext.AppContext, queueUrl string, timerContext context.Context) ([]notifications.ReceivedMessage, error) { + ret := _m.Called(appCtx, queueUrl, timerContext) if len(ret) == 0 { panic("no return value specified for ReceiveMessages") @@ -98,19 +101,19 @@ func (_m *NotificationReceiver) ReceiveMessages(appCtx appcontext.AppContext, qu var r0 []notifications.ReceivedMessage var r1 error - if rf, ok := ret.Get(0).(func(appcontext.AppContext, string) ([]notifications.ReceivedMessage, error)); ok { - return rf(appCtx, queueUrl) + if rf, ok := ret.Get(0).(func(appcontext.AppContext, string, context.Context) ([]notifications.ReceivedMessage, error)); ok { + return rf(appCtx, queueUrl, timerContext) } - if rf, ok := ret.Get(0).(func(appcontext.AppContext, string) []notifications.ReceivedMessage); ok { - r0 = rf(appCtx, queueUrl) + if rf, ok := ret.Get(0).(func(appcontext.AppContext, string, context.Context) []notifications.ReceivedMessage); ok { + r0 = rf(appCtx, queueUrl, timerContext) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]notifications.ReceivedMessage) } } - if rf, ok := ret.Get(1).(func(appcontext.AppContext, string) error); ok { - r1 = rf(appCtx, queueUrl) + if rf, ok := ret.Get(1).(func(appcontext.AppContext, string, context.Context) error); ok { + r1 = rf(appCtx, queueUrl, timerContext) } else { r1 = ret.Error(1) } diff --git a/pkg/notifications/notification_receiver.go b/pkg/notifications/notification_receiver.go index 82bc32a02a8..09f9cd8b072 100644 --- a/pkg/notifications/notification_receiver.go +++ b/pkg/notifications/notification_receiver.go @@ -55,12 +55,14 @@ const ( type SnsClient interface { Subscribe(ctx context.Context, params *sns.SubscribeInput, optFns ...func(*sns.Options)) (*sns.SubscribeOutput, error) Unsubscribe(ctx context.Context, params *sns.UnsubscribeInput, optFns ...func(*sns.Options)) (*sns.UnsubscribeOutput, error) + ListSubscriptionsByTopic(context.Context, *sns.ListSubscriptionsByTopicInput, ...func(*sns.Options)) (*sns.ListSubscriptionsByTopicOutput, error) } type SqsClient interface { CreateQueue(ctx context.Context, params *sqs.CreateQueueInput, optFns ...func(*sqs.Options)) (*sqs.CreateQueueOutput, error) ReceiveMessage(ctx context.Context, params *sqs.ReceiveMessageInput, optFns ...func(*sqs.Options)) (*sqs.ReceiveMessageOutput, error) DeleteQueue(ctx context.Context, params *sqs.DeleteQueueInput, optFns ...func(*sqs.Options)) (*sqs.DeleteQueueOutput, error) + ListQueues(ctx context.Context, params *sqs.ListQueuesInput, optFns ...func(*sqs.Options)) (*sqs.ListQueuesOutput, error) } type ViperType interface { @@ -216,7 +218,7 @@ func (n NotificationReceiverContext) GetDefaultTopic() (string, error) { } // InitReceiver initializes the receiver backend, only call this once -func InitReceiver(v ViperType, logger *zap.Logger) (NotificationReceiver, error) { +func InitReceiver(v ViperType, logger *zap.Logger, wipeAllNotificationQueues bool) (NotificationReceiver, error) { if v.GetString(cli.ReceiverBackendFlag) == "sns&sqs" { // Setup notification receiver service with SNS & SQS backend dependencies @@ -239,9 +241,11 @@ func InitReceiver(v ViperType, logger *zap.Logger) (NotificationReceiver, error) notificationReceiver := NewNotificationReceiver(v, snsService, sqsService, awsSNSRegion, awsAccountId) // Remove any remaining previous notification queues on server start - err = notificationReceiver.wipeAllNotificationQueues(snsService, sqsService, logger) - if err != nil { - return nil, err + if wipeAllNotificationQueues { + err = notificationReceiver.wipeAllNotificationQueues(logger) + if err != nil { + return nil, err + } } return notificationReceiver, nil @@ -255,15 +259,14 @@ func (n NotificationReceiverContext) constructArn(awsService string, endpointNam } // Removes ALL previously created notification queues -func (n *NotificationReceiverContext) wipeAllNotificationQueues(snsService *sns.Client, sqsService *sqs.Client, logger *zap.Logger) error { - +func (n *NotificationReceiverContext) wipeAllNotificationQueues(logger *zap.Logger) error { defaultTopic, err := n.GetDefaultTopic() if err != nil { return err } logger.Info("Removing previous subscriptions...") - paginator := sns.NewListSubscriptionsByTopicPaginator(snsService, &sns.ListSubscriptionsByTopicInput{ + paginator := sns.NewListSubscriptionsByTopicPaginator(n.snsService, &sns.ListSubscriptionsByTopicInput{ TopicArn: aws.String(n.constructArn("sns", defaultTopic)), }) @@ -276,7 +279,7 @@ func (n *NotificationReceiverContext) wipeAllNotificationQueues(snsService *sns. if strings.Contains(*subscription.Endpoint, string(QueuePrefixObjectTagsAdded)) { logger.Info("Subscription ARN: ", zap.String("subscription arn", *subscription.SubscriptionArn)) logger.Info("Endpoint ARN: ", zap.String("endpoint arn", *subscription.Endpoint)) - _, err = snsService.Unsubscribe(context.Background(), &sns.UnsubscribeInput{ + _, err = n.snsService.Unsubscribe(context.Background(), &sns.UnsubscribeInput{ SubscriptionArn: subscription.SubscriptionArn, }) if err != nil { @@ -287,7 +290,7 @@ func (n *NotificationReceiverContext) wipeAllNotificationQueues(snsService *sns. } logger.Info("Removing previous queues...") - result, err := sqsService.ListQueues(context.Background(), &sqs.ListQueuesInput{ + result, err := n.sqsService.ListQueues(context.Background(), &sqs.ListQueuesInput{ QueueNamePrefix: aws.String(string(QueuePrefixObjectTagsAdded)), }) if err != nil { @@ -295,7 +298,7 @@ func (n *NotificationReceiverContext) wipeAllNotificationQueues(snsService *sns. } for _, url := range result.QueueUrls { - _, err = sqsService.DeleteQueue(context.Background(), &sqs.DeleteQueueInput{ + _, err = n.sqsService.DeleteQueue(context.Background(), &sqs.DeleteQueueInput{ QueueUrl: &url, }) if err != nil { diff --git a/pkg/notifications/notification_receiver_test.go b/pkg/notifications/notification_receiver_test.go index e3275827e21..a996a67ce4e 100644 --- a/pkg/notifications/notification_receiver_test.go +++ b/pkg/notifications/notification_receiver_test.go @@ -66,6 +66,10 @@ func (_m *MockSnsClient) Unsubscribe(ctx context.Context, params *sns.Unsubscrib return &sns.UnsubscribeOutput{}, nil } +func (_m *MockSnsClient) ListSubscriptionsByTopic(context.Context, *sns.ListSubscriptionsByTopicInput, ...func(*sns.Options)) (*sns.ListSubscriptionsByTopicOutput, error) { + return &sns.ListSubscriptionsByTopicOutput{}, nil +} + // mock - SQS type MockSqsClient struct { mock.Mock @@ -90,11 +94,15 @@ func (_m *MockSqsClient) DeleteQueue(ctx context.Context, params *sqs.DeleteQueu return &sqs.DeleteQueueOutput{}, nil } +func (_m *MockSqsClient) ListQueues(ctx context.Context, params *sqs.ListQueuesInput, optFns ...func(*sqs.Options)) (*sqs.ListQueuesOutput, error) { + return &sqs.ListQueuesOutput{}, nil +} + func (suite *notificationReceiverSuite) TestSuccessPath() { suite.Run("local backend - notification receiver stub", func() { v := viper.New() - localReceiver, err := InitReceiver(v, suite.Logger()) + localReceiver, err := InitReceiver(v, suite.Logger(), true) suite.NoError(err) suite.IsType(StubNotificationReceiver{}, localReceiver) @@ -121,10 +129,12 @@ func (suite *notificationReceiverSuite) TestSuccessPath() { suite.Equal(*receivedMessages[0].Body, fmt.Sprintf("%s:stubMessageBody", createdQueueUrl)) }) - suite.Run("aws backend - notification receiver init", func() { + suite.Run("aws backend - notification receiver InitReceiver", func() { v := Viper{} - receiver, _ := InitReceiver(&v, suite.Logger()) + receiver, err := InitReceiver(&v, suite.Logger(), false) + + suite.NoError(err) suite.IsType(NotificationReceiverContext{}, receiver) defaultTopic, err := receiver.GetDefaultTopic() suite.Equal("fake_sns_topic", defaultTopic) From 0964e6195f9bd74a3c7295dd6079df941ffae393 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Tue, 14 Jan 2025 17:53:36 +0000 Subject: [PATCH 12/36] B-22056 - using generated mocks for unit tests instead. --- .../mocks/NotificationReceiver.go | 136 ------------- pkg/notifications/notification_receiver.go | 5 +- .../notification_receiver_test.go | 124 +++++------- pkg/notifications/receiverMocks/SnsClient.go | 141 ++++++++++++++ pkg/notifications/receiverMocks/SqsClient.go | 178 ++++++++++++++++++ pkg/notifications/receiverMocks/ViperType.go | 51 +++++ 6 files changed, 420 insertions(+), 215 deletions(-) delete mode 100644 pkg/notifications/mocks/NotificationReceiver.go create mode 100644 pkg/notifications/receiverMocks/SnsClient.go create mode 100644 pkg/notifications/receiverMocks/SqsClient.go create mode 100644 pkg/notifications/receiverMocks/ViperType.go diff --git a/pkg/notifications/mocks/NotificationReceiver.go b/pkg/notifications/mocks/NotificationReceiver.go deleted file mode 100644 index 04c7d931659..00000000000 --- a/pkg/notifications/mocks/NotificationReceiver.go +++ /dev/null @@ -1,136 +0,0 @@ -// Code generated by mockery. DO NOT EDIT. - -package mocks - -import ( - context "context" - - appcontext "github.com/transcom/mymove/pkg/appcontext" - - mock "github.com/stretchr/testify/mock" - - notifications "github.com/transcom/mymove/pkg/notifications" -) - -// NotificationReceiver is an autogenerated mock type for the NotificationReceiver type -type NotificationReceiver struct { - mock.Mock -} - -// CloseoutQueue provides a mock function with given fields: appCtx, queueUrl -func (_m *NotificationReceiver) CloseoutQueue(appCtx appcontext.AppContext, queueUrl string) error { - ret := _m.Called(appCtx, queueUrl) - - if len(ret) == 0 { - panic("no return value specified for CloseoutQueue") - } - - var r0 error - if rf, ok := ret.Get(0).(func(appcontext.AppContext, string) error); ok { - r0 = rf(appCtx, queueUrl) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// CreateQueueWithSubscription provides a mock function with given fields: appCtx, params -func (_m *NotificationReceiver) CreateQueueWithSubscription(appCtx appcontext.AppContext, params notifications.NotificationQueueParams) (string, error) { - ret := _m.Called(appCtx, params) - - if len(ret) == 0 { - panic("no return value specified for CreateQueueWithSubscription") - } - - var r0 string - var r1 error - if rf, ok := ret.Get(0).(func(appcontext.AppContext, notifications.NotificationQueueParams) (string, error)); ok { - return rf(appCtx, params) - } - if rf, ok := ret.Get(0).(func(appcontext.AppContext, notifications.NotificationQueueParams) string); ok { - r0 = rf(appCtx, params) - } else { - r0 = ret.Get(0).(string) - } - - if rf, ok := ret.Get(1).(func(appcontext.AppContext, notifications.NotificationQueueParams) error); ok { - r1 = rf(appCtx, params) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetDefaultTopic provides a mock function with given fields: -func (_m *NotificationReceiver) GetDefaultTopic() (string, error) { - ret := _m.Called() - - if len(ret) == 0 { - panic("no return value specified for GetDefaultTopic") - } - - var r0 string - var r1 error - if rf, ok := ret.Get(0).(func() (string, error)); ok { - return rf() - } - if rf, ok := ret.Get(0).(func() string); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(string) - } - - if rf, ok := ret.Get(1).(func() error); ok { - r1 = rf() - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// ReceiveMessages provides a mock function with given fields: appCtx, queueUrl, timerContext -func (_m *NotificationReceiver) ReceiveMessages(appCtx appcontext.AppContext, queueUrl string, timerContext context.Context) ([]notifications.ReceivedMessage, error) { - ret := _m.Called(appCtx, queueUrl, timerContext) - - if len(ret) == 0 { - panic("no return value specified for ReceiveMessages") - } - - var r0 []notifications.ReceivedMessage - var r1 error - if rf, ok := ret.Get(0).(func(appcontext.AppContext, string, context.Context) ([]notifications.ReceivedMessage, error)); ok { - return rf(appCtx, queueUrl, timerContext) - } - if rf, ok := ret.Get(0).(func(appcontext.AppContext, string, context.Context) []notifications.ReceivedMessage); ok { - r0 = rf(appCtx, queueUrl, timerContext) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]notifications.ReceivedMessage) - } - } - - if rf, ok := ret.Get(1).(func(appcontext.AppContext, string, context.Context) error); ok { - r1 = rf(appCtx, queueUrl, timerContext) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// NewNotificationReceiver creates a new instance of NotificationReceiver. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewNotificationReceiver(t interface { - mock.TestingT - Cleanup(func()) -}) *NotificationReceiver { - mock := &NotificationReceiver{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/pkg/notifications/notification_receiver.go b/pkg/notifications/notification_receiver.go index 09f9cd8b072..76c9d3bebbe 100644 --- a/pkg/notifications/notification_receiver.go +++ b/pkg/notifications/notification_receiver.go @@ -25,8 +25,6 @@ type NotificationQueueParams struct { } // NotificationReceiver is an interface for receiving notifications -// -//go:generate mockery --name NotificationReceiver type NotificationReceiver interface { CreateQueueWithSubscription(appCtx appcontext.AppContext, params NotificationQueueParams) (string, error) ReceiveMessages(appCtx appcontext.AppContext, queueUrl string, timerContext context.Context) ([]ReceivedMessage, error) @@ -52,12 +50,14 @@ const ( QueuePrefixObjectTagsAdded QueuePrefixType = "ObjectTagsAdded" ) +//go:generate mockery --name SnsClient --output ./receiverMocks type SnsClient interface { Subscribe(ctx context.Context, params *sns.SubscribeInput, optFns ...func(*sns.Options)) (*sns.SubscribeOutput, error) Unsubscribe(ctx context.Context, params *sns.UnsubscribeInput, optFns ...func(*sns.Options)) (*sns.UnsubscribeOutput, error) ListSubscriptionsByTopic(context.Context, *sns.ListSubscriptionsByTopicInput, ...func(*sns.Options)) (*sns.ListSubscriptionsByTopicOutput, error) } +//go:generate mockery --name SqsClient --output ./receiverMocks type SqsClient interface { CreateQueue(ctx context.Context, params *sqs.CreateQueueInput, optFns ...func(*sqs.Options)) (*sqs.CreateQueueOutput, error) ReceiveMessage(ctx context.Context, params *sqs.ReceiveMessageInput, optFns ...func(*sqs.Options)) (*sqs.ReceiveMessageOutput, error) @@ -65,6 +65,7 @@ type SqsClient interface { ListQueues(ctx context.Context, params *sqs.ListQueuesInput, optFns ...func(*sqs.Options)) (*sqs.ListQueuesOutput, error) } +//go:generate mockery --name ViperType --output ./receiverMocks type ViperType interface { GetString(string) string SetEnvKeyReplacer(*strings.Replacer) diff --git a/pkg/notifications/notification_receiver_test.go b/pkg/notifications/notification_receiver_test.go index a996a67ce4e..e895a7f2e3b 100644 --- a/pkg/notifications/notification_receiver_test.go +++ b/pkg/notifications/notification_receiver_test.go @@ -3,7 +3,6 @@ package notifications import ( "context" "fmt" - "strings" "testing" "time" @@ -11,11 +10,11 @@ import ( "github.com/aws/aws-sdk-go-v2/service/sns" "github.com/aws/aws-sdk-go-v2/service/sqs" "github.com/aws/aws-sdk-go-v2/service/sqs/types" - "github.com/spf13/viper" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" "github.com/transcom/mymove/pkg/cli" + mocks "github.com/transcom/mymove/pkg/notifications/receiverMocks" "github.com/transcom/mymove/pkg/testingsuite" ) @@ -33,76 +32,16 @@ func TestNotificationReceiverSuite(t *testing.T) { hs.PopTestSuite.TearDown() } -// mock - Viper -type Viper struct { - mock.Mock -} - -func (_m *Viper) GetString(key string) string { - switch key { - case cli.ReceiverBackendFlag: - return "sns&sqs" - case cli.SNSRegionFlag: - return "us-gov-west-1" - case cli.SNSAccountId: - return "12345" - case cli.SNSTagsUpdatedTopicFlag: - return "fake_sns_topic" - } - return "" -} -func (_m *Viper) SetEnvKeyReplacer(_ *strings.Replacer) {} - -// mock - SNS -type MockSnsClient struct { - mock.Mock -} - -func (_m *MockSnsClient) Subscribe(ctx context.Context, params *sns.SubscribeInput, optFns ...func(*sns.Options)) (*sns.SubscribeOutput, error) { - return &sns.SubscribeOutput{SubscriptionArn: aws.String("FakeSubscriptionArn")}, nil -} - -func (_m *MockSnsClient) Unsubscribe(ctx context.Context, params *sns.UnsubscribeInput, optFns ...func(*sns.Options)) (*sns.UnsubscribeOutput, error) { - return &sns.UnsubscribeOutput{}, nil -} - -func (_m *MockSnsClient) ListSubscriptionsByTopic(context.Context, *sns.ListSubscriptionsByTopicInput, ...func(*sns.Options)) (*sns.ListSubscriptionsByTopicOutput, error) { - return &sns.ListSubscriptionsByTopicOutput{}, nil -} - -// mock - SQS -type MockSqsClient struct { - mock.Mock -} - -func (_m *MockSqsClient) CreateQueue(ctx context.Context, params *sqs.CreateQueueInput, optFns ...func(*sqs.Options)) (*sqs.CreateQueueOutput, error) { - return &sqs.CreateQueueOutput{ - QueueUrl: aws.String("FakeQueueUrl"), - }, nil -} -func (_m *MockSqsClient) ReceiveMessage(ctx context.Context, params *sqs.ReceiveMessageInput, optFns ...func(*sqs.Options)) (*sqs.ReceiveMessageOutput, error) { - messages := make([]types.Message, 0) - messages = append(messages, types.Message{ - MessageId: aws.String("fakeMessageId"), - Body: aws.String(*params.QueueUrl + ":fakeMessageBody"), - }) - return &sqs.ReceiveMessageOutput{ - Messages: messages, - }, nil -} -func (_m *MockSqsClient) DeleteQueue(ctx context.Context, params *sqs.DeleteQueueInput, optFns ...func(*sqs.Options)) (*sqs.DeleteQueueOutput, error) { - return &sqs.DeleteQueueOutput{}, nil -} - -func (_m *MockSqsClient) ListQueues(ctx context.Context, params *sqs.ListQueuesInput, optFns ...func(*sqs.Options)) (*sqs.ListQueuesOutput, error) { - return &sqs.ListQueuesOutput{}, nil -} - func (suite *notificationReceiverSuite) TestSuccessPath() { suite.Run("local backend - notification receiver stub", func() { - v := viper.New() - localReceiver, err := InitReceiver(v, suite.Logger(), true) + // Setup mocks + mockedViper := mocks.ViperType{} + mockedViper.On("GetString", cli.ReceiverBackendFlag).Return("local") + mockedViper.On("GetString", cli.SNSRegionFlag).Return("us-gov-west-1") + mockedViper.On("GetString", cli.SNSAccountId).Return("12345") + mockedViper.On("GetString", cli.SNSTagsUpdatedTopicFlag).Return("fake_sns_topic") + localReceiver, err := InitReceiver(&mockedViper, suite.Logger(), true) suite.NoError(err) suite.IsType(StubNotificationReceiver{}, localReceiver) @@ -130,9 +69,14 @@ func (suite *notificationReceiverSuite) TestSuccessPath() { }) suite.Run("aws backend - notification receiver InitReceiver", func() { - v := Viper{} + // Setup mocks + mockedViper := mocks.ViperType{} + mockedViper.On("GetString", cli.ReceiverBackendFlag).Return("sns&sqs") + mockedViper.On("GetString", cli.SNSRegionFlag).Return("us-gov-west-1") + mockedViper.On("GetString", cli.SNSAccountId).Return("12345") + mockedViper.On("GetString", cli.SNSTagsUpdatedTopicFlag).Return("fake_sns_topic") - receiver, err := InitReceiver(&v, suite.Logger(), false) + receiver, err := InitReceiver(&mockedViper, suite.Logger(), false) suite.NoError(err) suite.IsType(NotificationReceiverContext{}, receiver) @@ -142,11 +86,37 @@ func (suite *notificationReceiverSuite) TestSuccessPath() { }) suite.Run("aws backend - notification receiver with mock services", func() { - v := Viper{} - snsService := MockSnsClient{} - sqsService := MockSqsClient{} - - receiver := NewNotificationReceiver(&v, &snsService, &sqsService, "", "") + // Setup mocks + mockedViper := mocks.ViperType{} + mockedViper.On("GetString", cli.ReceiverBackendFlag).Return("sns&sqs") + mockedViper.On("GetString", cli.SNSRegionFlag).Return("us-gov-west-1") + mockedViper.On("GetString", cli.SNSAccountId).Return("12345") + mockedViper.On("GetString", cli.SNSTagsUpdatedTopicFlag).Return("fake_sns_topic") + + mockedSns := mocks.SnsClient{} + mockedSns.On("Subscribe", mock.Anything, mock.AnythingOfType("*sns.SubscribeInput")).Return(&sns.SubscribeOutput{ + SubscriptionArn: aws.String("FakeSubscriptionArn"), + }, nil) + mockedSns.On("Unsubscribe", mock.Anything, mock.AnythingOfType("*sns.UnsubscribeInput")).Return(&sns.UnsubscribeOutput{}, nil) + mockedSns.On("ListSubscriptionsByTopic", mock.Anything, mock.AnythingOfType("*sns.ListSubscriptionsByTopicInput")).Return(&sns.ListSubscriptionsByTopicOutput{}, nil) + + mockedSqs := mocks.SqsClient{} + mockedSqs.On("CreateQueue", mock.Anything, mock.AnythingOfType("*sqs.CreateQueueInput")).Return(&sqs.CreateQueueOutput{ + QueueUrl: aws.String("fakeQueueUrl"), + }, nil) + mockedSqs.On("ReceiveMessage", mock.Anything, mock.AnythingOfType("*sqs.ReceiveMessageInput")).Return(&sqs.ReceiveMessageOutput{ + Messages: []types.Message{ + { + MessageId: aws.String("fakeMessageId"), + Body: aws.String("fakeQueueUrl:fakeMessageBody"), + }, + }, + }, nil) + mockedSqs.On("DeleteQueue", mock.Anything, mock.AnythingOfType("*sqs.DeleteQueueInput")).Return(&sqs.DeleteQueueOutput{}, nil) + mockedSqs.On("ListQueues", mock.Anything, mock.AnythingOfType("*sqs.ListQueuesInput")).Return(&sqs.ListQueuesOutput{}, nil) + + // Run test + receiver := NewNotificationReceiver(&mockedViper, &mockedSns, &mockedSqs, "", "") suite.IsType(NotificationReceiverContext{}, receiver) defaultTopic, err := receiver.GetDefaultTopic() @@ -158,7 +128,7 @@ func (suite *notificationReceiverSuite) TestSuccessPath() { } createdQueueUrl, err := receiver.CreateQueueWithSubscription(suite.AppContextForTest(), queueParams) suite.NoError(err) - suite.Equal("FakeQueueUrl", createdQueueUrl) + suite.Equal("fakeQueueUrl", createdQueueUrl) timerContext, cancelTimerContext := context.WithTimeout(context.Background(), 2*time.Second) defer cancelTimerContext() diff --git a/pkg/notifications/receiverMocks/SnsClient.go b/pkg/notifications/receiverMocks/SnsClient.go new file mode 100644 index 00000000000..0c562896a0d --- /dev/null +++ b/pkg/notifications/receiverMocks/SnsClient.go @@ -0,0 +1,141 @@ +// Code generated by mockery. DO NOT EDIT. + +package mocks + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" + + sns "github.com/aws/aws-sdk-go-v2/service/sns" +) + +// SnsClient is an autogenerated mock type for the SnsClient type +type SnsClient struct { + mock.Mock +} + +// ListSubscriptionsByTopic provides a mock function with given fields: _a0, _a1, _a2 +func (_m *SnsClient) ListSubscriptionsByTopic(_a0 context.Context, _a1 *sns.ListSubscriptionsByTopicInput, _a2 ...func(*sns.Options)) (*sns.ListSubscriptionsByTopicOutput, error) { + _va := make([]interface{}, len(_a2)) + for _i := range _a2 { + _va[_i] = _a2[_i] + } + var _ca []interface{} + _ca = append(_ca, _a0, _a1) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + if len(ret) == 0 { + panic("no return value specified for ListSubscriptionsByTopic") + } + + var r0 *sns.ListSubscriptionsByTopicOutput + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *sns.ListSubscriptionsByTopicInput, ...func(*sns.Options)) (*sns.ListSubscriptionsByTopicOutput, error)); ok { + return rf(_a0, _a1, _a2...) + } + if rf, ok := ret.Get(0).(func(context.Context, *sns.ListSubscriptionsByTopicInput, ...func(*sns.Options)) *sns.ListSubscriptionsByTopicOutput); ok { + r0 = rf(_a0, _a1, _a2...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*sns.ListSubscriptionsByTopicOutput) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *sns.ListSubscriptionsByTopicInput, ...func(*sns.Options)) error); ok { + r1 = rf(_a0, _a1, _a2...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Subscribe provides a mock function with given fields: ctx, params, optFns +func (_m *SnsClient) Subscribe(ctx context.Context, params *sns.SubscribeInput, optFns ...func(*sns.Options)) (*sns.SubscribeOutput, error) { + _va := make([]interface{}, len(optFns)) + for _i := range optFns { + _va[_i] = optFns[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, params) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + if len(ret) == 0 { + panic("no return value specified for Subscribe") + } + + var r0 *sns.SubscribeOutput + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *sns.SubscribeInput, ...func(*sns.Options)) (*sns.SubscribeOutput, error)); ok { + return rf(ctx, params, optFns...) + } + if rf, ok := ret.Get(0).(func(context.Context, *sns.SubscribeInput, ...func(*sns.Options)) *sns.SubscribeOutput); ok { + r0 = rf(ctx, params, optFns...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*sns.SubscribeOutput) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *sns.SubscribeInput, ...func(*sns.Options)) error); ok { + r1 = rf(ctx, params, optFns...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Unsubscribe provides a mock function with given fields: ctx, params, optFns +func (_m *SnsClient) Unsubscribe(ctx context.Context, params *sns.UnsubscribeInput, optFns ...func(*sns.Options)) (*sns.UnsubscribeOutput, error) { + _va := make([]interface{}, len(optFns)) + for _i := range optFns { + _va[_i] = optFns[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, params) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + if len(ret) == 0 { + panic("no return value specified for Unsubscribe") + } + + var r0 *sns.UnsubscribeOutput + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *sns.UnsubscribeInput, ...func(*sns.Options)) (*sns.UnsubscribeOutput, error)); ok { + return rf(ctx, params, optFns...) + } + if rf, ok := ret.Get(0).(func(context.Context, *sns.UnsubscribeInput, ...func(*sns.Options)) *sns.UnsubscribeOutput); ok { + r0 = rf(ctx, params, optFns...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*sns.UnsubscribeOutput) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *sns.UnsubscribeInput, ...func(*sns.Options)) error); ok { + r1 = rf(ctx, params, optFns...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// NewSnsClient creates a new instance of SnsClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewSnsClient(t interface { + mock.TestingT + Cleanup(func()) +}) *SnsClient { + mock := &SnsClient{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/pkg/notifications/receiverMocks/SqsClient.go b/pkg/notifications/receiverMocks/SqsClient.go new file mode 100644 index 00000000000..0ab970fc530 --- /dev/null +++ b/pkg/notifications/receiverMocks/SqsClient.go @@ -0,0 +1,178 @@ +// Code generated by mockery. DO NOT EDIT. + +package mocks + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" + + sqs "github.com/aws/aws-sdk-go-v2/service/sqs" +) + +// SqsClient is an autogenerated mock type for the SqsClient type +type SqsClient struct { + mock.Mock +} + +// CreateQueue provides a mock function with given fields: ctx, params, optFns +func (_m *SqsClient) CreateQueue(ctx context.Context, params *sqs.CreateQueueInput, optFns ...func(*sqs.Options)) (*sqs.CreateQueueOutput, error) { + _va := make([]interface{}, len(optFns)) + for _i := range optFns { + _va[_i] = optFns[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, params) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + if len(ret) == 0 { + panic("no return value specified for CreateQueue") + } + + var r0 *sqs.CreateQueueOutput + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *sqs.CreateQueueInput, ...func(*sqs.Options)) (*sqs.CreateQueueOutput, error)); ok { + return rf(ctx, params, optFns...) + } + if rf, ok := ret.Get(0).(func(context.Context, *sqs.CreateQueueInput, ...func(*sqs.Options)) *sqs.CreateQueueOutput); ok { + r0 = rf(ctx, params, optFns...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*sqs.CreateQueueOutput) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *sqs.CreateQueueInput, ...func(*sqs.Options)) error); ok { + r1 = rf(ctx, params, optFns...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// DeleteQueue provides a mock function with given fields: ctx, params, optFns +func (_m *SqsClient) DeleteQueue(ctx context.Context, params *sqs.DeleteQueueInput, optFns ...func(*sqs.Options)) (*sqs.DeleteQueueOutput, error) { + _va := make([]interface{}, len(optFns)) + for _i := range optFns { + _va[_i] = optFns[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, params) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + if len(ret) == 0 { + panic("no return value specified for DeleteQueue") + } + + var r0 *sqs.DeleteQueueOutput + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *sqs.DeleteQueueInput, ...func(*sqs.Options)) (*sqs.DeleteQueueOutput, error)); ok { + return rf(ctx, params, optFns...) + } + if rf, ok := ret.Get(0).(func(context.Context, *sqs.DeleteQueueInput, ...func(*sqs.Options)) *sqs.DeleteQueueOutput); ok { + r0 = rf(ctx, params, optFns...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*sqs.DeleteQueueOutput) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *sqs.DeleteQueueInput, ...func(*sqs.Options)) error); ok { + r1 = rf(ctx, params, optFns...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ListQueues provides a mock function with given fields: ctx, params, optFns +func (_m *SqsClient) ListQueues(ctx context.Context, params *sqs.ListQueuesInput, optFns ...func(*sqs.Options)) (*sqs.ListQueuesOutput, error) { + _va := make([]interface{}, len(optFns)) + for _i := range optFns { + _va[_i] = optFns[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, params) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + if len(ret) == 0 { + panic("no return value specified for ListQueues") + } + + var r0 *sqs.ListQueuesOutput + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *sqs.ListQueuesInput, ...func(*sqs.Options)) (*sqs.ListQueuesOutput, error)); ok { + return rf(ctx, params, optFns...) + } + if rf, ok := ret.Get(0).(func(context.Context, *sqs.ListQueuesInput, ...func(*sqs.Options)) *sqs.ListQueuesOutput); ok { + r0 = rf(ctx, params, optFns...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*sqs.ListQueuesOutput) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *sqs.ListQueuesInput, ...func(*sqs.Options)) error); ok { + r1 = rf(ctx, params, optFns...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ReceiveMessage provides a mock function with given fields: ctx, params, optFns +func (_m *SqsClient) ReceiveMessage(ctx context.Context, params *sqs.ReceiveMessageInput, optFns ...func(*sqs.Options)) (*sqs.ReceiveMessageOutput, error) { + _va := make([]interface{}, len(optFns)) + for _i := range optFns { + _va[_i] = optFns[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, params) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + if len(ret) == 0 { + panic("no return value specified for ReceiveMessage") + } + + var r0 *sqs.ReceiveMessageOutput + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *sqs.ReceiveMessageInput, ...func(*sqs.Options)) (*sqs.ReceiveMessageOutput, error)); ok { + return rf(ctx, params, optFns...) + } + if rf, ok := ret.Get(0).(func(context.Context, *sqs.ReceiveMessageInput, ...func(*sqs.Options)) *sqs.ReceiveMessageOutput); ok { + r0 = rf(ctx, params, optFns...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*sqs.ReceiveMessageOutput) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *sqs.ReceiveMessageInput, ...func(*sqs.Options)) error); ok { + r1 = rf(ctx, params, optFns...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// NewSqsClient creates a new instance of SqsClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewSqsClient(t interface { + mock.TestingT + Cleanup(func()) +}) *SqsClient { + mock := &SqsClient{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/pkg/notifications/receiverMocks/ViperType.go b/pkg/notifications/receiverMocks/ViperType.go new file mode 100644 index 00000000000..bf5e6f84090 --- /dev/null +++ b/pkg/notifications/receiverMocks/ViperType.go @@ -0,0 +1,51 @@ +// Code generated by mockery. DO NOT EDIT. + +package mocks + +import ( + mock "github.com/stretchr/testify/mock" + + strings "strings" +) + +// ViperType is an autogenerated mock type for the ViperType type +type ViperType struct { + mock.Mock +} + +// GetString provides a mock function with given fields: _a0 +func (_m *ViperType) GetString(_a0 string) string { + ret := _m.Called(_a0) + + if len(ret) == 0 { + panic("no return value specified for GetString") + } + + var r0 string + if rf, ok := ret.Get(0).(func(string) string); ok { + r0 = rf(_a0) + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} + +// SetEnvKeyReplacer provides a mock function with given fields: _a0 +func (_m *ViperType) SetEnvKeyReplacer(_a0 *strings.Replacer) { + _m.Called(_a0) +} + +// NewViperType creates a new instance of ViperType. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewViperType(t interface { + mock.TestingT + Cleanup(func()) +}) *ViperType { + mock := &ViperType{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} From 25ed4b2a63f3acd0c692b0f5d9a62d28af7549d4 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Tue, 14 Jan 2025 21:26:55 +0000 Subject: [PATCH 13/36] B-22056 - additional security for sqs based on best practices --- pkg/notifications/notification_receiver.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/pkg/notifications/notification_receiver.go b/pkg/notifications/notification_receiver.go index 76c9d3bebbe..49222a69fdb 100644 --- a/pkg/notifications/notification_receiver.go +++ b/pkg/notifications/notification_receiver.go @@ -111,11 +111,22 @@ func (n NotificationReceiverContext) CreateQueueWithSubscription(appCtx appconte "Resource": "%s", "Condition": { "ArnEquals": { - "aws:SourceArn": "%s" + "aws:SourceArn": "%s" } } + }, { + "Sid": "DenyNonSSLAccess", + "Effect": "Deny", + "Principal": "*", + "Action": "sqs:*", + "Resource": "%s", + "Condition": { + "Bool": { + "aws:SecureTransport": "false" + } + } }] - }`, queueArn, topicArn) + }`, queueArn, topicArn, queueArn) input := &sqs.CreateQueueInput{ QueueName: &queueName, From f80886efe531f4f41af5efd499614c78728a0051 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Tue, 14 Jan 2025 21:37:52 +0000 Subject: [PATCH 14/36] B-22056 - deploy to exp. --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b8d3c39da69..b5bd5920986 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,30 +40,30 @@ references: # In addition, it's common practice to disable acceptance tests and # ignore tests for dp3 deploys. See the branch settings below. - dp3-branch: &dp3-branch placeholder_branch_name + dp3-branch: &dp3-branch MAIN-B-22056_sns_sqs_deps_w_endpoint # MUST BE ONE OF: loadtest, demo, exp. # These are used to pull in env vars so the spelling matters! - dp3-env: &dp3-env placeholder_env + dp3-env: &dp3-env exp # set integration-ignore-branch to the branch if you want to IGNORE # integration tests, or `placeholder_branch_name` if you do want to # run them - integration-ignore-branch: &integration-ignore-branch placeholder_branch_name + integration-ignore-branch: &integration-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint # set integration-mtls-ignore-branch to the branch if you want to # IGNORE mtls integration tests, or `placeholder_branch_name` if you # do want to run them - integration-mtls-ignore-branch: &integration-mtls-ignore-branch placeholder_branch_name + integration-mtls-ignore-branch: &integration-mtls-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint # set client-ignore-branch to the branch if you want to IGNORE # client tests, or `placeholder_branch_name` if you do want to run # them - client-ignore-branch: &client-ignore-branch placeholder_branch_name + client-ignore-branch: &client-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint # set server-ignore-branch to the branch if you want to IGNORE # server tests, or `placeholder_branch_name` if you do want to run # them - server-ignore-branch: &server-ignore-branch placeholder_branch_name + server-ignore-branch: &server-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint executors: base_small: From e59cff8ef4dbd87d03a26ceed8f0412b29b5f59f Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Tue, 14 Jan 2025 22:54:47 +0000 Subject: [PATCH 15/36] B-22056 - restore exp env. --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b5bd5920986..b8d3c39da69 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,30 +40,30 @@ references: # In addition, it's common practice to disable acceptance tests and # ignore tests for dp3 deploys. See the branch settings below. - dp3-branch: &dp3-branch MAIN-B-22056_sns_sqs_deps_w_endpoint + dp3-branch: &dp3-branch placeholder_branch_name # MUST BE ONE OF: loadtest, demo, exp. # These are used to pull in env vars so the spelling matters! - dp3-env: &dp3-env exp + dp3-env: &dp3-env placeholder_env # set integration-ignore-branch to the branch if you want to IGNORE # integration tests, or `placeholder_branch_name` if you do want to # run them - integration-ignore-branch: &integration-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint + integration-ignore-branch: &integration-ignore-branch placeholder_branch_name # set integration-mtls-ignore-branch to the branch if you want to # IGNORE mtls integration tests, or `placeholder_branch_name` if you # do want to run them - integration-mtls-ignore-branch: &integration-mtls-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint + integration-mtls-ignore-branch: &integration-mtls-ignore-branch placeholder_branch_name # set client-ignore-branch to the branch if you want to IGNORE # client tests, or `placeholder_branch_name` if you do want to run # them - client-ignore-branch: &client-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint + client-ignore-branch: &client-ignore-branch placeholder_branch_name # set server-ignore-branch to the branch if you want to IGNORE # server tests, or `placeholder_branch_name` if you do want to run # them - server-ignore-branch: &server-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint + server-ignore-branch: &server-ignore-branch placeholder_branch_name executors: base_small: From bdade45a193bd601c5566babfcdbcb2b763acc14 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Wed, 15 Jan 2025 15:47:41 +0000 Subject: [PATCH 16/36] B-22056 - delete message after receive. --- .../routing/internalapi_test/uploads_test.go | 2 +- pkg/notifications/notification_receiver.go | 11 +++++- .../notification_receiver_stub.go | 2 +- .../notification_receiver_test.go | 1 + pkg/notifications/receiverMocks/SqsClient.go | 37 +++++++++++++++++++ 5 files changed, 50 insertions(+), 3 deletions(-) diff --git a/pkg/handlers/routing/internalapi_test/uploads_test.go b/pkg/handlers/routing/internalapi_test/uploads_test.go index 382cd74a5bf..c75445cc191 100644 --- a/pkg/handlers/routing/internalapi_test/uploads_test.go +++ b/pkg/handlers/routing/internalapi_test/uploads_test.go @@ -69,7 +69,7 @@ func (suite *InternalAPISuite) TestUploads() { fakeS3.EmptyTags = true } go func() { - time.Sleep(2 * time.Second) + time.Sleep(3 * time.Second) if ok && fakeS3 != nil { fakeS3.EmptyTags = false } diff --git a/pkg/notifications/notification_receiver.go b/pkg/notifications/notification_receiver.go index 49222a69fdb..b685cfacaa1 100644 --- a/pkg/notifications/notification_receiver.go +++ b/pkg/notifications/notification_receiver.go @@ -61,6 +61,7 @@ type SnsClient interface { type SqsClient interface { CreateQueue(ctx context.Context, params *sqs.CreateQueueInput, optFns ...func(*sqs.Options)) (*sqs.CreateQueueOutput, error) ReceiveMessage(ctx context.Context, params *sqs.ReceiveMessageInput, optFns ...func(*sqs.Options)) (*sqs.ReceiveMessageOutput, error) + DeleteMessage(ctx context.Context, params *sqs.DeleteMessageInput, optFns ...func(*sqs.Options)) (*sqs.DeleteMessageOutput, error) DeleteQueue(ctx context.Context, params *sqs.DeleteQueueInput, optFns ...func(*sqs.Options)) (*sqs.DeleteQueueOutput, error) ListQueues(ctx context.Context, params *sqs.ListQueuesInput, optFns ...func(*sqs.Options)) (*sqs.ListQueuesOutput, error) } @@ -188,6 +189,14 @@ func (n NotificationReceiverContext) ReceiveMessages(appCtx appcontext.AppContex MessageId: *value.MessageId, Body: value.Body, } + + _, err := n.sqsService.DeleteMessage(recCtx, &sqs.DeleteMessageInput{ + QueueUrl: &queueUrl, + ReceiptHandle: value.ReceiptHandle, + }) + if err != nil { + appCtx.Logger().Info("Couldn't delete message from queue. Error: %v\n", zap.Error(err)) + } } return receivedMessages, recCtx.Err() @@ -195,7 +204,7 @@ func (n NotificationReceiverContext) ReceiveMessages(appCtx appcontext.AppContex // CloseoutQueue stops receiving messages and cleans up the queue and its subscriptions func (n NotificationReceiverContext) CloseoutQueue(appCtx appcontext.AppContext, queueUrl string) error { - appCtx.Logger().Info("Closing out queue: %v", zap.String("queueUrl", queueUrl)) + appCtx.Logger().Info("Closing out queue: ", zap.String("queueUrl", queueUrl)) if cancelFunc, exists := n.receiverCancelMap[queueUrl]; exists { cancelFunc() diff --git a/pkg/notifications/notification_receiver_stub.go b/pkg/notifications/notification_receiver_stub.go index b09b61363fc..e98f0c8aa1e 100644 --- a/pkg/notifications/notification_receiver_stub.go +++ b/pkg/notifications/notification_receiver_stub.go @@ -29,7 +29,7 @@ func (n StubNotificationReceiver) CreateQueueWithSubscription(appCtx appcontext. } func (n StubNotificationReceiver) ReceiveMessages(appCtx appcontext.AppContext, queueUrl string, timerContext context.Context) ([]ReceivedMessage, error) { - time.Sleep(2 * time.Second) + time.Sleep(3 * time.Second) messageId := "stubMessageId" body := queueUrl + ":stubMessageBody" mockMessages := make([]ReceivedMessage, 1) diff --git a/pkg/notifications/notification_receiver_test.go b/pkg/notifications/notification_receiver_test.go index e895a7f2e3b..934cb7db20b 100644 --- a/pkg/notifications/notification_receiver_test.go +++ b/pkg/notifications/notification_receiver_test.go @@ -112,6 +112,7 @@ func (suite *notificationReceiverSuite) TestSuccessPath() { }, }, }, nil) + mockedSqs.On("DeleteMessage", mock.Anything, mock.AnythingOfType("*sqs.DeleteMessageInput")).Return(&sqs.DeleteMessageOutput{}, nil) mockedSqs.On("DeleteQueue", mock.Anything, mock.AnythingOfType("*sqs.DeleteQueueInput")).Return(&sqs.DeleteQueueOutput{}, nil) mockedSqs.On("ListQueues", mock.Anything, mock.AnythingOfType("*sqs.ListQueuesInput")).Return(&sqs.ListQueuesOutput{}, nil) diff --git a/pkg/notifications/receiverMocks/SqsClient.go b/pkg/notifications/receiverMocks/SqsClient.go index 0ab970fc530..c8e6e6aa284 100644 --- a/pkg/notifications/receiverMocks/SqsClient.go +++ b/pkg/notifications/receiverMocks/SqsClient.go @@ -52,6 +52,43 @@ func (_m *SqsClient) CreateQueue(ctx context.Context, params *sqs.CreateQueueInp return r0, r1 } +// DeleteMessage provides a mock function with given fields: ctx, params, optFns +func (_m *SqsClient) DeleteMessage(ctx context.Context, params *sqs.DeleteMessageInput, optFns ...func(*sqs.Options)) (*sqs.DeleteMessageOutput, error) { + _va := make([]interface{}, len(optFns)) + for _i := range optFns { + _va[_i] = optFns[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, params) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + if len(ret) == 0 { + panic("no return value specified for DeleteMessage") + } + + var r0 *sqs.DeleteMessageOutput + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *sqs.DeleteMessageInput, ...func(*sqs.Options)) (*sqs.DeleteMessageOutput, error)); ok { + return rf(ctx, params, optFns...) + } + if rf, ok := ret.Get(0).(func(context.Context, *sqs.DeleteMessageInput, ...func(*sqs.Options)) *sqs.DeleteMessageOutput); ok { + r0 = rf(ctx, params, optFns...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*sqs.DeleteMessageOutput) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *sqs.DeleteMessageInput, ...func(*sqs.Options)) error); ok { + r1 = rf(ctx, params, optFns...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // DeleteQueue provides a mock function with given fields: ctx, params, optFns func (_m *SqsClient) DeleteQueue(ctx context.Context, params *sqs.DeleteQueueInput, optFns ...func(*sqs.Options)) (*sqs.DeleteQueueOutput, error) { _va := make([]interface{}, len(optFns)) From b8d1a369b6cf38d17cdeb5af4b708860bfbe294b Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Wed, 15 Jan 2025 16:40:49 +0000 Subject: [PATCH 17/36] B-22056 - attempting to fix test. --- .../routing/internalapi_test/uploads_test.go | 13 ++++++------- pkg/notifications/notification_receiver_stub.go | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/pkg/handlers/routing/internalapi_test/uploads_test.go b/pkg/handlers/routing/internalapi_test/uploads_test.go index c75445cc191..3a0d64cc01f 100644 --- a/pkg/handlers/routing/internalapi_test/uploads_test.go +++ b/pkg/handlers/routing/internalapi_test/uploads_test.go @@ -65,14 +65,13 @@ func (suite *InternalAPISuite) TestUploads() { rr := httptest.NewRecorder() fakeS3, ok := suite.HandlerConfig().FileStorer().(*storageTest.FakeS3Storage) - if ok && fakeS3 != nil { - fakeS3.EmptyTags = true - } + suite.True(ok) + suite.NotNil(fakeS3, "FileStorer should be fakeS3") + + fakeS3.EmptyTags = true go func() { - time.Sleep(3 * time.Second) - if ok && fakeS3 != nil { - fakeS3.EmptyTags = false - } + time.Sleep(5 * time.Second) + fakeS3.EmptyTags = false }() suite.SetupSiteHandler().ServeHTTP(rr, req) diff --git a/pkg/notifications/notification_receiver_stub.go b/pkg/notifications/notification_receiver_stub.go index e98f0c8aa1e..637989040ff 100644 --- a/pkg/notifications/notification_receiver_stub.go +++ b/pkg/notifications/notification_receiver_stub.go @@ -29,7 +29,7 @@ func (n StubNotificationReceiver) CreateQueueWithSubscription(appCtx appcontext. } func (n StubNotificationReceiver) ReceiveMessages(appCtx appcontext.AppContext, queueUrl string, timerContext context.Context) ([]ReceivedMessage, error) { - time.Sleep(3 * time.Second) + time.Sleep(5 * time.Second) messageId := "stubMessageId" body := queueUrl + ":stubMessageBody" mockMessages := make([]ReceivedMessage, 1) From 890700f4769163af6ca1cc2958172c69dc424517 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Wed, 15 Jan 2025 17:54:34 +0000 Subject: [PATCH 18/36] B-22056 - attempting to fix test. --- pkg/handlers/routing/internalapi_test/uploads_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/handlers/routing/internalapi_test/uploads_test.go b/pkg/handlers/routing/internalapi_test/uploads_test.go index 3a0d64cc01f..f774545504a 100644 --- a/pkg/handlers/routing/internalapi_test/uploads_test.go +++ b/pkg/handlers/routing/internalapi_test/uploads_test.go @@ -70,7 +70,7 @@ func (suite *InternalAPISuite) TestUploads() { fakeS3.EmptyTags = true go func() { - time.Sleep(5 * time.Second) + time.Sleep(4 * time.Second) fakeS3.EmptyTags = false }() From 228ac54f4d2afbbb846e262ad54a43a61cda41af Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Wed, 15 Jan 2025 18:38:43 +0000 Subject: [PATCH 19/36] B-22056 - attempting to fix test. --- pkg/handlers/routing/internalapi_test/uploads_test.go | 2 +- pkg/notifications/notification_receiver_stub.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/handlers/routing/internalapi_test/uploads_test.go b/pkg/handlers/routing/internalapi_test/uploads_test.go index f774545504a..4d3562d963b 100644 --- a/pkg/handlers/routing/internalapi_test/uploads_test.go +++ b/pkg/handlers/routing/internalapi_test/uploads_test.go @@ -70,7 +70,7 @@ func (suite *InternalAPISuite) TestUploads() { fakeS3.EmptyTags = true go func() { - time.Sleep(4 * time.Second) + time.Sleep(8 * time.Second) fakeS3.EmptyTags = false }() diff --git a/pkg/notifications/notification_receiver_stub.go b/pkg/notifications/notification_receiver_stub.go index 637989040ff..e7a54063ef1 100644 --- a/pkg/notifications/notification_receiver_stub.go +++ b/pkg/notifications/notification_receiver_stub.go @@ -29,7 +29,7 @@ func (n StubNotificationReceiver) CreateQueueWithSubscription(appCtx appcontext. } func (n StubNotificationReceiver) ReceiveMessages(appCtx appcontext.AppContext, queueUrl string, timerContext context.Context) ([]ReceivedMessage, error) { - time.Sleep(5 * time.Second) + time.Sleep(18 * time.Second) messageId := "stubMessageId" body := queueUrl + ":stubMessageBody" mockMessages := make([]ReceivedMessage, 1) From e22abd66dc6455c2c61fab226823503188e28025 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Wed, 15 Jan 2025 20:38:11 +0000 Subject: [PATCH 20/36] B-22056 - attempting to fix test. --- .../routing/internalapi_test/uploads_test.go | 22 ++++++++++++++++++- .../notification_receiver_stub.go | 2 +- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/pkg/handlers/routing/internalapi_test/uploads_test.go b/pkg/handlers/routing/internalapi_test/uploads_test.go index 4d3562d963b..d7fe179f9e0 100644 --- a/pkg/handlers/routing/internalapi_test/uploads_test.go +++ b/pkg/handlers/routing/internalapi_test/uploads_test.go @@ -3,6 +3,7 @@ package internalapi_test import ( "net/http" "net/http/httptest" + "strings" "time" "github.com/transcom/mymove/pkg/factory" @@ -70,7 +71,26 @@ func (suite *InternalAPISuite) TestUploads() { fakeS3.EmptyTags = true go func() { - time.Sleep(8 * time.Second) + ch := make(chan bool) + + go func() { + time.Sleep(10 * time.Second) + ch <- true + }() + + for !strings.Contains(rr.Body.String(), "PROCESSING") { + suite.Logger().Info(rr.Body.String()) + + select { + case <-ch: + fakeS3.EmptyTags = false + close(ch) + return + default: + time.Sleep(1 * time.Second) + } + } + fakeS3.EmptyTags = false }() diff --git a/pkg/notifications/notification_receiver_stub.go b/pkg/notifications/notification_receiver_stub.go index e7a54063ef1..e98f0c8aa1e 100644 --- a/pkg/notifications/notification_receiver_stub.go +++ b/pkg/notifications/notification_receiver_stub.go @@ -29,7 +29,7 @@ func (n StubNotificationReceiver) CreateQueueWithSubscription(appCtx appcontext. } func (n StubNotificationReceiver) ReceiveMessages(appCtx appcontext.AppContext, queueUrl string, timerContext context.Context) ([]ReceivedMessage, error) { - time.Sleep(18 * time.Second) + time.Sleep(3 * time.Second) messageId := "stubMessageId" body := queueUrl + ":stubMessageBody" mockMessages := make([]ReceivedMessage, 1) From c0bf4e249784378c61c80fa6b8773d276120f600 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Thu, 16 Jan 2025 00:47:15 +0000 Subject: [PATCH 21/36] B-22056 - attempting to fix test. --- .../routing/internalapi_test/uploads_test.go | 30 +++---------------- 1 file changed, 4 insertions(+), 26 deletions(-) diff --git a/pkg/handlers/routing/internalapi_test/uploads_test.go b/pkg/handlers/routing/internalapi_test/uploads_test.go index d7fe179f9e0..168e291ab46 100644 --- a/pkg/handlers/routing/internalapi_test/uploads_test.go +++ b/pkg/handlers/routing/internalapi_test/uploads_test.go @@ -3,7 +3,6 @@ package internalapi_test import ( "net/http" "net/http/httptest" - "strings" "time" "github.com/transcom/mymove/pkg/factory" @@ -71,26 +70,7 @@ func (suite *InternalAPISuite) TestUploads() { fakeS3.EmptyTags = true go func() { - ch := make(chan bool) - - go func() { - time.Sleep(10 * time.Second) - ch <- true - }() - - for !strings.Contains(rr.Body.String(), "PROCESSING") { - suite.Logger().Info(rr.Body.String()) - - select { - case <-ch: - fakeS3.EmptyTags = false - close(ch) - return - default: - time.Sleep(1 * time.Second) - } - } - + time.Sleep(10 * time.Second) fakeS3.EmptyTags = false }() @@ -99,10 +79,8 @@ func (suite *InternalAPISuite) TestUploads() { suite.Equal(http.StatusOK, rr.Code) suite.Equal("text/event-stream", rr.Header().Get("content-type")) - message1 := "id: 0\nevent: message\ndata: PROCESSING\n\n" - message2 := "id: 1\nevent: message\ndata: CLEAN\n\n" - messageClose := "id: 2\nevent: close\ndata: Connection closed\n\n" - - suite.Equal(message1+message2+messageClose, rr.Body.String()) + suite.Contains(rr.Body.String(), "PROCESSING") + suite.Contains(rr.Body.String(), "CLEAN") + suite.Contains(rr.Body.String(), "Connection closed") }) } From 12c55df0b62e4514e3add107b19920131df09c1b Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Thu, 16 Jan 2025 01:48:01 +0000 Subject: [PATCH 22/36] B-22056 - attempting to fix test. --- pkg/handlers/routing/internalapi_test/uploads_test.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pkg/handlers/routing/internalapi_test/uploads_test.go b/pkg/handlers/routing/internalapi_test/uploads_test.go index 168e291ab46..7cea09d4d8e 100644 --- a/pkg/handlers/routing/internalapi_test/uploads_test.go +++ b/pkg/handlers/routing/internalapi_test/uploads_test.go @@ -3,7 +3,6 @@ package internalapi_test import ( "net/http" "net/http/httptest" - "time" "github.com/transcom/mymove/pkg/factory" "github.com/transcom/mymove/pkg/models" @@ -69,17 +68,12 @@ func (suite *InternalAPISuite) TestUploads() { suite.NotNil(fakeS3, "FileStorer should be fakeS3") fakeS3.EmptyTags = true - go func() { - time.Sleep(10 * time.Second) - fakeS3.EmptyTags = false - }() suite.SetupSiteHandler().ServeHTTP(rr, req) suite.Equal(http.StatusOK, rr.Code) suite.Equal("text/event-stream", rr.Header().Get("content-type")) - suite.Contains(rr.Body.String(), "PROCESSING") suite.Contains(rr.Body.String(), "CLEAN") suite.Contains(rr.Body.String(), "Connection closed") }) From 06a593caba4311b8a8b766c0e7a70f88a7a3f080 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Thu, 16 Jan 2025 15:18:41 +0000 Subject: [PATCH 23/36] B-22056 - deploy to exp. --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b8d3c39da69..b5bd5920986 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,30 +40,30 @@ references: # In addition, it's common practice to disable acceptance tests and # ignore tests for dp3 deploys. See the branch settings below. - dp3-branch: &dp3-branch placeholder_branch_name + dp3-branch: &dp3-branch MAIN-B-22056_sns_sqs_deps_w_endpoint # MUST BE ONE OF: loadtest, demo, exp. # These are used to pull in env vars so the spelling matters! - dp3-env: &dp3-env placeholder_env + dp3-env: &dp3-env exp # set integration-ignore-branch to the branch if you want to IGNORE # integration tests, or `placeholder_branch_name` if you do want to # run them - integration-ignore-branch: &integration-ignore-branch placeholder_branch_name + integration-ignore-branch: &integration-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint # set integration-mtls-ignore-branch to the branch if you want to # IGNORE mtls integration tests, or `placeholder_branch_name` if you # do want to run them - integration-mtls-ignore-branch: &integration-mtls-ignore-branch placeholder_branch_name + integration-mtls-ignore-branch: &integration-mtls-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint # set client-ignore-branch to the branch if you want to IGNORE # client tests, or `placeholder_branch_name` if you do want to run # them - client-ignore-branch: &client-ignore-branch placeholder_branch_name + client-ignore-branch: &client-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint # set server-ignore-branch to the branch if you want to IGNORE # server tests, or `placeholder_branch_name` if you do want to run # them - server-ignore-branch: &server-ignore-branch placeholder_branch_name + server-ignore-branch: &server-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint executors: base_small: From dc22fd44be695be76579b4d8e209e57aa10930b6 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Thu, 16 Jan 2025 17:07:05 +0000 Subject: [PATCH 24/36] B-22056 - update param while in exp. --- pkg/cli/receiver.go | 8 ++++---- pkg/handlers/routing/internalapi_test/uploads_test.go | 6 ++++++ pkg/notifications/notification_receiver.go | 6 +++--- pkg/notifications/notification_receiver_test.go | 4 ++-- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/pkg/cli/receiver.go b/pkg/cli/receiver.go index be30daf135d..d5fdc2436a0 100644 --- a/pkg/cli/receiver.go +++ b/pkg/cli/receiver.go @@ -20,7 +20,7 @@ const ( // InitReceiverFlags initializes Storage command line flags func InitReceiverFlags(flag *pflag.FlagSet) { - flag.String(ReceiverBackendFlag, "local", "Receiver backend to use, either local or sns&sqs.") + flag.String(ReceiverBackendFlag, "local", "Receiver backend to use, either local or sns_sqs.") flag.String(SNSTagsUpdatedTopicFlag, "", "SNS Topic for receiving event messages") flag.String(SNSRegionFlag, "", "Region used for SNS and SQS") flag.String(SNSAccountId, "", "SNS account Id") @@ -30,11 +30,11 @@ func InitReceiverFlags(flag *pflag.FlagSet) { func CheckReceiver(v *viper.Viper) error { receiverBackend := v.GetString(ReceiverBackendFlag) - if !stringSliceContains([]string{"local", "sns&sqs"}, receiverBackend) { - return fmt.Errorf("invalid receiver-backend %s, expecting local or sns&sqs", receiverBackend) + if !stringSliceContains([]string{"local", "sns_sqs"}, receiverBackend) { + return fmt.Errorf("invalid receiver-backend %s, expecting local or sns_sqs", receiverBackend) } - if receiverBackend == "sns&sqs" { + if receiverBackend == "sns_sqs" { r := v.GetString(SNSRegionFlag) if r == "" { return fmt.Errorf("invalid value for %s: %s", SNSRegionFlag, r) diff --git a/pkg/handlers/routing/internalapi_test/uploads_test.go b/pkg/handlers/routing/internalapi_test/uploads_test.go index 7cea09d4d8e..0d957e1de6a 100644 --- a/pkg/handlers/routing/internalapi_test/uploads_test.go +++ b/pkg/handlers/routing/internalapi_test/uploads_test.go @@ -3,6 +3,7 @@ package internalapi_test import ( "net/http" "net/http/httptest" + "time" "github.com/transcom/mymove/pkg/factory" "github.com/transcom/mymove/pkg/models" @@ -68,12 +69,17 @@ func (suite *InternalAPISuite) TestUploads() { suite.NotNil(fakeS3, "FileStorer should be fakeS3") fakeS3.EmptyTags = true + go func() { + time.Sleep(12 * time.Second) + fakeS3.EmptyTags = false + }() suite.SetupSiteHandler().ServeHTTP(rr, req) suite.Equal(http.StatusOK, rr.Code) suite.Equal("text/event-stream", rr.Header().Get("content-type")) + suite.Contains(rr.Body.String(), "PROCESSING") suite.Contains(rr.Body.String(), "CLEAN") suite.Contains(rr.Body.String(), "Connection closed") }) diff --git a/pkg/notifications/notification_receiver.go b/pkg/notifications/notification_receiver.go index b685cfacaa1..a4bec916e86 100644 --- a/pkg/notifications/notification_receiver.go +++ b/pkg/notifications/notification_receiver.go @@ -232,7 +232,7 @@ func (n NotificationReceiverContext) CloseoutQueue(appCtx appcontext.AppContext, func (n NotificationReceiverContext) GetDefaultTopic() (string, error) { topicName := n.viper.GetString(cli.SNSTagsUpdatedTopicFlag) receiverBackend := n.viper.GetString(cli.ReceiverBackendFlag) - if topicName == "" && receiverBackend == "sns&sqs" { + if topicName == "" && receiverBackend == "sns_sqs" { return "", errors.New("sns_tags_updated_topic key not available") } return topicName, nil @@ -241,12 +241,12 @@ func (n NotificationReceiverContext) GetDefaultTopic() (string, error) { // InitReceiver initializes the receiver backend, only call this once func InitReceiver(v ViperType, logger *zap.Logger, wipeAllNotificationQueues bool) (NotificationReceiver, error) { - if v.GetString(cli.ReceiverBackendFlag) == "sns&sqs" { + if v.GetString(cli.ReceiverBackendFlag) == "sns_sqs" { // Setup notification receiver service with SNS & SQS backend dependencies awsSNSRegion := v.GetString(cli.SNSRegionFlag) awsAccountId := v.GetString(cli.SNSAccountId) - logger.Info("Using aws sns&sqs receiver backend", zap.String("region", awsSNSRegion)) + logger.Info("Using aws sns_sqs receiver backend", zap.String("region", awsSNSRegion)) cfg, err := config.LoadDefaultConfig(context.Background(), config.WithRegion(awsSNSRegion), diff --git a/pkg/notifications/notification_receiver_test.go b/pkg/notifications/notification_receiver_test.go index 934cb7db20b..f7dab5a91b7 100644 --- a/pkg/notifications/notification_receiver_test.go +++ b/pkg/notifications/notification_receiver_test.go @@ -71,7 +71,7 @@ func (suite *notificationReceiverSuite) TestSuccessPath() { suite.Run("aws backend - notification receiver InitReceiver", func() { // Setup mocks mockedViper := mocks.ViperType{} - mockedViper.On("GetString", cli.ReceiverBackendFlag).Return("sns&sqs") + mockedViper.On("GetString", cli.ReceiverBackendFlag).Return("sns_sqs") mockedViper.On("GetString", cli.SNSRegionFlag).Return("us-gov-west-1") mockedViper.On("GetString", cli.SNSAccountId).Return("12345") mockedViper.On("GetString", cli.SNSTagsUpdatedTopicFlag).Return("fake_sns_topic") @@ -88,7 +88,7 @@ func (suite *notificationReceiverSuite) TestSuccessPath() { suite.Run("aws backend - notification receiver with mock services", func() { // Setup mocks mockedViper := mocks.ViperType{} - mockedViper.On("GetString", cli.ReceiverBackendFlag).Return("sns&sqs") + mockedViper.On("GetString", cli.ReceiverBackendFlag).Return("sns_sqs") mockedViper.On("GetString", cli.SNSRegionFlag).Return("us-gov-west-1") mockedViper.On("GetString", cli.SNSAccountId).Return("12345") mockedViper.On("GetString", cli.SNSTagsUpdatedTopicFlag).Return("fake_sns_topic") From c50ed5fdc0e3b5f58ae9a87feef1e0903cf30734 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Thu, 16 Jan 2025 17:57:19 +0000 Subject: [PATCH 25/36] B-22056 - update logging while in exp. --- pkg/notifications/notification_receiver.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/notifications/notification_receiver.go b/pkg/notifications/notification_receiver.go index a4bec916e86..2ba55aa939f 100644 --- a/pkg/notifications/notification_receiver.go +++ b/pkg/notifications/notification_receiver.go @@ -272,6 +272,8 @@ func InitReceiver(v ViperType, logger *zap.Logger, wipeAllNotificationQueues boo return notificationReceiver, nil } + logger.Info("Using local sns_sqs receiver backend", zap.String("receiver_backend", v.GetString(cli.ReceiverBackendFlag)), zap.String("SNSRegion", v.GetString(cli.SNSRegionFlag))) + return NewStubNotificationReceiver(), nil } From 5bd091823441f0f1be94270313a0974fce50c739 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Thu, 16 Jan 2025 15:18:28 -0500 Subject: [PATCH 26/36] B-22056 - update logging while in exp. --- pkg/notifications/notification_receiver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/notifications/notification_receiver.go b/pkg/notifications/notification_receiver.go index 2ba55aa939f..0edba0f44a2 100644 --- a/pkg/notifications/notification_receiver.go +++ b/pkg/notifications/notification_receiver.go @@ -272,7 +272,7 @@ func InitReceiver(v ViperType, logger *zap.Logger, wipeAllNotificationQueues boo return notificationReceiver, nil } - logger.Info("Using local sns_sqs receiver backend", zap.String("receiver_backend", v.GetString(cli.ReceiverBackendFlag)), zap.String("SNSRegion", v.GetString(cli.SNSRegionFlag))) + logger.Info("Using local sns_sqs receiver backend", zap.String("receiver_backend", v.GetString(cli.ReceiverBackendFlag))) return NewStubNotificationReceiver(), nil } From f597e9694a555a3ff69dc4b58ea1856b2aa45766 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Thu, 16 Jan 2025 21:25:18 +0000 Subject: [PATCH 27/36] B-22056 - restore exp env. --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b5bd5920986..b8d3c39da69 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,30 +40,30 @@ references: # In addition, it's common practice to disable acceptance tests and # ignore tests for dp3 deploys. See the branch settings below. - dp3-branch: &dp3-branch MAIN-B-22056_sns_sqs_deps_w_endpoint + dp3-branch: &dp3-branch placeholder_branch_name # MUST BE ONE OF: loadtest, demo, exp. # These are used to pull in env vars so the spelling matters! - dp3-env: &dp3-env exp + dp3-env: &dp3-env placeholder_env # set integration-ignore-branch to the branch if you want to IGNORE # integration tests, or `placeholder_branch_name` if you do want to # run them - integration-ignore-branch: &integration-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint + integration-ignore-branch: &integration-ignore-branch placeholder_branch_name # set integration-mtls-ignore-branch to the branch if you want to # IGNORE mtls integration tests, or `placeholder_branch_name` if you # do want to run them - integration-mtls-ignore-branch: &integration-mtls-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint + integration-mtls-ignore-branch: &integration-mtls-ignore-branch placeholder_branch_name # set client-ignore-branch to the branch if you want to IGNORE # client tests, or `placeholder_branch_name` if you do want to run # them - client-ignore-branch: &client-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint + client-ignore-branch: &client-ignore-branch placeholder_branch_name # set server-ignore-branch to the branch if you want to IGNORE # server tests, or `placeholder_branch_name` if you do want to run # them - server-ignore-branch: &server-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint + server-ignore-branch: &server-ignore-branch placeholder_branch_name executors: base_small: From 855e52ebc81f3ac5866704c07e6521f1effa4757 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Thu, 16 Jan 2025 21:37:04 +0000 Subject: [PATCH 28/36] B-22056 - deploy to exp with updated param format --- .circleci/config.yml | 12 ++++++------ pkg/cli/receiver.go | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b8d3c39da69..b5bd5920986 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,30 +40,30 @@ references: # In addition, it's common practice to disable acceptance tests and # ignore tests for dp3 deploys. See the branch settings below. - dp3-branch: &dp3-branch placeholder_branch_name + dp3-branch: &dp3-branch MAIN-B-22056_sns_sqs_deps_w_endpoint # MUST BE ONE OF: loadtest, demo, exp. # These are used to pull in env vars so the spelling matters! - dp3-env: &dp3-env placeholder_env + dp3-env: &dp3-env exp # set integration-ignore-branch to the branch if you want to IGNORE # integration tests, or `placeholder_branch_name` if you do want to # run them - integration-ignore-branch: &integration-ignore-branch placeholder_branch_name + integration-ignore-branch: &integration-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint # set integration-mtls-ignore-branch to the branch if you want to # IGNORE mtls integration tests, or `placeholder_branch_name` if you # do want to run them - integration-mtls-ignore-branch: &integration-mtls-ignore-branch placeholder_branch_name + integration-mtls-ignore-branch: &integration-mtls-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint # set client-ignore-branch to the branch if you want to IGNORE # client tests, or `placeholder_branch_name` if you do want to run # them - client-ignore-branch: &client-ignore-branch placeholder_branch_name + client-ignore-branch: &client-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint # set server-ignore-branch to the branch if you want to IGNORE # server tests, or `placeholder_branch_name` if you do want to run # them - server-ignore-branch: &server-ignore-branch placeholder_branch_name + server-ignore-branch: &server-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint executors: base_small: diff --git a/pkg/cli/receiver.go b/pkg/cli/receiver.go index d5fdc2436a0..335c987cd76 100644 --- a/pkg/cli/receiver.go +++ b/pkg/cli/receiver.go @@ -9,7 +9,7 @@ import ( const ( // ReceiverBackend is the Receiver Backend Flag - ReceiverBackendFlag string = "receiver-backend" + ReceiverBackendFlag string = "receiver_backend" // SNSTagsUpdatedTopicFlag is the SNS Tags Updated Topic Flag SNSTagsUpdatedTopicFlag string = "sns-tags-updated-topic" // SNSRegionFlag is the SNS Region flag @@ -31,7 +31,7 @@ func CheckReceiver(v *viper.Viper) error { receiverBackend := v.GetString(ReceiverBackendFlag) if !stringSliceContains([]string{"local", "sns_sqs"}, receiverBackend) { - return fmt.Errorf("invalid receiver-backend %s, expecting local or sns_sqs", receiverBackend) + return fmt.Errorf("invalid receiver_backend %s, expecting local or sns_sqs", receiverBackend) } if receiverBackend == "sns_sqs" { From d17b881efe97e8f52f6bdccaf956b67494afeccd Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Thu, 16 Jan 2025 21:43:10 +0000 Subject: [PATCH 29/36] B-22056 - restore format. --- pkg/cli/receiver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cli/receiver.go b/pkg/cli/receiver.go index 335c987cd76..9338a1d17d0 100644 --- a/pkg/cli/receiver.go +++ b/pkg/cli/receiver.go @@ -9,7 +9,7 @@ import ( const ( // ReceiverBackend is the Receiver Backend Flag - ReceiverBackendFlag string = "receiver_backend" + ReceiverBackendFlag string = "receiver-backend" // SNSTagsUpdatedTopicFlag is the SNS Tags Updated Topic Flag SNSTagsUpdatedTopicFlag string = "sns-tags-updated-topic" // SNSRegionFlag is the SNS Region flag From 6ddfa3d55640c933d471f9a17595dac0cbc74256 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Thu, 16 Jan 2025 23:49:32 +0000 Subject: [PATCH 30/36] B-22056 - restore exp env. --- .circleci/config.yml | 12 ++++++------ .envrc | 4 ++-- pkg/notifications/notification_receiver.go | 4 ++++ 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b5bd5920986..b8d3c39da69 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,30 +40,30 @@ references: # In addition, it's common practice to disable acceptance tests and # ignore tests for dp3 deploys. See the branch settings below. - dp3-branch: &dp3-branch MAIN-B-22056_sns_sqs_deps_w_endpoint + dp3-branch: &dp3-branch placeholder_branch_name # MUST BE ONE OF: loadtest, demo, exp. # These are used to pull in env vars so the spelling matters! - dp3-env: &dp3-env exp + dp3-env: &dp3-env placeholder_env # set integration-ignore-branch to the branch if you want to IGNORE # integration tests, or `placeholder_branch_name` if you do want to # run them - integration-ignore-branch: &integration-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint + integration-ignore-branch: &integration-ignore-branch placeholder_branch_name # set integration-mtls-ignore-branch to the branch if you want to # IGNORE mtls integration tests, or `placeholder_branch_name` if you # do want to run them - integration-mtls-ignore-branch: &integration-mtls-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint + integration-mtls-ignore-branch: &integration-mtls-ignore-branch placeholder_branch_name # set client-ignore-branch to the branch if you want to IGNORE # client tests, or `placeholder_branch_name` if you do want to run # them - client-ignore-branch: &client-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint + client-ignore-branch: &client-ignore-branch placeholder_branch_name # set server-ignore-branch to the branch if you want to IGNORE # server tests, or `placeholder_branch_name` if you do want to run # them - server-ignore-branch: &server-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint + server-ignore-branch: &server-ignore-branch placeholder_branch_name executors: base_small: diff --git a/.envrc b/.envrc index 7eb37fa168f..7f7d66b0fcd 100644 --- a/.envrc +++ b/.envrc @@ -229,12 +229,12 @@ export TZ="UTC" # AWS development access # -# To use S3/SES or SNS&SQS for local builds, you'll need to uncomment the following. +# To use S3/SES or SNS & SQS for local builds, you'll need to uncomment the following. # Do not commit the change: # # export STORAGE_BACKEND=s3 # export EMAIL_BACKEND=ses -# export RECEIVER_BACKEND="sns&sqs" +# export RECEIVER_BACKEND=sns_sqs # # Instructions for using S3 storage backend here: https://dp3.atlassian.net/wiki/spaces/MT/pages/1470955567/How+to+test+storing+data+in+S3+locally # Instructions for using SES email backend here: https://dp3.atlassian.net/wiki/spaces/MT/pages/1467973894/How+to+test+sending+email+locally diff --git a/pkg/notifications/notification_receiver.go b/pkg/notifications/notification_receiver.go index 0edba0f44a2..e0bc10e8c97 100644 --- a/pkg/notifications/notification_receiver.go +++ b/pkg/notifications/notification_receiver.go @@ -2,6 +2,7 @@ package notifications import ( "context" + "encoding/json" "errors" "fmt" "strings" @@ -190,6 +191,9 @@ func (n NotificationReceiverContext) ReceiveMessages(appCtx appcontext.AppContex Body: value.Body, } + val, _ := json.Marshal(value) + appCtx.Logger().Info("messages incoming", zap.ByteString("message", val)) + _, err := n.sqsService.DeleteMessage(recCtx, &sqs.DeleteMessageInput{ QueueUrl: &queueUrl, ReceiptHandle: value.ReceiptHandle, From 7900c0c04d6fc0d238b63ffe42cc83575cf053b1 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Fri, 17 Jan 2025 16:59:11 +0000 Subject: [PATCH 31/36] B-22056 - remove timer from test. --- .envrc | 2 +- pkg/handlers/routing/internalapi_test/uploads_test.go | 6 ------ pkg/notifications/notification_receiver.go | 4 +--- pkg/storage/test/s3.go | 1 + 4 files changed, 3 insertions(+), 10 deletions(-) diff --git a/.envrc b/.envrc index 7f7d66b0fcd..783cd183534 100644 --- a/.envrc +++ b/.envrc @@ -258,7 +258,7 @@ export AWS_S3_KEY_NAMESPACE=$USER export AWS_SES_DOMAIN="devlocal.dp3.us" export AWS_SES_REGION="us-gov-west-1" -if [ "$RECEIVER_BACKEND" == "sns&sqs" ]; then +if [ "$RECEIVER_BACKEND" == "sns_sqs" ]; then export SNS_TAGS_UPDATED_TOPIC="app_s3_tag_events" export SNS_REGION="us-gov-west-1" fi diff --git a/pkg/handlers/routing/internalapi_test/uploads_test.go b/pkg/handlers/routing/internalapi_test/uploads_test.go index 0d957e1de6a..06610d84be2 100644 --- a/pkg/handlers/routing/internalapi_test/uploads_test.go +++ b/pkg/handlers/routing/internalapi_test/uploads_test.go @@ -3,7 +3,6 @@ package internalapi_test import ( "net/http" "net/http/httptest" - "time" "github.com/transcom/mymove/pkg/factory" "github.com/transcom/mymove/pkg/models" @@ -69,11 +68,6 @@ func (suite *InternalAPISuite) TestUploads() { suite.NotNil(fakeS3, "FileStorer should be fakeS3") fakeS3.EmptyTags = true - go func() { - time.Sleep(12 * time.Second) - fakeS3.EmptyTags = false - }() - suite.SetupSiteHandler().ServeHTTP(rr, req) suite.Equal(http.StatusOK, rr.Code) diff --git a/pkg/notifications/notification_receiver.go b/pkg/notifications/notification_receiver.go index e0bc10e8c97..1ec6ecd2358 100644 --- a/pkg/notifications/notification_receiver.go +++ b/pkg/notifications/notification_receiver.go @@ -2,7 +2,6 @@ package notifications import ( "context" - "encoding/json" "errors" "fmt" "strings" @@ -191,8 +190,7 @@ func (n NotificationReceiverContext) ReceiveMessages(appCtx appcontext.AppContex Body: value.Body, } - val, _ := json.Marshal(value) - appCtx.Logger().Info("messages incoming", zap.ByteString("message", val)) + appCtx.Logger().Info("Message received.", zap.String("messageId", *value.MessageId)) _, err := n.sqsService.DeleteMessage(recCtx, &sqs.DeleteMessageInput{ QueueUrl: &queueUrl, diff --git a/pkg/storage/test/s3.go b/pkg/storage/test/s3.go index 901edf370e5..cbbab7802d5 100644 --- a/pkg/storage/test/s3.go +++ b/pkg/storage/test/s3.go @@ -96,6 +96,7 @@ func (fake *FakeS3Storage) Tags(_ string) (map[string]string, error) { } if fake.EmptyTags { tags = map[string]string{} + fake.EmptyTags = false } return tags, nil } From 80cf55244c75f49243df9b45298f338e62cdfeaa Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Fri, 17 Jan 2025 19:36:23 +0000 Subject: [PATCH 32/36] B-22056 - tests for fakeS3 and local storage. --- pkg/storage/filesystem_test.go | 18 ++++++ pkg/storage/memory_test.go | 18 ++++++ pkg/storage/test/s3_test.go | 101 +++++++++++++++++++++++++++++++++ 3 files changed, 137 insertions(+) create mode 100644 pkg/storage/test/s3_test.go diff --git a/pkg/storage/filesystem_test.go b/pkg/storage/filesystem_test.go index 27ecc5e951c..cecf69d2a7c 100644 --- a/pkg/storage/filesystem_test.go +++ b/pkg/storage/filesystem_test.go @@ -1,6 +1,7 @@ package storage import ( + "strings" "testing" ) @@ -21,3 +22,20 @@ func TestFilesystemPresignedURL(t *testing.T) { t.Errorf("wrong presigned url: expected %s, got %s", expected, url) } } + +func TestFilesystemTags(t *testing.T) { + fsParams := FilesystemParams{ + root: "/home/username", + webRoot: "https://example.text/files", + } + fs := NewFilesystem(fsParams) + + tags, err := fs.Tags("anyKey") + if err != nil { + t.Fatalf("could not get tags: %s", err) + } + + if tag, exists := tags["av-status"]; exists && strings.Compare(tag, "CLEAN") != 0 { + t.Fatal("tag 'av-status' should return CLEAN") + } +} diff --git a/pkg/storage/memory_test.go b/pkg/storage/memory_test.go index 59384c5acee..68b96b1b0eb 100644 --- a/pkg/storage/memory_test.go +++ b/pkg/storage/memory_test.go @@ -1,6 +1,7 @@ package storage import ( + "strings" "testing" ) @@ -21,3 +22,20 @@ func TestMemoryPresignedURL(t *testing.T) { t.Errorf("wrong presigned url: expected %s, got %s", expected, url) } } + +func TestMemoryTags(t *testing.T) { + fsParams := MemoryParams{ + root: "/home/username", + webRoot: "https://example.text/files", + } + fs := NewMemory(fsParams) + + tags, err := fs.Tags("anyKey") + if err != nil { + t.Fatalf("could not get tags: %s", err) + } + + if tag, exists := tags["av-status"]; exists && strings.Compare(tag, "CLEAN") != 0 { + t.Fatal("tag 'av-status' should return CLEAN") + } +} diff --git a/pkg/storage/test/s3_test.go b/pkg/storage/test/s3_test.go new file mode 100644 index 00000000000..a3fa89c5c9a --- /dev/null +++ b/pkg/storage/test/s3_test.go @@ -0,0 +1,101 @@ +package test + +import ( + "errors" + "io" + "strings" + "testing" +) + +// Tests all functions of FakeS3Storage +func TestFakeS3ReturnsSuccessful(t *testing.T) { + fakeS3 := NewFakeS3Storage(true) + if fakeS3 == nil { + t.Fatal("could not create new fakeS3") + } + + storeValue := strings.NewReader("anyValue") + _, err := fakeS3.Store("anyKey", storeValue, "", nil) + if err != nil { + t.Fatalf("could not store in fakeS3: %s", err) + } + + retReader, err := fakeS3.Fetch("anyKey") + if err != nil { + t.Fatalf("could not fetch from fakeS3: %s", err) + } + + err = fakeS3.Delete("anyKey") + if err != nil { + t.Fatalf("could not delete on fakeS3: %s", err) + } + + retValue, err := io.ReadAll(retReader) + if strings.Compare(string(retValue[:]), "anyValue") != 0 { + t.Fatalf("could not fetch from fakeS3: %s", err) + } + + fileSystem := fakeS3.FileSystem() + if fileSystem == nil { + t.Fatal("could not retrieve filesystem from fakeS3") + } + + tempFileSystem := fakeS3.TempFileSystem() + if tempFileSystem == nil { + t.Fatal("could not retrieve filesystem from fakeS3") + } + + tags, err := fakeS3.Tags("anyKey") + if err != nil { + t.Fatalf("could not fetch from fakeS3: %s", err) + } + if len(tags) != 2 { + t.Fatal("return tags must have both tagName and av-status for fakeS3") + } + + presignedUrl, err := fakeS3.PresignedURL("anyKey", "anyContentType", "anyFileName") + if err != nil { + t.Fatal("could not retrieve presignedUrl from fakeS3") + } + + if strings.Compare(presignedUrl, "https://example.com/dir/anyKey?response-content-disposition=attachment%3B+filename%3D%22anyFileName%22&response-content-type=anyContentType&signed=test") != 0 { + t.Fatalf("could not retrieve proper presignedUrl from fakeS3 %s", presignedUrl) + } +} + +// Test for willSucceed false +func TestFakeS3WillNotSucceed(t *testing.T) { + fakeS3 := NewFakeS3Storage(false) + if fakeS3 == nil { + t.Fatalf("could not create new fakeS3") + } + + storeValue := strings.NewReader("anyValue") + _, err := fakeS3.Store("anyKey", storeValue, "", nil) + if err == nil || errors.Is(err, errors.New("failed to push")) { + t.Fatalf("should not be able to store when willSucceed false: %s", err) + } + + _, err = fakeS3.Fetch("anyKey") + if err == nil || errors.Is(err, errors.New("failed to fetch file")) { + t.Fatalf("should not find file on Fetch for willSucceed false: %s", err) + } +} + +// Tests empty tag returns empty tags on FakeS3Storage +func TestFakeS3ReturnsEmptyTags(t *testing.T) { + fakeS3 := NewFakeS3Storage(true) + if fakeS3 == nil { + t.Fatal("could not create new fakeS3") + } + + fakeS3.EmptyTags = true + + tags, err := fakeS3.Tags("anyKey") + if err != nil { + t.Fatalf("could not fetch from fakeS3: %s", err) + } + if len(tags) != 0 { + t.Fatal("return tags must be empty for FakeS3 when EmptyTags set to true") + } +} From 0f0f6d4a73a0cf73b4f1c777aeca4f2597004da7 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Fri, 17 Jan 2025 20:45:14 +0000 Subject: [PATCH 33/36] B-22056 - more tests for memory and filesystem. --- pkg/storage/filesystem_test.go | 43 ++++++++++++++++++++++++++++++++++ pkg/storage/memory_test.go | 43 ++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) diff --git a/pkg/storage/filesystem_test.go b/pkg/storage/filesystem_test.go index cecf69d2a7c..9c37b9204c8 100644 --- a/pkg/storage/filesystem_test.go +++ b/pkg/storage/filesystem_test.go @@ -1,6 +1,7 @@ package storage import ( + "io" "strings" "testing" ) @@ -23,6 +24,48 @@ func TestFilesystemPresignedURL(t *testing.T) { } } +func TestFilesystemReturnsSuccessful(t *testing.T) { + fsParams := FilesystemParams{ + root: "./", + webRoot: "https://example.text/files", + } + filesystem := NewFilesystem(fsParams) + if filesystem == nil { + t.Fatal("could not create new filesystem") + } + + storeValue := strings.NewReader("anyValue") + _, err := filesystem.Store("anyKey", storeValue, "", nil) + if err != nil { + t.Fatalf("could not store in filesystem: %s", err) + } + + retReader, err := filesystem.Fetch("anyKey") + if err != nil { + t.Fatalf("could not fetch from filesystem: %s", err) + } + + err = filesystem.Delete("anyKey") + if err != nil { + t.Fatalf("could not delete on filesystem: %s", err) + } + + retValue, err := io.ReadAll(retReader) + if strings.Compare(string(retValue[:]), "anyValue") != 0 { + t.Fatalf("could not fetch from filesystem: %s", err) + } + + fileSystem := filesystem.FileSystem() + if fileSystem == nil { + t.Fatal("could not retrieve filesystem from filesystem") + } + + tempFileSystem := filesystem.TempFileSystem() + if tempFileSystem == nil { + t.Fatal("could not retrieve filesystem from filesystem") + } +} + func TestFilesystemTags(t *testing.T) { fsParams := FilesystemParams{ root: "/home/username", diff --git a/pkg/storage/memory_test.go b/pkg/storage/memory_test.go index 68b96b1b0eb..bdf3133e9c8 100644 --- a/pkg/storage/memory_test.go +++ b/pkg/storage/memory_test.go @@ -1,6 +1,7 @@ package storage import ( + "io" "strings" "testing" ) @@ -23,6 +24,48 @@ func TestMemoryPresignedURL(t *testing.T) { } } +func TestMemoryReturnsSuccessful(t *testing.T) { + fsParams := MemoryParams{ + root: "/home/username", + webRoot: "https://example.text/files", + } + memory := NewMemory(fsParams) + if memory == nil { + t.Fatal("could not create new memory") + } + + storeValue := strings.NewReader("anyValue") + _, err := memory.Store("anyKey", storeValue, "", nil) + if err != nil { + t.Fatalf("could not store in memory: %s", err) + } + + retReader, err := memory.Fetch("anyKey") + if err != nil { + t.Fatalf("could not fetch from memory: %s", err) + } + + err = memory.Delete("anyKey") + if err != nil { + t.Fatalf("could not delete on memory: %s", err) + } + + retValue, err := io.ReadAll(retReader) + if strings.Compare(string(retValue[:]), "anyValue") != 0 { + t.Fatalf("could not fetch from memory: %s", err) + } + + fileSystem := memory.FileSystem() + if fileSystem == nil { + t.Fatal("could not retrieve filesystem from memory") + } + + tempFileSystem := memory.TempFileSystem() + if tempFileSystem == nil { + t.Fatal("could not retrieve filesystem from memory") + } +} + func TestMemoryTags(t *testing.T) { fsParams := MemoryParams{ root: "/home/username", From e6a690c92a5048f02da670600d60d3a49eb766c0 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Fri, 17 Jan 2025 22:18:19 +0000 Subject: [PATCH 34/36] B-22056 - change local receiver log message. --- pkg/notifications/notification_receiver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/notifications/notification_receiver.go b/pkg/notifications/notification_receiver.go index 1ec6ecd2358..1eba5c4e1a7 100644 --- a/pkg/notifications/notification_receiver.go +++ b/pkg/notifications/notification_receiver.go @@ -274,7 +274,7 @@ func InitReceiver(v ViperType, logger *zap.Logger, wipeAllNotificationQueues boo return notificationReceiver, nil } - logger.Info("Using local sns_sqs receiver backend", zap.String("receiver_backend", v.GetString(cli.ReceiverBackendFlag))) + logger.Info("Using local notification receiver backend", zap.String("receiver_backend", v.GetString(cli.ReceiverBackendFlag))) return NewStubNotificationReceiver(), nil } From 821067caa4d62afa3e6d64dc6698a59e6e39ac7a Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Fri, 17 Jan 2025 23:02:44 +0000 Subject: [PATCH 35/36] B-22056 - deploy to exp. --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a0df9b774a6..51a34eab813 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,30 +40,30 @@ references: # In addition, it's common practice to disable acceptance tests and # ignore tests for dp3 deploys. See the branch settings below. - dp3-branch: &dp3-branch placeholder_branch_name + dp3-branch: &dp3-branch MAIN-B-22056_sns_sqs_deps_w_endpoint # MUST BE ONE OF: loadtest, demo, exp. # These are used to pull in env vars so the spelling matters! - dp3-env: &dp3-env placeholder_env + dp3-env: &dp3-env exp # set integration-ignore-branch to the branch if you want to IGNORE # integration tests, or `placeholder_branch_name` if you do want to # run them - integration-ignore-branch: &integration-ignore-branch placeholder_branch_name + integration-ignore-branch: &integration-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint # set integration-mtls-ignore-branch to the branch if you want to # IGNORE mtls integration tests, or `placeholder_branch_name` if you # do want to run them - integration-mtls-ignore-branch: &integration-mtls-ignore-branch placeholder_branch_name + integration-mtls-ignore-branch: &integration-mtls-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint # set client-ignore-branch to the branch if you want to IGNORE # client tests, or `placeholder_branch_name` if you do want to run # them - client-ignore-branch: &client-ignore-branch placeholder_branch_name + client-ignore-branch: &client-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint # set server-ignore-branch to the branch if you want to IGNORE # server tests, or `placeholder_branch_name` if you do want to run # them - server-ignore-branch: &server-ignore-branch placeholder_branch_name + server-ignore-branch: &server-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint executors: base_small: From 78f62bb865734c2f9c632e846f76fee7f694d44e Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Sat, 18 Jan 2025 00:07:03 +0000 Subject: [PATCH 36/36] B-22056 - restore exp env. --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 51a34eab813..a0df9b774a6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,30 +40,30 @@ references: # In addition, it's common practice to disable acceptance tests and # ignore tests for dp3 deploys. See the branch settings below. - dp3-branch: &dp3-branch MAIN-B-22056_sns_sqs_deps_w_endpoint + dp3-branch: &dp3-branch placeholder_branch_name # MUST BE ONE OF: loadtest, demo, exp. # These are used to pull in env vars so the spelling matters! - dp3-env: &dp3-env exp + dp3-env: &dp3-env placeholder_env # set integration-ignore-branch to the branch if you want to IGNORE # integration tests, or `placeholder_branch_name` if you do want to # run them - integration-ignore-branch: &integration-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint + integration-ignore-branch: &integration-ignore-branch placeholder_branch_name # set integration-mtls-ignore-branch to the branch if you want to # IGNORE mtls integration tests, or `placeholder_branch_name` if you # do want to run them - integration-mtls-ignore-branch: &integration-mtls-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint + integration-mtls-ignore-branch: &integration-mtls-ignore-branch placeholder_branch_name # set client-ignore-branch to the branch if you want to IGNORE # client tests, or `placeholder_branch_name` if you do want to run # them - client-ignore-branch: &client-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint + client-ignore-branch: &client-ignore-branch placeholder_branch_name # set server-ignore-branch to the branch if you want to IGNORE # server tests, or `placeholder_branch_name` if you do want to run # them - server-ignore-branch: &server-ignore-branch MAIN-B-22056_sns_sqs_deps_w_endpoint + server-ignore-branch: &server-ignore-branch placeholder_branch_name executors: base_small: