From 3367c8debc387327622fb0b47048664f86a3fed0 Mon Sep 17 00:00:00 2001 From: Rangel Ivanov Date: Tue, 25 Jul 2023 11:31:17 +0300 Subject: [PATCH] Add async file upload support --- clients/baseclient/client_util_test.go | 4 +- clients/baseclient/test_token_factory.go | 4 + clients/baseclient/token_factory.go | 1 + clients/csrf/transport.go | 31 +-- .../fakes/fake_mta_client_builder.go | 17 +- .../fakes/fake_mta_client_operations.go | 123 +++++++---- clients/mtaclient/mta_client_operations.go | 9 +- clients/mtaclient/mta_rest_client.go | 209 ++++++++++++++++-- .../mtaclient/operations/operations_client.go | 60 ----- .../upload_mta_archive_from_url_parameters.go | 157 ------------- .../upload_mta_archive_from_url_responses.go | 69 ------ .../operations/upload_mta_file_parameters.go | 177 --------------- .../operations/upload_mta_file_responses.go | 68 ------ .../mtaclient/retryable_mta_rest_client.go | 28 ++- commands/action.go | 4 +- commands/action_test.go | 10 +- commands/base_command.go | 7 +- commands/base_command_test.go | 8 +- commands/blue_green_deploy_command.go | 3 +- commands/default_token_factory.go | 11 +- commands/deploy_command.go | 152 ++++++++++--- commands/deploy_command_test.go | 89 ++++++-- commands/download_mta_op_logs_command_test.go | 2 +- commands/execution_monitor.go | 6 +- commands/file_uploader.go | 112 +++++++--- commands/file_uploader_test.go | 14 +- commands/flags_parser.go | 4 +- commands/test_token_factory.go | 6 +- commands/undeploy_command_test.go | 10 +- go.mod | 3 +- log/log.go | 2 +- multiapps_plugin.go | 2 +- testutil/test_results.go | 15 +- util/cf_context.go | 6 +- util/cf_context_test.go | 6 +- util/file_splitter.go | 24 +- util/named_read_seeker.go | 8 + 37 files changed, 675 insertions(+), 786 deletions(-) delete mode 100644 clients/mtaclient/operations/upload_mta_archive_from_url_parameters.go delete mode 100644 clients/mtaclient/operations/upload_mta_archive_from_url_responses.go delete mode 100644 clients/mtaclient/operations/upload_mta_file_parameters.go delete mode 100644 clients/mtaclient/operations/upload_mta_file_responses.go create mode 100644 util/named_read_seeker.go diff --git a/clients/baseclient/client_util_test.go b/clients/baseclient/client_util_test.go index 53a01dc..9086fc4 100755 --- a/clients/baseclient/client_util_test.go +++ b/clients/baseclient/client_util_test.go @@ -75,7 +75,7 @@ var _ = Describe("ClientUtil", func() { toReturn := testStruct{} return toReturn, nil } - result, err := CallWithRetry(getInterface, 4, (time.Duration(0) * time.Second)) + result, err := CallWithRetry(getInterface, 4, time.Duration(0)) Expect(err).NotTo(HaveOccurred()) Expect(result).To(Equal(result)) }) @@ -87,7 +87,7 @@ var _ = Describe("ClientUtil", func() { err := ClientError{404, "Not Found", "The server cannot find requesred resource"} return toReturn, &err } - result, err := CallWithRetry(mockCallback, 4, (time.Duration(0) * time.Second)) + result, err := CallWithRetry(mockCallback, 4, time.Duration(0)) Expect(err).To(HaveOccurred()) Expect(result).To(Equal(result)) }) diff --git a/clients/baseclient/test_token_factory.go b/clients/baseclient/test_token_factory.go index e4c2986..74b947a 100644 --- a/clients/baseclient/test_token_factory.go +++ b/clients/baseclient/test_token_factory.go @@ -17,3 +17,7 @@ type customTokenfactory struct { func (c *customTokenfactory) NewToken() (runtime.ClientAuthInfoWriter, error) { return testutil.NewCustomBearerToken(c.tokenString), nil } + +func (c *customTokenfactory) NewRawToken() (string, error) { + return c.tokenString, nil +} diff --git a/clients/baseclient/token_factory.go b/clients/baseclient/token_factory.go index f96e063..97c8c94 100644 --- a/clients/baseclient/token_factory.go +++ b/clients/baseclient/token_factory.go @@ -5,4 +5,5 @@ import "github.com/go-openapi/runtime" // TokenFactory factory for generating new OAuth token type TokenFactory interface { NewToken() (runtime.ClientAuthInfoWriter, error) + NewRawToken() (string, error) } diff --git a/clients/csrf/transport.go b/clients/csrf/transport.go index e5f28d0..20a948f 100644 --- a/clients/csrf/transport.go +++ b/clients/csrf/transport.go @@ -2,10 +2,9 @@ package csrf import ( "net/http" - "strconv" + "strings" "github.com/cloudfoundry-incubator/multiapps-cli-plugin/log" - "github.com/jinzhu/copier" ) type Csrf struct { @@ -26,18 +25,24 @@ type Transport struct { } func (t Transport) RoundTrip(req *http.Request) (*http.Response, error) { - req2 := http.Request{} - copier.Copy(&req2, req) + //TODO rework this to be like so: + // get the auth header from the outbound request + // call the csrf endpoint or use a cached token + // add the csrf header + // --- + // that way, we avoid creating and using a cli connection again - csrfTokenManager := NewDefaultCsrfTokenManager(&t, &req2) + reqCopy := req.Clone(req.Context()) + + csrfTokenManager := NewDefaultCsrfTokenManager(&t, reqCopy) err := csrfTokenManager.updateToken() if err != nil { return nil, err } - log.Tracef("Sending a request with CSRF '" + req2.Header.Get("X-Csrf-Header") + " : " + req2.Header.Get("X-Csrf-Token") + "'\n") - log.Tracef("Cookies used are: " + prettyPrintCookies(req2.Cookies()) + "\n") - res, err := t.OriginalTransport.RoundTrip(&req2) + log.Tracef("Sending a request with CSRF '%s : %s'\n", reqCopy.Header.Get("X-Csrf-Header"), reqCopy.Header.Get("X-Csrf-Token")) + log.Tracef("Cookies used are: %s\n", prettyPrintCookies(reqCopy.Cookies())) + res, err := t.OriginalTransport.RoundTrip(reqCopy) if err != nil { return nil, err } @@ -47,17 +52,17 @@ func (t Transport) RoundTrip(req *http.Request) (*http.Response, error) { } if tokenWasRefreshed { - log.Tracef("Response code '" + strconv.Itoa(res.StatusCode) + "' from bad token. Must Retry.\n") + log.Tracef("Response code '%d' from bad token. Must Retry.\n", res.StatusCode) return nil, &ForbiddenError{} } - return res, err } func prettyPrintCookies(cookies []*http.Cookie) string { - result := "" + var result strings.Builder for _, cookie := range cookies { - result = result + cookie.String() + " " + result.WriteString(cookie.String()) + result.WriteRune(' ') } - return result + return result.String() } diff --git a/clients/mtaclient/fakes/fake_mta_client_builder.go b/clients/mtaclient/fakes/fake_mta_client_builder.go index 27e47fa..6cd3b8e 100644 --- a/clients/mtaclient/fakes/fake_mta_client_builder.go +++ b/clients/mtaclient/fakes/fake_mta_client_builder.go @@ -1,6 +1,7 @@ package fakes import ( + "net/http" "os" "github.com/cloudfoundry-incubator/multiapps-cli-plugin/clients/models" @@ -8,11 +9,11 @@ import ( ) type FakeMtaClientBuilder struct { - FakeMtaClient FakeMtaClientOperations + FakeMtaClient *FakeMtaClientOperations } func NewFakeMtaClientBuilder() *FakeMtaClientBuilder { - return &FakeMtaClientBuilder{} + return &FakeMtaClientBuilder{&FakeMtaClientOperations{}} } func (fb *FakeMtaClientBuilder) ExecuteAction(operationID, actionID string, resultHeader mtaclient.ResponseHeader, resultError error) *FakeMtaClientBuilder { @@ -51,12 +52,16 @@ func (fb *FakeMtaClientBuilder) StartMtaOperation(operation models.Operation, re fb.FakeMtaClient.StartMtaOperationReturns(result, resultError) return fb } -func (fb *FakeMtaClientBuilder) UploadMtaFile(file os.File, result *models.FileMetadata, resultError error) *FakeMtaClientBuilder { +func (fb *FakeMtaClientBuilder) UploadMtaFile(file *os.File, result *models.FileMetadata, resultError error) *FakeMtaClientBuilder { fb.FakeMtaClient.UploadMtaFileReturns(result, resultError) return fb } -func (fb *FakeMtaClientBuilder) UploadMtaArchiveFromUrl(fileUrl string, result *models.FileMetadata, resultError error) *FakeMtaClientBuilder { - fb.FakeMtaClient.UploadMtaArchiveFromUrlReturnsOnCall(fileUrl, result, resultError) +func (fb *FakeMtaClientBuilder) StartUploadMtaArchiveFromUrl(fileUrl string, namespace *string, result http.Header, resultError error) *FakeMtaClientBuilder { + fb.FakeMtaClient.StartUploadMtaArchiveFromUrlReturnsOnCall(fileUrl, namespace, result, resultError) + return fb +} +func (fb *FakeMtaClientBuilder) GetAsyncUploadJob(jobId string, namespace *string, appInstanceId string, result mtaclient.AsyncUploadJobResult, resultErr error) *FakeMtaClientBuilder { + fb.FakeMtaClient.GetAsyncUploadJobReturnsOnCall(jobId, namespace, appInstanceId, result, resultErr) return fb } func (fb *FakeMtaClientBuilder) GetMtaOperationLogContent(operationID, logID string, result string, resultError error) *FakeMtaClientBuilder { @@ -64,6 +69,6 @@ func (fb *FakeMtaClientBuilder) GetMtaOperationLogContent(operationID, logID str return fb } -func (fb *FakeMtaClientBuilder) Build() FakeMtaClientOperations { +func (fb *FakeMtaClientBuilder) Build() *FakeMtaClientOperations { return fb.FakeMtaClient } diff --git a/clients/mtaclient/fakes/fake_mta_client_operations.go b/clients/mtaclient/fakes/fake_mta_client_operations.go index 801624c..b1abaaa 100644 --- a/clients/mtaclient/fakes/fake_mta_client_operations.go +++ b/clients/mtaclient/fakes/fake_mta_client_operations.go @@ -1,7 +1,8 @@ package fakes import ( - "os" + "github.com/cloudfoundry-incubator/multiapps-cli-plugin/util" + "net/http" "sync" "github.com/cloudfoundry-incubator/multiapps-cli-plugin/clients/models" @@ -126,10 +127,10 @@ type FakeMtaClientOperations struct { result1 mtaclient.ResponseHeader result2 error } - UploadMtaFileStub func(file os.File) (*models.FileMetadata, error) + UploadMtaFileStub func(file util.NamedReadSeeker) (*models.FileMetadata, error) uploadMtaFileMutex sync.RWMutex uploadMtaFileArgsForCall []struct { - file os.File + file util.NamedReadSeeker } uploadMtaFileReturns struct { result1 *models.FileMetadata @@ -139,14 +140,23 @@ type FakeMtaClientOperations struct { result1 *models.FileMetadata result2 error } - uploadMtaArchiveFromUrlMutex sync.RWMutex - uploadMtaArchiveFromUrlReturns struct { - result1 *models.FileMetadata + startUploadMtaArchiveFromUrlMutex sync.RWMutex + startUploadMtaArchiveFromUrlReturns struct { + result1 http.Header + result2 error + } + startUploadMtaArchiveFromUrlReturnsOnCall map[string]struct { + headers http.Header + err error + } + getAsyncUploadJobMutex sync.RWMutex + getAsyncUploadJobReturns struct { + result1 mtaclient.AsyncUploadJobResult result2 error } - uploadMtaArchiveFromUrlReturnsOnCall map[string]struct { - file *models.FileMetadata - err error + getAsyncUploadJobReturnsOnCall map[string]struct { + result mtaclient.AsyncUploadJobResult + err error } GetMtaOperationLogContentStub func(operationID, logID string) (string, error) getMtaOperationLogContentMutex sync.RWMutex @@ -166,7 +176,7 @@ type FakeMtaClientOperations struct { invocationsMutex sync.RWMutex } -func (fake FakeMtaClientOperations) ExecuteAction(operationID string, actionID string) (mtaclient.ResponseHeader, error) { +func (fake *FakeMtaClientOperations) ExecuteAction(operationID string, actionID string) (mtaclient.ResponseHeader, error) { fake.executeActionMutex.Lock() ret, specificReturn := fake.executeActionReturnsOnCall[len(fake.executeActionArgsForCall)] fake.executeActionArgsForCall = append(fake.executeActionArgsForCall, struct { @@ -218,7 +228,7 @@ func (fake *FakeMtaClientOperations) ExecuteActionReturnsOnCall(i int, result1 m }{result1, result2} } -func (fake FakeMtaClientOperations) GetMta(mtaID string) (*models.Mta, error) { +func (fake *FakeMtaClientOperations) GetMta(mtaID string) (*models.Mta, error) { fake.getMtaMutex.Lock() ret, specificReturn := fake.getMtaReturnsOnCall[len(fake.getMtaArgsForCall)] fake.getMtaArgsForCall = append(fake.getMtaArgsForCall, struct { @@ -269,7 +279,7 @@ func (fake *FakeMtaClientOperations) GetMtaReturnsOnCall(i int, result1 *models. }{result1, result2} } -func (fake FakeMtaClientOperations) GetMtaFiles(namespace *string) ([]*models.FileMetadata, error) { +func (fake *FakeMtaClientOperations) GetMtaFiles(namespace *string) ([]*models.FileMetadata, error) { fake.getMtaFilesMutex.Lock() ret, specificReturn := fake.getMtaFilesReturnsOnCall[len(fake.getMtaFilesArgsForCall)] fake.getMtaFilesArgsForCall = append(fake.getMtaFilesArgsForCall, struct{}{}) @@ -312,7 +322,7 @@ func (fake *FakeMtaClientOperations) GetMtaFilesReturnsOnCall(i int, result1 []* }{result1, result2} } -func (fake FakeMtaClientOperations) GetMtaOperation(operationID string, embed string) (*models.Operation, error) { +func (fake *FakeMtaClientOperations) GetMtaOperation(operationID string, embed string) (*models.Operation, error) { fake.getMtaOperationMutex.Lock() ret, specificReturn := fake.getMtaOperationReturnsOnCall[len(fake.getMtaOperationArgsForCall)] fake.getMtaOperationArgsForCall = append(fake.getMtaOperationArgsForCall, struct { @@ -364,7 +374,7 @@ func (fake *FakeMtaClientOperations) GetMtaOperationReturnsOnCall(i int, result1 }{result1, result2} } -func (fake FakeMtaClientOperations) GetMtaOperationLogs(operationID string) ([]*models.Log, error) { +func (fake *FakeMtaClientOperations) GetMtaOperationLogs(operationID string) ([]*models.Log, error) { fake.getMtaOperationLogsMutex.Lock() ret, specificReturn := fake.getMtaOperationLogsReturnsOnCall[len(fake.getMtaOperationLogsArgsForCall)] fake.getMtaOperationLogsArgsForCall = append(fake.getMtaOperationLogsArgsForCall, struct { @@ -415,7 +425,7 @@ func (fake *FakeMtaClientOperations) GetMtaOperationLogsReturnsOnCall(i int, res }{result1, result2} } -func (fake FakeMtaClientOperations) GetMtaOperations(mtaId *string, last *int64, status []string) ([]*models.Operation, error) { +func (fake *FakeMtaClientOperations) GetMtaOperations(mtaId *string, last *int64, status []string) ([]*models.Operation, error) { var statusCopy []string if status != nil { statusCopy = make([]string, len(status)) @@ -473,7 +483,7 @@ func (fake *FakeMtaClientOperations) GetMtaOperationsReturnsOnCall(i int, result }{result1, result2} } -func (fake FakeMtaClientOperations) GetMtas() ([]*models.Mta, error) { +func (fake *FakeMtaClientOperations) GetMtas() ([]*models.Mta, error) { fake.getMtasMutex.Lock() ret, specificReturn := fake.getMtasReturnsOnCall[len(fake.getMtasArgsForCall)] fake.getMtasArgsForCall = append(fake.getMtasArgsForCall, struct{}{}) @@ -516,7 +526,7 @@ func (fake *FakeMtaClientOperations) GetMtasReturnsOnCall(i int, result1 []*mode }{result1, result2} } -func (fake FakeMtaClientOperations) GetOperationActions(operationID string) ([]string, error) { +func (fake *FakeMtaClientOperations) GetOperationActions(operationID string) ([]string, error) { fake.getOperationActionsMutex.Lock() ret, specificReturn := fake.getOperationActionsReturnsOnCall[len(fake.getOperationActionsArgsForCall)] fake.getOperationActionsArgsForCall = append(fake.getOperationActionsArgsForCall, struct { @@ -567,7 +577,7 @@ func (fake *FakeMtaClientOperations) GetOperationActionsReturnsOnCall(i int, res }{result1, result2} } -func (fake FakeMtaClientOperations) StartMtaOperation(operation models.Operation) (mtaclient.ResponseHeader, error) { +func (fake *FakeMtaClientOperations) StartMtaOperation(operation models.Operation) (mtaclient.ResponseHeader, error) { fake.startMtaOperationMutex.Lock() ret, specificReturn := fake.startMtaOperationReturnsOnCall[len(fake.startMtaOperationArgsForCall)] fake.startMtaOperationArgsForCall = append(fake.startMtaOperationArgsForCall, struct { @@ -618,11 +628,11 @@ func (fake *FakeMtaClientOperations) StartMtaOperationReturnsOnCall(i int, resul }{result1, result2} } -func (fake FakeMtaClientOperations) UploadMtaFile(file os.File, fileSize int64, namespace *string) (*models.FileMetadata, error) { +func (fake *FakeMtaClientOperations) UploadMtaFile(file util.NamedReadSeeker, fileSize int64, namespace *string) (*models.FileMetadata, error) { fake.uploadMtaFileMutex.Lock() ret, specificReturn := fake.uploadMtaFileReturnsOnCall[len(fake.uploadMtaFileArgsForCall)] fake.uploadMtaFileArgsForCall = append(fake.uploadMtaFileArgsForCall, struct { - file os.File + file util.NamedReadSeeker }{file}) fake.recordInvocation("UploadMtaFile", []interface{}{file}) fake.uploadMtaFileMutex.Unlock() @@ -641,7 +651,7 @@ func (fake *FakeMtaClientOperations) UploadMtaFileCallCount() int { return len(fake.uploadMtaFileArgsForCall) } -func (fake *FakeMtaClientOperations) UploadMtaFileArgsForCall(i int) os.File { +func (fake *FakeMtaClientOperations) UploadMtaFileArgsForCall(i int) util.NamedReadSeeker { fake.uploadMtaFileMutex.RLock() defer fake.uploadMtaFileMutex.RUnlock() return fake.uploadMtaFileArgsForCall[i].file @@ -669,38 +679,61 @@ func (fake *FakeMtaClientOperations) UploadMtaFileReturnsOnCall(i int, result1 * }{result1, result2} } -func (fake FakeMtaClientOperations) UploadMtaArchiveFromUrl(fileUrl string, namespace *string) (*models.FileMetadata, error) { - fake.uploadMtaArchiveFromUrlMutex.Lock() - result, specificReturn := fake.uploadMtaArchiveFromUrlReturnsOnCall[fileUrl] - fake.recordInvocation("UploadMtaArchiveFromUrl", []interface{}{fileUrl}) - fake.uploadMtaArchiveFromUrlMutex.Unlock() +func (fake *FakeMtaClientOperations) StartUploadMtaArchiveFromUrl(fileUrl string, namespace *string) (http.Header, error) { + fake.startUploadMtaArchiveFromUrlMutex.Lock() + result, specificReturn := fake.startUploadMtaArchiveFromUrlReturnsOnCall[fileUrl] + fake.recordInvocation("StartUploadMtaArchiveFromUrl", []interface{}{fileUrl, namespace}) + fake.startUploadMtaArchiveFromUrlMutex.Unlock() if specificReturn { - return result.file, result.err + return result.headers, result.err } - return fake.uploadMtaArchiveFromUrlReturns.result1, fake.uploadMtaArchiveFromUrlReturns.result2 + return fake.startUploadMtaArchiveFromUrlReturns.result1, fake.startUploadMtaArchiveFromUrlReturns.result2 } -func (fake *FakeMtaClientOperations) UploadMtaArchiveFromUrlReturns(result1 *models.FileMetadata, result2 error) { - fake.uploadMtaArchiveFromUrlReturns = struct { - result1 *models.FileMetadata - result2 error - }{result1, result2} +func (fake *FakeMtaClientOperations) StartUploadMtaArchiveFromUrlReturnsOnCall(fileUrl string, namespace *string, headers http.Header, err error) { + fake.startUploadMtaArchiveFromUrlMutex.Lock() + if fake.startUploadMtaArchiveFromUrlReturnsOnCall == nil { + fake.startUploadMtaArchiveFromUrlReturnsOnCall = make(map[string]struct { + headers http.Header + err error + }) + } + fake.startUploadMtaArchiveFromUrlReturnsOnCall[fileUrl] = struct { + headers http.Header + err error + }{headers, err} + fake.recordInvocation("StartUploadMtaArchiveFromUrl", []interface{}{fileUrl, namespace}) + fake.startUploadMtaArchiveFromUrlMutex.Unlock() } -func (fake *FakeMtaClientOperations) UploadMtaArchiveFromUrlReturnsOnCall(fileUrl string, result1 *models.FileMetadata, result2 error) { - if fake.uploadMtaArchiveFromUrlReturnsOnCall == nil { - fake.uploadMtaArchiveFromUrlReturnsOnCall = make(map[string]struct { - file *models.FileMetadata - err error +func (fake *FakeMtaClientOperations) GetAsyncUploadJob(jobId string, namespace *string, appInstanceId string) (mtaclient.AsyncUploadJobResult, error) { + fake.getAsyncUploadJobMutex.Lock() + result, specificReturn := fake.getAsyncUploadJobReturnsOnCall[jobId] + fake.recordInvocation("GetAsyncUploadJob", []interface{}{jobId, namespace, appInstanceId}) + fake.getAsyncUploadJobMutex.Unlock() + if specificReturn { + return result.result, result.err + } + return fake.getAsyncUploadJobReturns.result1, fake.startUploadMtaArchiveFromUrlReturns.result2 +} + +func (fake *FakeMtaClientOperations) GetAsyncUploadJobReturnsOnCall(jobId string, namespace *string, appInstanceId string, result mtaclient.AsyncUploadJobResult, err error) { + fake.getAsyncUploadJobMutex.Lock() + if fake.getAsyncUploadJobReturnsOnCall == nil { + fake.getAsyncUploadJobReturnsOnCall = make(map[string]struct { + result mtaclient.AsyncUploadJobResult + err error }) } - fake.uploadMtaArchiveFromUrlReturnsOnCall[fileUrl] = struct { - file *models.FileMetadata - err error - }{result1, result2} + fake.getAsyncUploadJobReturnsOnCall[jobId] = struct { + result mtaclient.AsyncUploadJobResult + err error + }{result, err} + fake.recordInvocation("GetAsyncUploadJob", []interface{}{jobId, namespace, appInstanceId}) + fake.getAsyncUploadJobMutex.Unlock() } -func (fake FakeMtaClientOperations) GetMtaOperationLogContent(operationID string, logID string) (string, error) { +func (fake *FakeMtaClientOperations) GetMtaOperationLogContent(operationID string, logID string) (string, error) { fake.getMtaOperationLogContentMutex.Lock() ret, specificReturn := fake.getMtaOperationLogContentReturnsOnCall[len(fake.getMtaOperationLogContentArgsForCall)] fake.getMtaOperationLogContentArgsForCall = append(fake.getMtaOperationLogContentArgsForCall, struct { @@ -775,8 +808,8 @@ func (fake *FakeMtaClientOperations) Invocations() map[string][][]interface{} { defer fake.startMtaOperationMutex.RUnlock() fake.uploadMtaFileMutex.RLock() defer fake.uploadMtaFileMutex.RUnlock() - fake.uploadMtaArchiveFromUrlMutex.RLock() - defer fake.uploadMtaArchiveFromUrlMutex.RUnlock() + fake.startUploadMtaArchiveFromUrlMutex.RLock() + defer fake.startUploadMtaArchiveFromUrlMutex.RUnlock() fake.getMtaOperationLogContentMutex.RLock() defer fake.getMtaOperationLogContentMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} diff --git a/clients/mtaclient/mta_client_operations.go b/clients/mtaclient/mta_client_operations.go index 955ee58..5bd5ed7 100644 --- a/clients/mtaclient/mta_client_operations.go +++ b/clients/mtaclient/mta_client_operations.go @@ -1,10 +1,10 @@ package mtaclient import ( - "os" - "github.com/cloudfoundry-incubator/multiapps-cli-plugin/clients/models" + "github.com/cloudfoundry-incubator/multiapps-cli-plugin/util" "github.com/go-openapi/strfmt" + "net/http" ) // MtaClientOperations drun drun drun @@ -18,8 +18,9 @@ type MtaClientOperations interface { GetMtas() ([]*models.Mta, error) GetOperationActions(operationID string) ([]string, error) StartMtaOperation(operation models.Operation) (ResponseHeader, error) - UploadMtaFile(file os.File, fileSize int64, namespace *string) (*models.FileMetadata, error) - UploadMtaArchiveFromUrl(fileUrl string, namespace *string) (*models.FileMetadata, error) + UploadMtaFile(file util.NamedReadSeeker, fileSize int64, namespace *string) (*models.FileMetadata, error) + StartUploadMtaArchiveFromUrl(fileUrl string, namespace *string) (http.Header, error) + GetAsyncUploadJob(jobId string, namespace *string, appInstanceId string) (AsyncUploadJobResult, error) GetMtaOperationLogContent(operationID, logID string) (string, error) } diff --git a/clients/mtaclient/mta_rest_client.go b/clients/mtaclient/mta_rest_client.go index fb22c6b..7c851a3 100644 --- a/clients/mtaclient/mta_rest_client.go +++ b/clients/mtaclient/mta_rest_client.go @@ -1,9 +1,16 @@ package mtaclient import ( + "bytes" "context" + "encoding/json" + "fmt" + "github.com/cloudfoundry-incubator/multiapps-cli-plugin/util" + "github.com/go-openapi/runtime/client" + "io" + "mime/multipart" "net/http" - "os" + "time" "github.com/cloudfoundry-incubator/multiapps-cli-plugin/clients/baseclient" "github.com/cloudfoundry-incubator/multiapps-cli-plugin/clients/models" @@ -18,13 +25,24 @@ const restBaseURL string = "api/v1/" type MtaRestClient struct { baseclient.BaseClient client *MtaClient + + dsHost string + spaceGuid string +} + +type AsyncUploadJobResult struct { + Status string `json:"status"` + Error string `json:"error,omitempty"` + File *models.FileMetadata `json:"file,omitempty"` + MtaId string `json:"mta_id,omitempty"` + BytesProcessed int64 `json:"bytes_processed,omitempty"` } func NewMtaClient(host, spaceID string, rt http.RoundTripper, tokenFactory baseclient.TokenFactory) MtaClientOperations { restURL := restBaseURL + spacesURL + spaceID t := baseclient.NewHTTPTransport(host, restURL, rt) httpMtaClient := New(t, strfmt.Default) - return &MtaRestClient{baseclient.BaseClient{TokenFactory: tokenFactory}, httpMtaClient} + return &MtaRestClient{baseclient.BaseClient{TokenFactory: tokenFactory}, httpMtaClient, host, spaceID} } func (c MtaRestClient) ExecuteAction(operationID, actionID string) (ResponseHeader, error) { @@ -171,39 +189,184 @@ func (c MtaRestClient) StartMtaOperation(operation models.Operation) (ResponseHe return ResponseHeader{Location: resp.Location}, nil } -func (c MtaRestClient) UploadMtaFile(file os.File, fileSize int64, namespace *string) (*models.FileMetadata, error) { - params := &operations.UploadMtaFileParams{ - Context: context.TODO(), - File: file, - FileSize: fileSize, - Namespace: namespace, +func (c MtaRestClient) UploadMtaFile(file util.NamedReadSeeker, fileSize int64, namespace *string) (*models.FileMetadata, error) { + requestUrl := "https://" + c.dsHost + "/" + restBaseURL + spacesURL + c.spaceGuid + "/files" + if namespace != nil && len(*namespace) != 0 { + requestUrl += "?namespace=" + *namespace } - result, err := executeRestOperation(c.TokenFactory, func(token runtime.ClientAuthInfoWriter) (interface{}, error) { - return c.client.Operations.UploadMtaFile(params, token) - }) + token, err := c.TokenFactory.NewRawToken() + if err != nil { + return nil, fmt.Errorf("could not get authentication token: %v", err) + } + contentLength, err := c.calculateRequestSize(file.Name(), fileSize) if err != nil { - return nil, baseclient.NewClientError(err) + return nil, fmt.Errorf("could not calculate upload file request size: %v", err) + } + + pipeReader, pipeWriter := io.Pipe() + form := multipart.NewWriter(pipeWriter) + + errChan := make(chan error, 1) + go func() { + defer pipeWriter.Close() + + fileWriter, err := form.CreateFormFile("file", file.Name()) + if err != nil { + errChan <- fmt.Errorf("could not write to HTTP request: %v", err) + return + } + + _, err = io.Copy(fileWriter, file) + if err != nil { + errChan <- fmt.Errorf("could not write file to HTTP request: %v", err) + return + } + + err = form.Close() + if err != nil { + errChan <- fmt.Errorf("could not write end boundary to HTTP request: %v", err) + } + errChan <- nil + }() + + req, _ := http.NewRequest(http.MethodPost, requestUrl, pipeReader) + req.Header.Set("Content-Type", form.FormDataContentType()) + req.Header.Set("Authorization", "Bearer "+token) + req.ContentLength = contentLength + + cl := c.client.Transport.(*client.Runtime) + httpClient := http.Client{Transport: cl.Transport, Jar: cl.Jar} + + resp, err := httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("could not upload file %q: %v", file.Name(), err) + } + defer resp.Body.Close() + + pipeErr := <-errChan + if pipeErr != nil { + return nil, fmt.Errorf("could not upload file %q: %v", file.Name(), pipeErr) + } + if resp.StatusCode/100 != 2 { + return nil, fmt.Errorf("could not upload file %q: %s", file.Name(), resp.Status) + } + + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("could not read file upload response: %v", err) + } + + fileEntry := &models.FileMetadata{} + err = json.Unmarshal(bodyBytes, fileEntry) + if err != nil { + return nil, fmt.Errorf("could not deserialize file upload response: %v", err) } - return result.(*operations.UploadMtaFileCreated).Payload, nil + return fileEntry, nil } -func (c MtaRestClient) UploadMtaArchiveFromUrl(fileUrl string, namespace *string) (*models.FileMetadata, error) { - params := &operations.UploadMtaArchiveFromUrlParams{ - Context: context.TODO(), - FileUrl: fileUrl, - Namespace: namespace, +func (c MtaRestClient) calculateRequestSize(fileName string, fileSize int64) (int64, error) { + var body bytes.Buffer + form := multipart.NewWriter(&body) + _, err := form.CreateFormFile("file", fileName) + if err != nil { + return 0, err + } + err = form.Close() + if err != nil { + return 0, err } + return int64(body.Len()) + fileSize, nil +} - result, err := executeRestOperation(c.TokenFactory, func(token runtime.ClientAuthInfoWriter) (interface{}, error) { - return c.client.Operations.UploadMtaArchiveFromUrl(params, token) - }) +func (c MtaRestClient) StartUploadMtaArchiveFromUrl(fileUrl string, namespace *string) (http.Header, error) { + requestUrl := "https://" + c.dsHost + "/" + restBaseURL + spacesURL + c.spaceGuid + "/files/uploadFromUrlAsync" + if namespace != nil && len(*namespace) != 0 { + requestUrl += "?namespace=" + *namespace + } + body := struct { + FileUrl string `json:"file_url"` + }{fileUrl} + + bodyBytes, err := json.Marshal(body) if err != nil { - return nil, baseclient.NewClientError(err) + return nil, fmt.Errorf("could not serialize start async file upload request: %v", err) + } + + req, err := http.NewRequest(http.MethodPost, requestUrl, bytes.NewReader(bodyBytes)) + if err != nil { + return nil, fmt.Errorf("could not create start async file upload request: %v", err) + } + + token, err := c.TokenFactory.NewRawToken() + if err != nil { + return nil, fmt.Errorf("could not get authentication token: %v", err) + } + + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Content-Type", "application/json") + + cl := c.client.Transport.(*client.Runtime) + httpClient := http.Client{Transport: cl.Transport, Jar: cl.Jar, Timeout: 5 * time.Minute} + httpClient.CheckRedirect = func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + } + + resp, err := httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("could not start async file uplaod: %v", err) + } + if resp.StatusCode/100 != 2 && resp.StatusCode/100 != 3 { + return nil, fmt.Errorf("could not start async file uplaod: %s", resp.Status) + } + return resp.Header, nil +} + +func (c MtaRestClient) GetAsyncUploadJob(jobId string, namespace *string, appInstanceId string) (AsyncUploadJobResult, error) { + requestUrl := "https://" + c.dsHost + "/" + restBaseURL + spacesURL + c.spaceGuid + "/files/jobs/" + jobId + if namespace != nil && len(*namespace) != 0 { + requestUrl += "?namespace=" + *namespace + } + + req, err := http.NewRequest(http.MethodGet, requestUrl, nil) + if err != nil { + return AsyncUploadJobResult{}, fmt.Errorf("could not create get async file upload job request: %v", err) + } + + token, err := c.TokenFactory.NewRawToken() + if err != nil { + return AsyncUploadJobResult{}, fmt.Errorf("could not get authentication token: %v", err) + } + + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("x-cf-app-instance", appInstanceId) + + cl := c.client.Transport.(*client.Runtime) + httpClient := http.Client{Transport: cl.Transport, Jar: cl.Jar, Timeout: 5 * time.Minute} + + resp, err := httpClient.Do(req) + if err != nil { + return AsyncUploadJobResult{}, fmt.Errorf("could not get async file upload job %s: %v", jobId, err) + } + defer resp.Body.Close() + + if resp.StatusCode/100 != 2 { + return AsyncUploadJobResult{}, fmt.Errorf("could not get async file upload job %s: %s", jobId, resp.Status) + } + + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + return AsyncUploadJobResult{}, fmt.Errorf("could not read async file upload job %s response: %v", jobId, err) + } + + var responseBody AsyncUploadJobResult + err = json.Unmarshal(bodyBytes, &responseBody) + if err != nil { + return AsyncUploadJobResult{}, fmt.Errorf("could not deserialize async file upload job %s response: %v", jobId, err) } - return result.(*operations.UploadMtaArchiveFromUrlCreated).Payload, nil + return responseBody, nil } func (c MtaRestClient) GetMtaOperationLogContent(operationID, logID string) (string, error) { diff --git a/clients/mtaclient/operations/operations_client.go b/clients/mtaclient/operations/operations_client.go index 19e5002..e1d290d 100644 --- a/clients/mtaclient/operations/operations_client.go +++ b/clients/mtaclient/operations/operations_client.go @@ -324,66 +324,6 @@ func (a *Client) StartMtaOperation(params *StartMtaOperationParams, authInfo run } -/* -UploadMtaFile Uploads an Multi Target Application file - -*/ -func (a *Client) UploadMtaFile(params *UploadMtaFileParams, authInfo runtime.ClientAuthInfoWriter) (*UploadMtaFileCreated, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewUploadMtaFileParams() - } - - result, err := a.transport.Submit(&runtime.ClientOperation{ - ID: "UploadMtaFile", - Method: "POST", - PathPattern: "/files", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"multipart/form-data"}, - Schemes: []string{"https"}, - Params: params, - Reader: &UploadMtaFileReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - }) - if err != nil { - return nil, err - } - return result.(*UploadMtaFileCreated), nil - -} - -/* -UploadMtaArchiveFromUrl Uploads an Multi Target Application archive referenced by a URL - -*/ -func (a *Client) UploadMtaArchiveFromUrl(params *UploadMtaArchiveFromUrlParams, authInfo runtime.ClientAuthInfoWriter) (*UploadMtaArchiveFromUrlCreated, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewUploadMtaArchiveFromUrlParams() - } - - result, err := a.transport.Submit(&runtime.ClientOperation{ - ID: "UploadMtaArchiveFromUrl", - Method: "POST", - PathPattern: "/files", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"multipart/form-data"}, - Schemes: []string{"https"}, - Params: params, - Reader: &UploadMtaArchiveFromUrlReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - }) - if err != nil { - return nil, err - } - return result.(*UploadMtaArchiveFromUrlCreated), nil - -} - /* GetCsrfToken Retrieves a csrf-token header diff --git a/clients/mtaclient/operations/upload_mta_archive_from_url_parameters.go b/clients/mtaclient/operations/upload_mta_archive_from_url_parameters.go deleted file mode 100644 index bd745ca..0000000 --- a/clients/mtaclient/operations/upload_mta_archive_from_url_parameters.go +++ /dev/null @@ -1,157 +0,0 @@ -// Code generated by go-swagger; DO NOT EDIT. - -package operations - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - "time" - - "golang.org/x/net/context" - - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - - "github.com/go-openapi/strfmt" -) - -// NewUploadMtaArchiveFromUrlParams creates a new UploadMtaArchiveFromUrlParams object -// with the default values initialized. -func NewUploadMtaArchiveFromUrlParams() *UploadMtaArchiveFromUrlParams { - return &UploadMtaArchiveFromUrlParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewUploadMtaArchiveFromUrlParamsWithTimeout creates a new UploadMtaArchiveFromUrlParams object -// with the default values initialized, and the ability to set a timeout on a request -func NewUploadMtaArchiveFromUrlParamsWithTimeout(timeout time.Duration) *UploadMtaArchiveFromUrlParams { - return &UploadMtaArchiveFromUrlParams{ - timeout: timeout, - } -} - -// NewUploadMtaArchiveFromUrlParamsWithContext creates a new UploadMtaArchiveFromUrlParams object -// with the default values initialized, and the ability to set a context for a request -func NewUploadMtaArchiveFromUrlParamsWithContext(ctx context.Context) *UploadMtaArchiveFromUrlParams { - return &UploadMtaArchiveFromUrlParams{ - Context: ctx, - } -} - -// NewUploadMtaArchiveFromUrlParamsWithHTTPClient creates a new UploadMtaArchiveFromUrlParams object -// with the default values initialized, and the ability to set a custom HTTPClient for a request -func NewUploadMtaArchiveFromUrlParamsWithHTTPClient(client *http.Client) *UploadMtaArchiveFromUrlParams { - return &UploadMtaArchiveFromUrlParams{ - HTTPClient: client, - } -} - -/*UploadMtaArchiveFromUrlParams contains all the parameters to send to the API endpoint -for the upload mta archive operation typically these are written to a http.Request -*/ -type UploadMtaArchiveFromUrlParams struct { - - /*Namespace - file namespace - - */ - Namespace *string - - /*FileUrl - url to remote file - - */ - FileUrl string - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithTimeout adds the timeout to the upload mta file params -func (o *UploadMtaArchiveFromUrlParams) WithTimeout(timeout time.Duration) *UploadMtaArchiveFromUrlParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the upload mta file params -func (o *UploadMtaArchiveFromUrlParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the upload mta file params -func (o *UploadMtaArchiveFromUrlParams) WithContext(ctx context.Context) *UploadMtaArchiveFromUrlParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the upload mta file params -func (o *UploadMtaArchiveFromUrlParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the upload mta file params -func (o *UploadMtaArchiveFromUrlParams) WithHTTPClient(client *http.Client) *UploadMtaArchiveFromUrlParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the upload mta file params -func (o *UploadMtaArchiveFromUrlParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithNamespace adds the namespace to the upload mta file params -func (o *UploadMtaArchiveFromUrlParams) WithNamespace(namespace *string) *UploadMtaArchiveFromUrlParams { - o.SetNamespace(namespace) - return o -} - -// SetNamespace adds the namespace to the upload mta file params -func (o *UploadMtaArchiveFromUrlParams) SetNamespace(namespace *string) { - o.Namespace = namespace -} - -// WithUrl adds the file url to the upload mta archive params -func (o *UploadMtaArchiveFromUrlParams) WithUrl(fileUrl string) *UploadMtaArchiveFromUrlParams { - o.SetFileUrl(fileUrl) - return o -} - -// SetFileUrl adds the file url to the upload mta archive params -func (o *UploadMtaArchiveFromUrlParams) SetFileUrl(fileUrl string) { - o.FileUrl = fileUrl -} - -// WriteToRequest writes these params to a swagger request -func (o *UploadMtaArchiveFromUrlParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - - if o.Namespace != nil { - - // query param namespace - var qrNamespace string - if o.Namespace != nil { - qrNamespace = *o.Namespace - } - qNamespace := qrNamespace - if qNamespace != "" { - if err := r.SetQueryParam("namespace", qNamespace); err != nil { - return err - } - } - - } - - if err := r.SetHeaderParam("X-File-URL", o.FileUrl); err != nil { - return err - } - - return nil -} diff --git a/clients/mtaclient/operations/upload_mta_archive_from_url_responses.go b/clients/mtaclient/operations/upload_mta_archive_from_url_responses.go deleted file mode 100644 index e9159b4..0000000 --- a/clients/mtaclient/operations/upload_mta_archive_from_url_responses.go +++ /dev/null @@ -1,69 +0,0 @@ -// Code generated by go-swagger; DO NOT EDIT. - -package operations - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - - "github.com/go-openapi/strfmt" - - "github.com/cloudfoundry-incubator/multiapps-cli-plugin/clients/baseclient" - "github.com/cloudfoundry-incubator/multiapps-cli-plugin/clients/models" -) - -// UploadMtaArchiveFromUrlReader is a Reader for the UploadMtaArchiveFromUrl structure. -type UploadMtaArchiveFromUrlReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *UploadMtaArchiveFromUrlReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - - case 201: - result := NewUploadMtaArchiveFromUrlCreated() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - - default: - return nil, baseclient.BuildErrorResponse(response, consumer, o.formats) - } -} - -// NewUploadMtaArchiveFromUrlCreated creates a UploadMtaArchiveFromUrlCreated with default headers values -func NewUploadMtaArchiveFromUrlCreated() *UploadMtaArchiveFromUrlCreated { - return &UploadMtaArchiveFromUrlCreated{} -} - -/*UploadMtaArchiveFromUrlCreated handles this case with default header values. - -Created -*/ -type UploadMtaArchiveFromUrlCreated struct { - Payload *models.FileMetadata -} - -func (o *UploadMtaArchiveFromUrlCreated) Error() string { - return fmt.Sprintf("[POST /files][%d] UploadMtaArchiveFromUrlCreated %+v", 201, o.Payload) -} - -func (o *UploadMtaArchiveFromUrlCreated) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.FileMetadata) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - diff --git a/clients/mtaclient/operations/upload_mta_file_parameters.go b/clients/mtaclient/operations/upload_mta_file_parameters.go deleted file mode 100644 index cef4075..0000000 --- a/clients/mtaclient/operations/upload_mta_file_parameters.go +++ /dev/null @@ -1,177 +0,0 @@ -// Code generated by go-swagger; DO NOT EDIT. - -package operations - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - "os" - "strconv" - "time" - - "golang.org/x/net/context" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - - strfmt "github.com/go-openapi/strfmt" -) - -// NewUploadMtaFileParams creates a new UploadMtaFileParams object -// with the default values initialized. -func NewUploadMtaFileParams() *UploadMtaFileParams { - return &UploadMtaFileParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewUploadMtaFileParamsWithTimeout creates a new UploadMtaFileParams object -// with the default values initialized, and the ability to set a timeout on a request -func NewUploadMtaFileParamsWithTimeout(timeout time.Duration) *UploadMtaFileParams { - return &UploadMtaFileParams{ - timeout: timeout, - } -} - -// NewUploadMtaFileParamsWithContext creates a new UploadMtaFileParams object -// with the default values initialized, and the ability to set a context for a request -func NewUploadMtaFileParamsWithContext(ctx context.Context) *UploadMtaFileParams { - return &UploadMtaFileParams{ - Context: ctx, - } -} - -// NewUploadMtaFileParamsWithHTTPClient creates a new UploadMtaFileParams object -// with the default values initialized, and the ability to set a custom HTTPClient for a request -func NewUploadMtaFileParamsWithHTTPClient(client *http.Client) *UploadMtaFileParams { - return &UploadMtaFileParams{ - HTTPClient: client, - } -} - -/*UploadMtaFileParams contains all the parameters to send to the API endpoint -for the upload mta file operation typically these are written to a http.Request -*/ -type UploadMtaFileParams struct { - - /*Namespace - file namespace - - */ - Namespace *string - - /*File*/ - File os.File - - FileSize int64 - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithTimeout adds the timeout to the upload mta file params -func (o *UploadMtaFileParams) WithTimeout(timeout time.Duration) *UploadMtaFileParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the upload mta file params -func (o *UploadMtaFileParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the upload mta file params -func (o *UploadMtaFileParams) WithContext(ctx context.Context) *UploadMtaFileParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the upload mta file params -func (o *UploadMtaFileParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the upload mta file params -func (o *UploadMtaFileParams) WithHTTPClient(client *http.Client) *UploadMtaFileParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the upload mta file params -func (o *UploadMtaFileParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithNamespace adds the namespace to the upload mta file params -func (o *UploadMtaFileParams) WithNamespace(namespace *string) *UploadMtaFileParams { - o.SetNamespace(namespace) - return o -} - -// SetNamespace adds the namespace to the upload mta file params -func (o *UploadMtaFileParams) SetNamespace(namespace *string) { - o.Namespace = namespace -} - -// WithFile adds the file to the upload mta file params -func (o *UploadMtaFileParams) WithFile(file os.File) *UploadMtaFileParams { - o.SetFile(file) - return o -} - -// SetFile adds the file to the upload mta file params -func (o *UploadMtaFileParams) SetFile(file os.File) { - o.File = file -} - -func (o *UploadMtaFileParams) WithFileSize(fileSize int64) *UploadMtaFileParams { - o.SetFileSize(fileSize) - return o -} - -func (o *UploadMtaFileParams) SetFileSize(fileSize int64) { - o.FileSize = fileSize -} - -// WriteToRequest writes these params to a swagger request -func (o *UploadMtaFileParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - - if o.Namespace != nil { - - // query param namespace - var qrNamespace string - if o.Namespace != nil { - qrNamespace = *o.Namespace - } - qNamespace := qrNamespace - if qNamespace != "" { - if err := r.SetQueryParam("namespace", qNamespace); err != nil { - return err - } - } - - } - - // form file param file - if err := r.SetFileParam("file", &o.File); err != nil { - return err - } - - if err := r.SetHeaderParam("X-File-Size", strconv.FormatInt(o.FileSize, 10)); err != nil { - return err - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/clients/mtaclient/operations/upload_mta_file_responses.go b/clients/mtaclient/operations/upload_mta_file_responses.go deleted file mode 100644 index 0e2bab1..0000000 --- a/clients/mtaclient/operations/upload_mta_file_responses.go +++ /dev/null @@ -1,68 +0,0 @@ -// Code generated by go-swagger; DO NOT EDIT. - -package operations - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - - strfmt "github.com/go-openapi/strfmt" - - baseclient "github.com/cloudfoundry-incubator/multiapps-cli-plugin/clients/baseclient" - "github.com/cloudfoundry-incubator/multiapps-cli-plugin/clients/models" -) - -// UploadMtaFileReader is a Reader for the UploadMtaFile structure. -type UploadMtaFileReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *UploadMtaFileReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - - case 201: - result := NewUploadMtaFileCreated() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - - default: - return nil, baseclient.BuildErrorResponse(response, consumer, o.formats) - } -} - -// NewUploadMtaFileCreated creates a UploadMtaFileCreated with default headers values -func NewUploadMtaFileCreated() *UploadMtaFileCreated { - return &UploadMtaFileCreated{} -} - -/*UploadMtaFileCreated handles this case with default header values. - -Created -*/ -type UploadMtaFileCreated struct { - Payload *models.FileMetadata -} - -func (o *UploadMtaFileCreated) Error() string { - return fmt.Sprintf("[POST /files][%d] uploadMtaFileCreated %+v", 201, o.Payload) -} - -func (o *UploadMtaFileCreated) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.FileMetadata) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/clients/mtaclient/retryable_mta_rest_client.go b/clients/mtaclient/retryable_mta_rest_client.go index 2132de9..b750cc3 100644 --- a/clients/mtaclient/retryable_mta_rest_client.go +++ b/clients/mtaclient/retryable_mta_rest_client.go @@ -1,12 +1,13 @@ package mtaclient import ( + "io" "net/http" - "os" "time" "github.com/cloudfoundry-incubator/multiapps-cli-plugin/clients/baseclient" "github.com/cloudfoundry-incubator/multiapps-cli-plugin/clients/models" + "github.com/cloudfoundry-incubator/multiapps-cli-plugin/util" ) type RetryableMtaRestClient struct { @@ -119,14 +120,14 @@ func (c RetryableMtaRestClient) StartMtaOperation(operation models.Operation) (R return resp.(ResponseHeader), nil } -func (c RetryableMtaRestClient) UploadMtaFile(file os.File, fileSize int64, namespace *string) (*models.FileMetadata, error) { +func (c RetryableMtaRestClient) UploadMtaFile(file util.NamedReadSeeker, fileSize int64, namespace *string) (*models.FileMetadata, error) { uploadMtaFileCb := func() (interface{}, error) { - reopenedFile, err := os.Open(file.Name()) + //rewind the file part to the beginning in case of a network error + _, err := file.Seek(0, io.SeekStart) if err != nil { return nil, err } - - return c.mtaClient.UploadMtaFile(*reopenedFile, fileSize, namespace) + return c.mtaClient.UploadMtaFile(file, fileSize, namespace) } resp, err := baseclient.CallWithRetry(uploadMtaFileCb, c.MaxRetriesCount, c.RetryInterval) if err != nil { @@ -135,15 +136,26 @@ func (c RetryableMtaRestClient) UploadMtaFile(file os.File, fileSize int64, name return resp.(*models.FileMetadata), nil } -func (c RetryableMtaRestClient) UploadMtaArchiveFromUrl(fileUrl string, namespace *string) (*models.FileMetadata, error) { +func (c RetryableMtaRestClient) StartUploadMtaArchiveFromUrl(fileUrl string, namespace *string) (http.Header, error) { uploadMtaArchiveFromUrlCb := func() (interface{}, error) { - return c.mtaClient.UploadMtaArchiveFromUrl(fileUrl, namespace) + return c.mtaClient.StartUploadMtaArchiveFromUrl(fileUrl, namespace) } resp, err := baseclient.CallWithRetry(uploadMtaArchiveFromUrlCb, c.MaxRetriesCount, c.RetryInterval) if err != nil { return nil, err } - return resp.(*models.FileMetadata), nil + return resp.(http.Header), nil +} + +func (c RetryableMtaRestClient) GetAsyncUploadJob(jobId string, namespace *string, appInstanceId string) (AsyncUploadJobResult, error) { + getAsyncUploadJobCb := func() (interface{}, error) { + return c.mtaClient.GetAsyncUploadJob(jobId, namespace, appInstanceId) + } + resp, err := baseclient.CallWithRetry(getAsyncUploadJobCb, c.MaxRetriesCount, c.RetryInterval) + if err != nil { + return AsyncUploadJobResult{}, err + } + return resp.(AsyncUploadJobResult), nil } func (c RetryableMtaRestClient) GetMtaOperationLogContent(operationID, logID string) (string, error) { diff --git a/commands/action.go b/commands/action.go index 9ea713a..640671f 100644 --- a/commands/action.go +++ b/commands/action.go @@ -71,11 +71,11 @@ func (a *action) Execute(operationID string, mtaClient mtaclient.MtaClientOperat func (a *action) executeInSession(operationID string, mtaClient mtaclient.MtaClientOperations) ExecutionStatus { if a.verbosityLevel == VerbosityLevelVERBOSE { - ui.Say("Executing action '%s' on operation %s...", a.actionID, terminal.EntityNameColor(operationID)) + ui.Say("Executing action %q on operation %s...", a.actionID, terminal.EntityNameColor(operationID)) } _, err := mtaClient.ExecuteAction(operationID, a.actionID) if err != nil { - ui.Failed("Could not execute action '%s' on operation %s: %s", a.actionID, terminal.EntityNameColor(operationID), err) + ui.Failed("Could not execute action %q on operation %s: %s", a.actionID, terminal.EntityNameColor(operationID), err) return Failure } if a.verbosityLevel == VerbosityLevelVERBOSE { diff --git a/commands/action_test.go b/commands/action_test.go index 0686994..18b660e 100644 --- a/commands/action_test.go +++ b/commands/action_test.go @@ -17,7 +17,7 @@ import ( var _ = Describe("Actions", func() { const operationID = "test-process-id" const commandName = "deploy" - var mtaClient fakes.FakeMtaClientOperations + var mtaClient *fakes.FakeMtaClientOperations var action commands.Action var oc = testutil.NewUIOutputCapturer() @@ -42,7 +42,7 @@ var _ = Describe("Actions", func() { output, status := oc.CaptureOutputAndStatus(func() int { return action.Execute(operationID, mtaClient).ToInt() }) - ex.ExpectSuccessWithOutput(status, output, []string{"Executing action 'abort' on operation test-process-id...", "OK"}) + ex.ExpectSuccessWithOutput(status, output, []string{"Executing action \"abort\" on operation test-process-id...", "OK"}) }) }) Context("with an error returned from the backend", func() { @@ -54,7 +54,7 @@ var _ = Describe("Actions", func() { output, status := oc.CaptureOutputAndStatus(func() int { return action.Execute(operationID, mtaClient).ToInt() }) - ex.ExpectFailureOnLine(status, output, "Could not execute action 'abort' on operation test-process-id: test-error", 2) + ex.ExpectFailureOnLine(status, output, "Could not execute action \"abort\" on operation test-process-id: test-error", 2) }) }) }) @@ -76,7 +76,7 @@ var _ = Describe("Actions", func() { output, status := oc.CaptureOutputAndStatus(func() int { return action.Execute(operationID, mtaClient).ToInt() }) - ex.ExpectSuccessWithOutput(status, output, []string{"Executing action 'retry' on operation test-process-id...", "OK", + ex.ExpectSuccessWithOutput(status, output, []string{"Executing action \"retry\" on operation test-process-id...", "OK", "Process finished.", "Use \"cf dmol -i " + operationID + "\" to download the logs of the process."}) }) }) @@ -90,7 +90,7 @@ var _ = Describe("Actions", func() { output, status := oc.CaptureOutputAndStatus(func() int { return action.Execute(operationID, mtaClient).ToInt() }) - ex.ExpectFailureOnLine(status, output, "Could not execute action 'retry' on operation test-process-id: test-error", 2) + ex.ExpectFailureOnLine(status, output, "Could not execute action \"retry\" on operation test-process-id: test-error", 2) }) }) }) diff --git a/commands/base_command.go b/commands/base_command.go index c089d4d..072dc9e 100644 --- a/commands/base_command.go +++ b/commands/base_command.go @@ -1,7 +1,6 @@ package commands import ( - "crypto/tls" "flag" "fmt" "io" @@ -59,7 +58,7 @@ type BaseCommand struct { // Initialize initializes the command with the specified name and CLI connection func (c *BaseCommand) Initialize(name string, cliConnection plugin.CliConnection) { - log.Tracef("Initializing command '%s'\n", name) + log.Tracef("Initializing command %q\n", name) transport := newTransport() tokenFactory := NewDefaultTokenFactory(cliConnection) c.InitializeAll(name, cliConnection, transport, clients.NewDefaultClientFactory(), tokenFactory, util.NewDeployServiceURLCalculator(cliConnection)) @@ -267,11 +266,9 @@ func (c *BaseCommand) shouldAbortConflictingOperation(mtaID string, force bool) func newTransport() http.RoundTripper { csrfx := csrf.Csrf{Header: "", Token: "", IsInitialized: false, NonProtectedMethods: getNonProtectedMethods()} - // TODO Make sure SSL verification is only skipped if the CLI is configured this way httpTransport := http.DefaultTransport.(*http.Transport) - // Increase tls handshake timeout to cope with of slow internet connection. 3 x default value =30s. + // Increase tls handshake timeout to cope with slow internet connections. 3 x default value =30s. httpTransport.TLSHandshakeTimeout = 30 * time.Second - httpTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} return csrf.Transport{OriginalTransport: httpTransport, Csrf: &csrfx, Cookies: &csrf.Cookies{[]*http.Cookie{}}} } diff --git a/commands/base_command_test.go b/commands/base_command_test.go index 6484df4..98502bc 100644 --- a/commands/base_command_test.go +++ b/commands/base_command_test.go @@ -79,7 +79,7 @@ var _ = Describe("BaseCommand", func() { }) Expect(err).ShouldNot(HaveOccurred()) Expect(wasAborted).To(BeTrue()) - Expect(output).To(Equal([]string{"Executing action 'abort' on operation test...", "OK"})) + Expect(output).To(Equal([]string{"Executing action \"abort\" on operation test...", "OK"})) }) }) Context("with one ongoing operation which does not have an MTA ID", func() { @@ -118,7 +118,7 @@ var _ = Describe("BaseCommand", func() { }) Expect(err).ShouldNot(HaveOccurred()) Expect(wasAborted).To(BeTrue()) - Expect(output).To(Equal([]string{"Executing action 'abort' on operation test...", "OK"})) + Expect(output).To(Equal([]string{"Executing action \"abort\" on operation test...", "OK"})) }) }) Context("with valid ongoing operations and no force option specified", func() { @@ -156,7 +156,7 @@ var _ = Describe("BaseCommand", func() { output, status := oc.CaptureOutputAndStatus(func() int { return command.ExecuteAction("test-process-id", "abort", 0, "test-host", cfTarget).ToInt() }) - ex.ExpectSuccessWithOutput(status, output, []string{"Executing action 'abort' on operation test-process-id...", "OK"}) + ex.ExpectSuccessWithOutput(status, output, []string{"Executing action \"abort\" on operation test-process-id...", "OK"}) }) }) Context("with non-valid process id and valid action id", func() { @@ -182,7 +182,7 @@ var _ = Describe("BaseCommand", func() { output, status := oc.CaptureOutputAndStatus(func() int { return command.ExecuteAction("test-process-id", "retry", 0, "test-host", cfTarget).ToInt() }) - ex.ExpectSuccessWithOutput(status, output, []string{"Executing action 'retry' on operation test-process-id...", "OK", + ex.ExpectSuccessWithOutput(status, output, []string{"Executing action \"retry\" on operation test-process-id...", "OK", "Process finished.", "Use \"cf dmol -i test-process-id\" to download the logs of the process."}) }) }) diff --git a/commands/blue_green_deploy_command.go b/commands/blue_green_deploy_command.go index 119f2fe..52ab269 100644 --- a/commands/blue_green_deploy_command.go +++ b/commands/blue_green_deploy_command.go @@ -4,6 +4,7 @@ import ( "flag" "os" "strconv" + "time" "code.cloudfoundry.org/cli/plugin" "github.com/cloudfoundry-incubator/multiapps-cli-plugin/util" @@ -19,7 +20,7 @@ type BlueGreenDeployCommand struct { // NewBlueGreenDeployCommand creates a new BlueGreenDeployCommand. func NewBlueGreenDeployCommand() *BlueGreenDeployCommand { baseCmd := &BaseCommand{flagsParser: deployCommandLineArgumentsParser{}, flagsValidator: deployCommandFlagsValidator{}} - deployCmd := &DeployCommand{baseCmd, blueGreenDeployProcessParametersSetter(), &blueGreenDeployCommandProcessTypeProvider{}, os.Stdin} + deployCmd := &DeployCommand{baseCmd, blueGreenDeployProcessParametersSetter(), &blueGreenDeployCommandProcessTypeProvider{}, os.Stdin, 30 * time.Second} bgDeployCmd := &BlueGreenDeployCommand{deployCmd} baseCmd.Command = bgDeployCmd return bgDeployCmd diff --git a/commands/default_token_factory.go b/commands/default_token_factory.go index 87bc836..cefae48 100644 --- a/commands/default_token_factory.go +++ b/commands/default_token_factory.go @@ -27,12 +27,17 @@ func NewDefaultTokenFactory(cliConnection plugin.CliConnection) *DefaultTokenFac // NewToken retrives outh token func (t *DefaultTokenFactory) NewToken() (runtime.ClientAuthInfoWriter, error) { + rawToken, err := t.NewRawToken() + return client.BearerToken(rawToken), err +} + +func (t *DefaultTokenFactory) NewRawToken() (string, error) { var expirationTime int64 if t.cachedToken != "" { var err error expirationTime, err = getTokenExpirationTime(t.cachedToken) if err != nil { - return nil, err + return "", err } } currentTimeInSeconds := time.Now().Unix() @@ -40,12 +45,12 @@ func (t *DefaultTokenFactory) NewToken() (runtime.ClientAuthInfoWriter, error) { if currentTimeInSeconds-t.cachedTokenTime >= expirationTime { tokenString, err := t.cliConnection.AccessToken() if err != nil { - return nil, fmt.Errorf("Could not get access token: %s", err) + return "", fmt.Errorf("Could not get access token: %s", err) } t.cachedTokenTime = currentTimeInSeconds t.cachedToken = getTokenValue(tokenString) } - return client.BearerToken(t.cachedToken), nil + return t.cachedToken, nil } func getTokenExpirationTime(tokenString string) (int64, error) { diff --git a/commands/deploy_command.go b/commands/deploy_command.go index d33889b..c04a494 100644 --- a/commands/deploy_command.go +++ b/commands/deploy_command.go @@ -1,26 +1,29 @@ package commands import ( - "bytes" + "bufio" + "code.cloudfoundry.org/cli/cf/terminal" + "code.cloudfoundry.org/cli/plugin" "encoding/base64" "errors" "flag" "fmt" - "io" + "github.com/cloudfoundry-incubator/multiapps-cli-plugin/clients/baseclient" + "github.com/cloudfoundry-incubator/multiapps-cli-plugin/clients/models" + "github.com/cloudfoundry-incubator/multiapps-cli-plugin/clients/mtaclient" + "github.com/cloudfoundry-incubator/multiapps-cli-plugin/configuration" + "github.com/cloudfoundry-incubator/multiapps-cli-plugin/log" + "github.com/cloudfoundry-incubator/multiapps-cli-plugin/ui" + "github.com/cloudfoundry-incubator/multiapps-cli-plugin/util" + "gopkg.in/cheggaaa/pb.v1" + "io/fs" + "net/http" "net/url" "os" "path/filepath" "strconv" "strings" "time" - - "code.cloudfoundry.org/cli/cf/terminal" - "code.cloudfoundry.org/cli/plugin" - "github.com/cloudfoundry-incubator/multiapps-cli-plugin/clients/baseclient" - "github.com/cloudfoundry-incubator/multiapps-cli-plugin/clients/models" - "github.com/cloudfoundry-incubator/multiapps-cli-plugin/configuration" - "github.com/cloudfoundry-incubator/multiapps-cli-plugin/ui" - "github.com/cloudfoundry-incubator/multiapps-cli-plugin/util" ) const ( @@ -70,13 +73,14 @@ type DeployCommand struct { setProcessParameters ProcessParametersSetter processTypeProvider ProcessTypeProvider - FileUrlReader io.Reader + FileUrlReader fs.File + FileUrlReadTimeout time.Duration } // NewDeployCommand creates a new deploy command. func NewDeployCommand() *DeployCommand { baseCmd := &BaseCommand{flagsParser: deployCommandLineArgumentsParser{}, flagsValidator: deployCommandFlagsValidator{}} - deployCmd := &DeployCommand{baseCmd, deployProcessParametersSetter(), &deployCommandProcessTypeProvider{}, os.Stdin} + deployCmd := &DeployCommand{baseCmd, deployProcessParametersSetter(), &deployCommandProcessTypeProvider{}, os.Stdin, 30 * time.Second} baseCmd.Command = deployCmd return deployCmd } @@ -211,23 +215,35 @@ func (c *DeployCommand) executeInternal(positionalArgs []string, dsHost string, mtaClient := c.NewMtaClient(dsHost, cfTarget) namespace := strings.TrimSpace(GetStringOpt(namespaceOpt, flags)) + force := GetBoolOpt(forceOpt, flags) uploadChunkSizeInMB := configuration.NewSnapshot().GetUploadChunkSizeInMB() fileUploader := NewFileUploader(mtaClient, namespace, uploadChunkSizeInMB) if isUrl { - encodedFileUrl := base64.URLEncoding.EncodeToString([]byte(mtaArchive)) - uploadedArchive, err := mtaClient.UploadMtaArchiveFromUrl(encodedFileUrl, &namespace) + var fileId string + var isFailure bool + fileId, mtaId, isFailure = c.uploadFromUrl(mtaArchive, mtaClient, namespace) + if isFailure { + return Failure + } + + // Check for an ongoing operation for this MTA ID and abort it + wasAborted, err := c.CheckOngoingOperation(mtaId, namespace, dsHost, force, cfTarget) if err != nil { - ui.Failed("Could not upload from url: %s", baseclient.NewClientError(err)) + ui.Failed("Could not get MTA operations: %s", baseclient.NewClientError(err)) return Failure } - uploadedArchivePartIds = append(uploadedArchivePartIds, uploadedArchive.ID) + if !wasAborted { + return Failure + } + + uploadedArchivePartIds = append(uploadedArchivePartIds, fileId) ui.Ok() } else { // Get the full path of the MTA archive mtaArchivePath, err := filepath.Abs(mtaArchive) if err != nil { - ui.Failed("Could not get absolute path of file '%s'", mtaArchive) + ui.Failed("Could not get absolute path of file %q", mtaArchive) return Failure } @@ -242,9 +258,8 @@ func (c *DeployCommand) executeInternal(positionalArgs []string, dsHost string, } mtaId = descriptor.ID - force := GetBoolOpt(forceOpt, flags) // Check for an ongoing operation for this MTA ID and abort it - wasAborted, err := c.CheckOngoingOperation(descriptor.ID, namespace, dsHost, force, cfTarget) + wasAborted, err := c.CheckOngoingOperation(mtaId, namespace, dsHost, force, cfTarget) if err != nil { ui.Failed("Could not get MTA operations: %s", baseclient.NewClientError(err)) return Failure @@ -268,7 +283,7 @@ func (c *DeployCommand) executeInternal(positionalArgs []string, dsHost string, for _, extDescriptorFile := range extDescriptorFiles { extDescriptorPath, err := filepath.Abs(extDescriptorFile) if err != nil { - ui.Failed("Could not get absolute path of file '%s'", extDescriptorFile) + ui.Failed("Could not get absolute path of file %q", extDescriptorFile) return Failure } extDescriptorPaths = append(extDescriptorPaths, extDescriptorPath) @@ -285,9 +300,7 @@ func (c *DeployCommand) executeInternal(positionalArgs []string, dsHost string, processBuilder.Namespace(namespace) processBuilder.Parameter("appArchiveId", strings.Join(uploadedArchivePartIds, ",")) processBuilder.Parameter("mtaExtDescriptorId", strings.Join(uploadedExtDescriptorIDs, ",")) - if !isUrl { - processBuilder.Parameter("mtaId", mtaId) - } + processBuilder.Parameter("mtaId", mtaId) setModulesAndResourcesListParameters(modulesList, resourcesList, processBuilder, mtaElementsCalculator) c.setProcessParameters(flags, processBuilder) @@ -313,8 +326,65 @@ func parseMtaArchiveArgument(rawMtaArchive interface{}) (bool, string) { return false, "" } +func (c *DeployCommand) uploadFromUrl(url string, mtaClient mtaclient.MtaClientOperations, namespace string) (fileId, mtaId string, failure bool) { + progressBar := c.tryFetchMtarSize(url) + + encodedFileUrl := base64.URLEncoding.EncodeToString([]byte(url)) + responseHeaders, err := mtaClient.StartUploadMtaArchiveFromUrl(encodedFileUrl, &namespace) + if err != nil { + ui.Failed("Could not upload from url: %s", err) + return "", "", true + } + + var totalBytesProcessed int64 = 0 + if progressBar != nil { + progressBar.Start() + defer progressBar.Finish() + } + + uploadJobUrl := responseHeaders.Get("Location") + jobUrlParts := strings.Split(uploadJobUrl, "/") + jobId := jobUrlParts[len(jobUrlParts)-1] + + var file *models.FileMetadata + for file == nil { + jobResult, err := mtaClient.GetAsyncUploadJob(jobId, &namespace, responseHeaders.Get("x-cf-app-instance")) + if err != nil { + ui.Failed("Could not upload from url: %s", err) + return "", "", true + } + file, mtaId = jobResult.File, jobResult.MtaId + + if len(jobResult.Error) != 0 { + ui.Failed("Async upload job failed: %s", jobResult.Error) + return "", "", true + } + + if progressBar != nil && jobResult.BytesProcessed != -1 { + if jobResult.BytesProcessed < totalBytesProcessed { + //retry happened in backend, rewind the progress bar + progressBar.Add64(-totalBytesProcessed + jobResult.BytesProcessed) + } else { + progressBar.Add64(jobResult.BytesProcessed - totalBytesProcessed) + } + totalBytesProcessed = jobResult.BytesProcessed + } + + if len(mtaId) == 0 { + time.Sleep(2 * time.Second) + } + } + if progressBar != nil && totalBytesProcessed < progressBar.Total { + progressBar.Add64(progressBar.Total - totalBytesProcessed) + } + return file.ID, mtaId, false +} + func (c *DeployCommand) uploadFiles(files []string, fileUploader *FileUploader) ([]string, ExecutionStatus) { var resultIds []string + if len(files) == 0 { + return resultIds, Success + } uploadedFiles, status := fileUploader.UploadFiles(files) if status == Failure { @@ -364,18 +434,34 @@ func (c *DeployCommand) getMtaArchive(parsedArguments []string, mtaElementsCalcu } func (c *DeployCommand) tryReadingFileUrl() string { - fileUrlChan := make(chan []byte) - go func() { - fileUrl, _ := io.ReadAll(c.FileUrlReader) - fileUrlChan <- bytes.TrimSpace(fileUrl) - }() - - select { - case fileUrl := <-fileUrlChan: - return string(fileUrl) - case <-time.After(time.Millisecond * 100): + stat, err := c.FileUrlReader.Stat() + if err != nil { return "" } + + if stat.Mode()&fs.ModeCharDevice == 0 { + in := bufio.NewReader(c.FileUrlReader) + input, _ := in.ReadString('\n') + return strings.TrimSpace(input) + } + return "" +} + +func (c *DeployCommand) tryFetchMtarSize(url string) *pb.ProgressBar { + client := http.Client{Timeout: c.FileUrlReadTimeout} + resp, err := client.Head(url) + if err != nil { + log.Tracef("could not call remote MTAR endpoint: %v\n", err) + return nil + } + if resp.StatusCode/100 != 2 { + log.Tracef("could not fetch remote MTAR size: %s\n", resp.Status) + return nil + } + bar := pb.New64(resp.ContentLength).SetUnits(pb.U_BYTES) + bar.ShowElapsedTime = true + bar.ShowTimeLeft = false + return bar } func buildMtaArchiveFromDirectory(mtaDirectoryLocation string, mtaElementsCalculator mtaElementsToAddCalculator) (string, error) { diff --git a/commands/deploy_command_test.go b/commands/deploy_command_test.go index fea3451..75a8b47 100644 --- a/commands/deploy_command_test.go +++ b/commands/deploy_command_test.go @@ -3,9 +3,13 @@ package commands_test import ( "encoding/base64" "fmt" + "io" + "io/fs" + "net/http" "os" "path/filepath" "strings" + "time" cli_fakes "github.com/cloudfoundry-incubator/multiapps-cli-plugin/cli/fakes" "github.com/cloudfoundry-incubator/multiapps-cli-plugin/clients/models" @@ -22,6 +26,48 @@ import ( . "github.com/onsi/gomega" ) +type mockFile struct { + reader io.Reader +} + +type mockFileInfo struct{} + +func (mockFileInfo) Name() string { + return "" +} + +func (mockFileInfo) Size() int64 { + return 0 +} + +func (mockFileInfo) Mode() fs.FileMode { + return ^fs.ModeCharDevice +} + +func (mockFileInfo) ModTime() time.Time { + return time.Time{} +} + +func (mockFileInfo) IsDir() bool { + return false +} + +func (mockFileInfo) Sys() any { + return nil +} + +func (f *mockFile) Stat() (fs.FileInfo, error) { + return mockFileInfo{}, nil +} + +func (f *mockFile) Read(p []byte) (int, error) { + return f.reader.Read(p) +} + +func (f *mockFile) Close() error { + return nil +} + var _ = Describe("DeployCommand", func() { Describe("Execute", func() { const org = "test-org" @@ -36,7 +82,7 @@ var _ = Describe("DeployCommand", func() { var name string var cliConnection *plugin_fakes.FakeCliConnection // var fakeSession csrffake.FakeSessionProvider - var mtaClient mtafake.FakeMtaClientOperations + var mtaClient *mtafake.FakeMtaClientOperations // var restClient *restfake.FakeRestClientOperations var testClientFactory *commands.TestClientFactory var command *commands.DeployCommand @@ -45,12 +91,16 @@ var _ = Describe("DeployCommand", func() { var fullMtaArchivePath, _ = filepath.Abs(mtaArchivePath) var fullExtDescriptorPath, _ = filepath.Abs(extDescriptorPath) - var correctMtaUrl = "https://host/path/anatz.mtar?query=true" - var incorrectMtaUrl = "https://alabala.com" + var correctMtaUrl = "https://localhost/path/anatz.mtar?query=true" + var incorrectMtaUrl = "https://localhost" + + var newMockFileReader = func(url string) fs.File { + return &mockFile{strings.NewReader(url)} + } var getLinesForAbortingProcess = func() []string { return []string{ - "Executing action 'abort' on operation test-process-id...", + "Executing action \"abort\" on operation test-process-id...", "OK", } } @@ -66,7 +116,7 @@ var _ = Describe("DeployCommand", func() { lines = append(lines, "") if processAborted { lines = append(lines, - "Executing action 'abort' on operation test-process-id...", + "Executing action \"abort\" on operation test-process-id...", "OK", ) } @@ -107,7 +157,7 @@ var _ = Describe("DeployCommand", func() { var getFile = func(path string) (*os.File, *models.FileMetadata) { file, _ := os.Open(path) digest, _ := util.ComputeFileChecksum(path, "MD5") - f := testutil.GetFile(*file, strings.ToUpper(digest), namespace) + f := testutil.GetFile(file, strings.ToUpper(digest), namespace) return file, f } @@ -131,12 +181,20 @@ var _ = Describe("DeployCommand", func() { defer mtaArchiveFile.Close() extDescriptorFile, extDescriptor := getFile(extDescriptorPath) defer extDescriptorFile.Close() + fileUploadJobId := make(http.Header) + jobId := "one" + fileUploadJobId.Add("Location", jobId) + jobResult := mtaclient.AsyncUploadJobResult{ + File: mtaArchive, + MtaId: "anatz", + } mtaClient = mtafake.NewFakeMtaClientBuilder(). GetMtaFiles([]*models.FileMetadata{&testutil.SimpleFile}, nil). - UploadMtaFile(*mtaArchiveFile, mtaArchive, nil). - UploadMtaFile(*extDescriptorFile, extDescriptor, nil). - UploadMtaArchiveFromUrl(base64.URLEncoding.EncodeToString([]byte(correctMtaUrl)), mtaArchive, nil). - UploadMtaArchiveFromUrl(base64.URLEncoding.EncodeToString([]byte(incorrectMtaUrl)), nil, fmt.Errorf("connection refused")). + UploadMtaFile(mtaArchiveFile, mtaArchive, nil). + UploadMtaFile(extDescriptorFile, extDescriptor, nil). + StartUploadMtaArchiveFromUrl(base64.URLEncoding.EncodeToString([]byte(correctMtaUrl)), nil, fileUploadJobId, nil). + StartUploadMtaArchiveFromUrl(base64.URLEncoding.EncodeToString([]byte(incorrectMtaUrl)), nil, nil, fmt.Errorf("connection refused")). + GetAsyncUploadJob(jobId, nil, "", jobResult, nil). StartMtaOperation(testutil.OperationResult, mtaclient.ResponseHeader{Location: "operations/1000?embed=messages"}, nil). GetMtaOperation("1000", "messages", &testutil.OperationResult, nil). GetMtaOperationLogContent("1000", testutil.LogID, testutil.LogContent, nil). @@ -146,6 +204,7 @@ var _ = Describe("DeployCommand", func() { testTokenFactory := commands.NewTestTokenFactory(cliConnection) deployServiceURLCalculator := util_fakes.NewDeployServiceURLFakeCalculator("deploy-service.test.ondemand.com") command.InitializeAll(name, cliConnection, testutil.NewCustomTransport(200), testClientFactory, testTokenFactory, deployServiceURLCalculator) + command.FileUrlReadTimeout = time.Second }) // unknown flag - error @@ -171,8 +230,7 @@ var _ = Describe("DeployCommand", func() { Context("with a correct URL argument", func() { It("should upload the MTAR from the correct URL and initiate a deploy", func() { - reader := strings.NewReader(correctMtaUrl) - command.FileUrlReader = reader + command.FileUrlReader = newMockFileReader(correctMtaUrl) output, status := oc.CaptureOutputAndStatus(func() int { return command.Execute([]string{}).ToInt() }) @@ -182,8 +240,7 @@ var _ = Describe("DeployCommand", func() { Context("with an incorrect URL argument", func() { It("should fail with the error returned from the server", func() { - reader := strings.NewReader(incorrectMtaUrl) - command.FileUrlReader = reader + command.FileUrlReader = newMockFileReader(incorrectMtaUrl) output, status := oc.CaptureOutputAndStatus(func() int { return command.Execute([]string{}).ToInt() }) @@ -306,7 +363,7 @@ var _ = Describe("DeployCommand", func() { // existing ongoing operations and force option not supplied - success Context("with correct mta id from archive, with ongoing operations provided and no force option", func() { - It("should not try to abort confliction operations", func() { + It("should not try to abort conflicting operations", func() { output, status := oc.CaptureOutputAndStatus(func() int { return command.Execute([]string{mtaArchivePath}).ToInt() }) @@ -330,7 +387,7 @@ var _ = Describe("DeployCommand", func() { // }) // }) Context("with an error returned from getting ongoing operations", func() { - It("should display error and exit witn non-zero status", func() { + It("should display error and exit with non-zero status", func() { testClientFactory.MtaClient = mtafake.NewFakeMtaClientBuilder(). GetMtaOperations(nil, nil, nil, []*models.Operation{}, fmt.Errorf("test-error-from backend")).Build() output, status := oc.CaptureOutputAndStatus(func() int { diff --git a/commands/download_mta_op_logs_command_test.go b/commands/download_mta_op_logs_command_test.go index 25512fc..54f58e5 100644 --- a/commands/download_mta_op_logs_command_test.go +++ b/commands/download_mta_op_logs_command_test.go @@ -29,7 +29,7 @@ var _ = Describe("DownloadMtaOperationLogsCommand", func() { var name string var cliConnection *plugin_fakes.FakeCliConnection - var mtaClient mtafake.FakeMtaClientOperations + var mtaClient *mtafake.FakeMtaClientOperations // var restClient *restfake.FakeRestClientOperations var clientFactory *commands.TestClientFactory var command *commands.DownloadMtaOperationLogsCommand diff --git a/commands/execution_monitor.go b/commands/execution_monitor.go index 551cfbf..5d142f3 100644 --- a/commands/execution_monitor.go +++ b/commands/execution_monitor.go @@ -15,7 +15,7 @@ import ( mtaclient "github.com/cloudfoundry-incubator/multiapps-cli-plugin/clients/mtaclient" ) -//ExecutionMonitor monitors execution of a process +// ExecutionMonitor monitors execution of a process type ExecutionMonitor struct { mtaClient mtaclient.MtaClientOperations reportedMessages map[int64]bool @@ -45,7 +45,7 @@ func getMonitoringInformation(monitoringLocation string) (string, string) { return strings.Split(path, "operations/")[1], parsedQuery["embed"][0] } -//NewExecutionMonitor creates a new execution monitor +// NewExecutionMonitor creates a new execution monitor func NewExecutionMonitor(commandName, operationID, embed string, retries uint, reportedOperationMessages []*models.Message, mtaClient mtaclient.MtaClientOperations) *ExecutionMonitor { return &ExecutionMonitor{ mtaClient: mtaClient, @@ -104,7 +104,7 @@ func (m *ExecutionMonitor) Monitor() ExecutionStatus { intermediatePhase, flag := getIntermediatePhaseAndFlag(m.commandName) ui.Say("Process has entered %s phase. After testing your new deployment you can resume or abort the process.", intermediatePhase) m.reportAvaiableActions(m.operationID) - ui.Say("Hint: Use the '%s' option of the %s command to skip this phase.", flag, m.commandName) + ui.Say("Hint: Use the %q option of the %s command to skip this phase.", flag, m.commandName) return Success default: ui.Failed("Process is in illegal state %s.", terminal.EntityNameColor(string(operation.State))) diff --git a/commands/file_uploader.go b/commands/file_uploader.go index 33d009c..09a3f39 100644 --- a/commands/file_uploader.go +++ b/commands/file_uploader.go @@ -2,9 +2,12 @@ package commands import ( "fmt" + "gopkg.in/cheggaaa/pb.v1" + "io" "os" "path/filepath" "strings" + "sync/atomic" "code.cloudfoundry.org/cli/cf/terminal" @@ -17,14 +20,48 @@ import ( "github.com/cloudfoundry-incubator/multiapps-cli-plugin/clients/models" ) -//FileUploader uploads files in chunks for the specified namespace +// FileUploader uploads files in chunks for the specified namespace type FileUploader struct { mtaClient mtaclient.MtaClientOperations namespace string uploadChunkSizeInMB uint64 } -//NewFileUploader creates a new file uploader for the specified namespace +type progressBarReader struct { + pb *pb.ProgressBar + written atomic.Int64 + file io.ReadSeeker + fileName string +} + +func (r *progressBarReader) Name() string { + return r.fileName +} + +func (r *progressBarReader) Read(p []byte) (int, error) { + n, err := r.file.Read(p) + if n > 0 { + r.written.Add(int64(n)) + r.pb.Add(n) + } + return n, err +} + +func (r *progressBarReader) Seek(offset int64, whence int) (int64, error) { + newOffset, err := r.file.Seek(offset, whence) + if whence == io.SeekStart && r.written.Load() != 0 { + r.pb.Add64(-r.written.Load() - offset) + r.written.Store(offset) + } + return newOffset, err +} + +func (r *progressBarReader) Close() error { + //no-op as we close the file part manually + return nil +} + +// NewFileUploader creates a new file uploader for the specified namespace func NewFileUploader(mtaClient mtaclient.MtaClientOperations, namespace string, uploadChunkSizeInMB uint64) *FileUploader { return &FileUploader{ mtaClient: mtaClient, @@ -33,7 +70,7 @@ func NewFileUploader(mtaClient mtaclient.MtaClientOperations, namespace string, } } -//UploadFiles uploads the files +// UploadFiles uploads the files func (f *FileUploader) UploadFiles(files []string) ([]*models.FileMetadata, ExecutionStatus) { log.Tracef("Uploading files '%v'\n", files) @@ -45,7 +82,7 @@ func (f *FileUploader) UploadFiles(files []string) ([]*models.FileMetadata, Exec } // Determine which files to upload - var filesToUpload []os.File + var filesToUpload []*os.File var alreadyUploadedFiles []*models.FileMetadata for _, file := range files { // Check if the file exists @@ -66,8 +103,7 @@ func (f *FileUploader) UploadFiles(files []string) ([]*models.FileMetadata, Exec ui.Failed("Could not open file %s", terminal.EntityNameColor(file)) return nil, Failure } - defer fileToUpload.Close() - filesToUpload = append(filesToUpload, *fileToUpload) + filesToUpload = append(filesToUpload, fileToUpload) } } @@ -80,15 +116,11 @@ func (f *FileUploader) UploadFiles(files []string) ([]*models.FileMetadata, Exec // Iterate over all files to be uploaded for _, fileToUpload := range filesToUpload { // Print the full path of the file - fullPath, err := filepath.Abs(fileToUpload.Name()) - if err != nil { - ui.Failed("Could not get absolute path of file %s: %s", terminal.EntityNameColor(fileToUpload.Name()), err.Error()) - return nil, Failure - } - ui.Say(" " + fullPath) - + // as per the Go docs, the file name is the one passed to os.Open + // and we pass the absolute path + ui.Say(" " + fileToUpload.Name()) // Upload the file - uploaded, err := f.uploadInChunks(fullPath, fileToUpload) + uploaded, err := f.uploadInChunks(fileToUpload) if err != nil { ui.Failed("Could not upload file %s: %s", terminal.EntityNameColor(fileToUpload.Name()), err.Error()) return nil, Failure @@ -100,33 +132,35 @@ func (f *FileUploader) UploadFiles(files []string) ([]*models.FileMetadata, Exec return uploadedFiles, Success } -func (f *FileUploader) uploadInChunks(fullPath string, fileToUpload os.File) ([]*models.FileMetadata, error) { - // Upload the file - err := util.ValidateChunkSize(fullPath, f.uploadChunkSizeInMB) +func (f *FileUploader) uploadInChunks(fileToUpload *os.File) ([]*models.FileMetadata, error) { + err := util.ValidateChunkSize(fileToUpload.Name(), f.uploadChunkSizeInMB) if err != nil { - return nil, fmt.Errorf("Could not valide file %q: %v", fullPath, baseclient.NewClientError(err)) + return nil, fmt.Errorf("Could not valide file %q: %v", fileToUpload.Name(), err) } - fileToUploadParts, err := util.SplitFile(fullPath, f.uploadChunkSizeInMB) + fileToUploadParts, err := util.SplitFile(fileToUpload.Name(), f.uploadChunkSizeInMB) if err != nil { - return nil, fmt.Errorf("Could not process file %q: %v", fullPath, baseclient.NewClientError(err)) + return nil, fmt.Errorf("Could not process file %q: %v", fileToUpload.Name(), err) } defer attemptToRemoveFileParts(fileToUploadParts) uploadedFilesChannel := make(chan *models.FileMetadata) errorChannel := make(chan error) + fileInfo, err := fileToUpload.Stat() + if err != nil { + return nil, fmt.Errorf("could not get information on file %q: %v", fileToUpload.Name(), err) + } + + progressBar := pb.New64(fileInfo.Size()).SetUnits(pb.U_BYTES) + progressBar.ShowTimeLeft = false + progressBar.ShowElapsedTime = true + progressBar.Start() + defer progressBar.Finish() + for _, fileToUploadPart := range fileToUploadParts { - filePart, err := os.Open(fileToUploadPart) - if err != nil { - return nil, fmt.Errorf("Could not open file part %s of file %s", fileToUploadPart, fullPath) - } + filePartCopy := fileToUploadPart go func() { - fileInfo, err := filePart.Stat() - if err != nil { - errorChannel <- err - return - } - file, err := f.uploadFilePart(filePart, fileToUpload.Name(), fileInfo.Size()) + file, err := f.uploadFilePart(filePartCopy, fileToUpload.Name(), progressBar) if err != nil { errorChannel <- err return @@ -147,16 +181,16 @@ func (f *FileUploader) uploadInChunks(fullPath string, fileToUpload os.File) ([] return uploadedFileParts, nil } -func attemptToRemoveFileParts(fileParts []string) { +func attemptToRemoveFileParts(fileParts []*os.File) { // If more than one file parts exists, then remove them. // If there is only one, then this is the archive itself if len(fileParts) <= 1 { return } for _, filePart := range fileParts { - filePartAbsPath, err := filepath.Abs(filePart) + filePartAbsPath, err := filepath.Abs(filePart.Name()) if err != nil { - ui.Warn("Error retrieving absolute file path of %q", filePart) + ui.Warn("Error retrieving absolute file path of %q", filePart.Name()) } err = os.Remove(filePartAbsPath) if err != nil { @@ -165,11 +199,17 @@ func attemptToRemoveFileParts(fileParts []string) { } } -func (f *FileUploader) uploadFilePart(filePart *os.File, baseFileName string, fileSize int64) (*models.FileMetadata, error) { - uploadedFile, err := f.mtaClient.UploadMtaFile(*filePart, fileSize, &f.namespace) +func (f *FileUploader) uploadFilePart(filePart *os.File, fileName string, pb *pb.ProgressBar) (*models.FileMetadata, error) { defer filePart.Close() + fileInfo, err := filePart.Stat() + if err != nil { + return nil, fmt.Errorf("could not stat file part %q of file %q", filePart.Name(), fileName) + } + + file := &progressBarReader{file: filePart, fileName: fileInfo.Name(), pb: pb} + uploadedFile, err := f.mtaClient.UploadMtaFile(file, fileInfo.Size(), &f.namespace) if err != nil { - return nil, fmt.Errorf("Could not create file %s: %s", terminal.EntityNameColor(baseFileName), baseclient.NewClientError(err)) + return nil, fmt.Errorf("could not upload file %s: %s", terminal.EntityNameColor(fileName), err) } return uploadedFile, nil } diff --git a/commands/file_uploader_test.go b/commands/file_uploader_test.go index d610d82..3be5195 100644 --- a/commands/file_uploader_test.go +++ b/commands/file_uploader_test.go @@ -67,10 +67,10 @@ var _ = Describe("FileUploader", func() { Context("with non-existing service files and one file to upload", func() { It("should return the uploaded file", func() { - files := []*models.FileMetadata{testutil.GetFile(*testFile, testFileDigest, namespace)} + files := []*models.FileMetadata{testutil.GetFile(testFile, testFileDigest, namespace)} client := fakeMtaClientBuilder. GetMtaFiles([]*models.FileMetadata{}, nil). - UploadMtaFile(*testFile, testutil.GetFile(*testFile, testFileDigest, namespace), nil).Build() + UploadMtaFile(testFile, testutil.GetFile(testFile, testFileDigest, namespace), nil).Build() var uploadedFiles []*models.FileMetadata output := oc.CaptureOutput(func() { fileUploader = commands.NewFileUploader(client, namespace, properties.DefaultUploadChunkSizeInMB) @@ -89,10 +89,10 @@ var _ = Describe("FileUploader", func() { Context("with existing service files and one file to upload", func() { It("should display a message that the file upload will be skipped", func() { - files := []*models.FileMetadata{testutil.GetFile(*testFile, testFileDigest, "namespace")} + files := []*models.FileMetadata{testutil.GetFile(testFile, testFileDigest, "namespace")} client := fakeMtaClientBuilder. GetMtaFiles([]*models.FileMetadata{&testutil.SimpleFile}, nil). - UploadMtaFile(*testFile, testutil.GetFile(*testFile, testFileDigest, "namespace"), nil).Build() + UploadMtaFile(testFile, testutil.GetFile(testFile, testFileDigest, "namespace"), nil).Build() var uploadedFiles []*models.FileMetadata output := oc.CaptureOutput(func() { fileUploader = commands.NewFileUploader(client, namespace, properties.DefaultUploadChunkSizeInMB) @@ -107,10 +107,10 @@ var _ = Describe("FileUploader", func() { Context("with non-existing service files and one file to upload and service versions returned from the backend", func() { It("should return the uploaded file", func() { - fileMetadata := testutil.GetFile(*testFile, testFileDigest, namespace) + fileMetadata := testutil.GetFile(testFile, testFileDigest, namespace) client := fakeMtaClientBuilder. GetMtaFiles([]*models.FileMetadata{}, nil). - UploadMtaFile(*testFile, fileMetadata, nil).Build() + UploadMtaFile(testFile, fileMetadata, nil).Build() var uploadedFiles []*models.FileMetadata output := oc.CaptureOutput(func() { fileUploader = commands.NewFileUploader(client, namespace, properties.DefaultUploadChunkSizeInMB) @@ -132,7 +132,7 @@ var _ = Describe("FileUploader", func() { // files := []*models.File{testutil.GetFile("xs2-deploy", *testFile, testFileDigest)} client := fakeMtaClientBuilder. GetMtaFiles([]*models.FileMetadata{}, nil). - UploadMtaFile(*testFile, &models.FileMetadata{}, errors.New("Unexpected error from the backend")).Build() + UploadMtaFile(testFile, &models.FileMetadata{}, errors.New("Unexpected error from the backend")).Build() // var uploadedFiles []*models.FileMetadata output := oc.CaptureOutput(func() { fileUploader = commands.NewFileUploader(client, namespace, properties.DefaultUploadChunkSizeInMB) diff --git a/commands/flags_parser.go b/commands/flags_parser.go index 34fb105..84bb25f 100644 --- a/commands/flags_parser.go +++ b/commands/flags_parser.go @@ -77,11 +77,11 @@ func (p DefaultCommandFlagsParser) ParseFlags(flags *flag.FlagSet, args []string // Check for missing positional arguments positionalArgsCount := len(p.positionalArgNames) if len(args) < positionalArgsCount { - return fmt.Errorf(fmt.Sprintf("Missing positional argument '%s'", p.positionalArgNames[len(args)])) + return fmt.Errorf("Missing positional argument %q", p.positionalArgNames[len(args)]) } for i := 0; i < positionalArgsCount; i++ { if flags.Lookup(strings.Replace(args[i], "-", "", 1)) != nil { - return fmt.Errorf("Missing positional argument '%s'", p.positionalArgNames[i]) + return fmt.Errorf("Missing positional argument %q", p.positionalArgNames[i]) } } diff --git a/commands/test_token_factory.go b/commands/test_token_factory.go index 7ab7138..ea254fc 100644 --- a/commands/test_token_factory.go +++ b/commands/test_token_factory.go @@ -17,6 +17,10 @@ func NewTestTokenFactory(fakeCliConnection *fakes.FakeCliConnection) *TestTokenF } func (f *TestTokenFactory) NewToken() (runtime.ClientAuthInfoWriter, error) { - tokenString, _ := f.FakeCliConnection.AccessToken() + tokenString, _ := f.NewRawToken() return testutil.NewCustomBearerToken(tokenString), nil } + +func (f *TestTokenFactory) NewRawToken() (string, error) { + return f.FakeCliConnection.AccessToken() +} diff --git a/commands/undeploy_command_test.go b/commands/undeploy_command_test.go index 8ae8ab5..8c5f16b 100644 --- a/commands/undeploy_command_test.go +++ b/commands/undeploy_command_test.go @@ -31,7 +31,7 @@ var _ = Describe("UndeployCommand", func() { var name string var cliConnection *pluginFakes.FakeCliConnection - var mtaClient mtaFake.FakeMtaClientOperations + var mtaClient *mtaFake.FakeMtaClientOperations var testClientFactory *commands.TestClientFactory var command *commands.UndeployCommand var oc = testutil.NewUIOutputCapturer() @@ -63,7 +63,7 @@ var _ = Describe("UndeployCommand", func() { "Undeploying multi-target app "+mtaID+" in org "+org+" / space "+space+" as "+user+"...") if abortedProcessId != "" { lines = append(lines, - "Executing action 'abort' on operation "+abortedProcessId+"...", + "Executing action \"abort\" on operation "+abortedProcessId+"...", "OK") } lines = append(lines, @@ -123,7 +123,7 @@ var _ = Describe("UndeployCommand", func() { output, status := oc.CaptureOutputAndStatus(func() int { return command.Execute([]string{}).ToInt() }) - ex.ExpectFailure(status, output, "Incorrect usage. Missing positional argument 'MTA_ID'") + ex.ExpectFailure(status, output, "Incorrect usage. Missing positional argument \"MTA_ID\"") Expect(cliConnection.CliCommandArgsForCall(0)).To(Equal([]string{"help", name})) }) }) @@ -134,7 +134,7 @@ var _ = Describe("UndeployCommand", func() { output, status := oc.CaptureOutputAndStatus(func() int { return command.Execute([]string{"-f"}).ToInt() }) - ex.ExpectFailure(status, output, "Incorrect usage. Missing positional argument 'MTA_ID'") + ex.ExpectFailure(status, output, "Incorrect usage. Missing positional argument \"MTA_ID\"") Expect(cliConnection.CliCommandArgsForCall(0)).To(Equal([]string{"help", name})) }) }) @@ -180,7 +180,7 @@ var _ = Describe("UndeployCommand", func() { output, status := oc.CaptureOutputAndStatus(func() int { return command.Execute([]string{mtaID, "-f"}).ToInt() }) - ex.ExpectFailureOnLine(status, output, "Could not execute action 'abort' on operation 999: test-error", 3) + ex.ExpectFailureOnLine(status, output, "Could not execute action \"abort\" on operation 999: test-error", 3) }) It("should try to abort the conflicting process and success", func() { diff --git a/go.mod b/go.mod index 4a7cf8c..18d35de 100644 --- a/go.mod +++ b/go.mod @@ -11,11 +11,11 @@ require ( github.com/go-openapi/strfmt v0.20.2 github.com/go-openapi/swag v0.19.15 github.com/go-openapi/validate v0.20.2 - github.com/jinzhu/copier v0.3.2 github.com/onsi/ginkgo v1.16.4 github.com/onsi/gomega v1.13.0 github.com/pborman/uuid v1.2.0 golang.org/x/net v0.0.0-20210929193557-e81a3d93ecf6 + gopkg.in/cheggaaa/pb.v1 v1.0.28 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) @@ -77,7 +77,6 @@ require ( google.golang.org/genproto v0.0.0-20211102202547-e9cf271f7f2c // indirect google.golang.org/grpc v1.42.0 // indirect google.golang.org/protobuf v1.27.1 // indirect - gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/log/log.go b/log/log.go index a86d9f1..e1469ca 100644 --- a/log/log.go +++ b/log/log.go @@ -5,7 +5,7 @@ import ( "os" ) -var Debug = (os.Getenv("DEBUG") == "1") +var Debug = os.Getenv("DEBUG") == "1" type Exiter interface { Exit(status int) diff --git a/multiapps_plugin.go b/multiapps_plugin.go index 0f3a536..2861c23 100644 --- a/multiapps_plugin.go +++ b/multiapps_plugin.go @@ -77,7 +77,7 @@ func findCommand(name string) (commands.Command, error) { return command, nil } } - return nil, fmt.Errorf("Could not find command with name '%s'", name) + return nil, fmt.Errorf("Could not find command with name %q", name) } func parseSemver(version string) plugin.VersionType { diff --git a/testutil/test_results.go b/testutil/test_results.go index ef695e7..fd05f61 100644 --- a/testutil/test_results.go +++ b/testutil/test_results.go @@ -44,7 +44,6 @@ const ProcessID = "1000" const LogID = "OPERATION.log" const LogContent = "test-test-test" -// type RuntimeResponse struct { code int message string @@ -80,21 +79,18 @@ func NewCustomError(customCode int, opName, customMessage string) *runtime.APIEr return runtime.NewAPIError(opName, customResponse, customCode) } -// var notFoundResponse = RuntimeResponse{ code: 404, message: "Process with id 404 not found", } -// var ClientError = &runtime.APIError{ OperationName: "Getting process", Response: notFoundResponse, Code: 404, } -// -//D41D8CD98F00B204E9800998ECF8427E -> MD5 hash for empty file +// D41D8CD98F00B204E9800998ECF8427E -> MD5 hash for empty file var SimpleFile = models.FileMetadata{ ID: "test.mtar", Digest: "D41D8CD98F00B204E9800998ECF8427E", @@ -104,9 +100,8 @@ var SimpleFile = models.FileMetadata{ Namespace: "namespace", } -// -func GetFile(file os.File, digest string, namespace string) *models.FileMetadata { - stat, _ := os.Stat(file.Name()) +func GetFile(file *os.File, digest string, namespace string) *models.FileMetadata { + stat, _ := file.Stat() return &models.FileMetadata{ ID: stat.Name(), Space: "test-space", @@ -117,7 +112,6 @@ func GetFile(file os.File, digest string, namespace string) *models.FileMetadata } } - func GetOperation(processID, spaceID string, mtaID string, namespace string, processType string, state string, acquiredLock bool) *models.Operation { return &models.Operation{ ProcessID: processID, @@ -132,7 +126,6 @@ func GetOperation(processID, spaceID string, mtaID string, namespace string, pro } } -// func GetMta(id, version string, namespace string, modules []*models.Module, services []string) *models.Mta { return &models.Mta{ Metadata: &models.Metadata{ @@ -145,7 +138,6 @@ func GetMta(id, version string, namespace string, modules []*models.Module, serv } } -// func GetMtaModule(name string, services []string, providedDependencies []string) *models.Module { return &models.Module{ ModuleName: name, @@ -154,4 +146,3 @@ func GetMtaModule(name string, services []string, providedDependencies []string) ProvidedDendencyNames: providedDependencies, } } - diff --git a/util/cf_context.go b/util/cf_context.go index c6ad4ba..b655391 100644 --- a/util/cf_context.go +++ b/util/cf_context.go @@ -22,7 +22,7 @@ func (c *CloudFoundryContext) GetOrg() (plugin_models.Organization, error) { return plugin_models.Organization{}, fmt.Errorf("Could not get current org: %s", err) } if org.Name == "" { - return plugin_models.Organization{}, fmt.Errorf("No org and space targeted, use '%s' to target an org and a space", terminal.CommandColor("cf target -o ORG -s SPACE")) + return plugin_models.Organization{}, fmt.Errorf("No org and space targeted, use %q to target an org and a space", terminal.CommandColor("cf target -o ORG -s SPACE")) } return org, nil } @@ -35,7 +35,7 @@ func (c *CloudFoundryContext) GetSpace() (plugin_models.Space, error) { } if space.Name == "" || space.Guid == "" { - return plugin_models.Space{}, fmt.Errorf("No space targeted, use '%s' to target a space", terminal.CommandColor("cf target -s SPACE")) + return plugin_models.Space{}, fmt.Errorf("No space targeted, use %q to target a space", terminal.CommandColor("cf target -s SPACE")) } return space, nil } @@ -47,7 +47,7 @@ func (c *CloudFoundryContext) GetUsername() (string, error) { return "", fmt.Errorf("Could not get username: %s", err) } if username == "" { - return "", fmt.Errorf("Not logged in. Use '%s' to log in.", terminal.CommandColor("cf login")) + return "", fmt.Errorf("Not logged in. Use %q to log in.", terminal.CommandColor("cf login")) } return username, nil } diff --git a/util/cf_context_test.go b/util/cf_context_test.go index a1db37a..8492732 100644 --- a/util/cf_context_test.go +++ b/util/cf_context_test.go @@ -45,7 +45,7 @@ var _ = Describe("CloudFoundryContext", func() { CurrentOrg("", "", nil).Build() cfContext := NewCloudFoundryContext(fakeCliConnection) _, err := cfContext.GetOrg() - Expect(err).To(MatchError(fmt.Errorf("No org and space targeted, use '%s' to target an org and a space", terminal.CommandColor("cf target -o ORG -s SPACE")))) + Expect(err).To(MatchError(fmt.Errorf("No org and space targeted, use %q to target an org and a space", terminal.CommandColor("cf target -o ORG -s SPACE")))) }) }) }) @@ -65,7 +65,7 @@ var _ = Describe("CloudFoundryContext", func() { CurrentSpace("", "", nil).Build() cfContext := NewCloudFoundryContext(fakeCliConnection) _, err := cfContext.GetSpace() - Expect(err).To(MatchError(fmt.Errorf("No space targeted, use '%s' to target a space", terminal.CommandColor("cf target -s SPACE")))) + Expect(err).To(MatchError(fmt.Errorf("No space targeted, use %q to target a space", terminal.CommandColor("cf target -s SPACE")))) }) }) }) @@ -85,7 +85,7 @@ var _ = Describe("CloudFoundryContext", func() { Username("", nil).Build() cfContext := NewCloudFoundryContext(fakeCliConnection) _, err := cfContext.GetUsername() - Expect(err).To(MatchError(fmt.Errorf("Not logged in. Use '%s' to log in.", terminal.CommandColor("cf login")))) + Expect(err).To(MatchError(fmt.Errorf("Not logged in. Use %q to log in.", terminal.CommandColor("cf login")))) }) }) }) diff --git a/util/file_splitter.go b/util/file_splitter.go index d787645..1d0d344 100644 --- a/util/file_splitter.go +++ b/util/file_splitter.go @@ -19,19 +19,21 @@ func generateHash() string { } // SplitFile ... -func SplitFile(filePath string, fileChunkSizeInMB uint64) ([]string, error) { +func SplitFile(filePath string, fileChunkSizeInMB uint64) ([]*os.File, error) { if fileChunkSizeInMB == 0 { - return []string{filePath}, nil + return openSingleFile(filePath) } file, err := os.Open(filePath) if err != nil { return nil, err } - defer file.Close() - fileInfo, _ := file.Stat() + fileInfo, err := file.Stat() + if err != nil { + return nil, err + } var fileSize = uint64(fileInfo.Size()) var fileChunkSize = toBytes(fileChunkSizeInMB) @@ -39,7 +41,7 @@ func SplitFile(filePath string, fileChunkSizeInMB uint64) ([]string, error) { // calculate total number of parts the file will be chunked into totalPartsNum := uint64(math.Ceil(float64(fileSize) / float64(fileChunkSize))) if totalPartsNum <= 1 { - return []string{filePath}, nil + return openSingleFile(filePath) } partsTempDir := filepath.Join(os.TempDir(), generateHash()) @@ -49,7 +51,7 @@ func SplitFile(filePath string, fileChunkSizeInMB uint64) ([]string, error) { } baseFileName := filepath.Base(filePath) - var fileParts []string + var fileParts []*os.File for i := uint64(0); i < totalPartsNum; i++ { filePartName := baseFileName + ".part." + strconv.FormatUint(i, 10) @@ -58,19 +60,25 @@ func SplitFile(filePath string, fileChunkSizeInMB uint64) ([]string, error) { if err != nil { return nil, err } - defer filePart.Close() partSize := int64(minUint64(fileChunkSize, fileSize-i*fileChunkSize)) _, err = io.CopyN(filePart, file, partSize) if err != nil { + filePart.Close() return nil, err } - fileParts = append(fileParts, filePart.Name()) + filePart.Seek(0, io.SeekStart) + fileParts = append(fileParts, filePart) } return fileParts, nil } +func openSingleFile(path string) ([]*os.File, error) { + f, err := os.Open(path) + return []*os.File{f}, err +} + // ValidateChunkSize validate the chunk size func ValidateChunkSize(filePath string, fileChunkSizeInMB uint64) error { if fileChunkSizeInMB == 0 { diff --git a/util/named_read_seeker.go b/util/named_read_seeker.go new file mode 100644 index 0000000..eb91061 --- /dev/null +++ b/util/named_read_seeker.go @@ -0,0 +1,8 @@ +package util + +import "io" + +type NamedReadSeeker interface { + io.ReadSeeker + Name() string +}