Make host orchestrator client standalone

so that it can be used outside the context of the cloud orchestrator

The follwing changes were implemented:
- Be more specific on what requests require retries
- Make the credentials header configurable
- Make file upload configuration optional
- Reduce the timeout for retries on some tests

pkg/cli/cli.go

@@ -1093,14 +1093,13 @@ func buildServiceBuilder(builder client.ServiceBuilder) serviceBuilder {
 			dumpOut = c.ErrOrStderr()
 		}
 		opts := &client.ServiceOptions{
-			RootEndpoint:           buildServiceRootEndpoint(flags.ServiceURL, flags.Zone),
-			ProxyURL:               proxyURL,
-			DumpOut:                dumpOut,
-			ErrOut:                 c.ErrOrStderr(),
-			RetryAttempts:          3,
-			RetryDelay:             5 * time.Second,
-			ChunkSizeBytes:         chunkSizeBytes,
-			ChunkUploadBackOffOpts: client.DefaultChunkUploadBackOffOpts(),
+			RootEndpoint:   buildServiceRootEndpoint(flags.ServiceURL, flags.Zone),
+			ProxyURL:       proxyURL,
+			DumpOut:        dumpOut,
+			ErrOut:         c.ErrOrStderr(),
+			RetryAttempts:  3,
+			RetryDelay:     5 * time.Second,
+			ChunkSizeBytes: chunkSizeBytes,
 		}
 		return builder(opts)
 	}

pkg/cli/cli_test.go

@@ -126,11 +126,11 @@ func (fakeHostService) ConnectWebRTC(device string, observer wclient.Observer, l
 	return nil, nil
 }
 
-func (fakeHostService) FetchArtifacts(req *hoapi.FetchArtifactsRequest) (*hoapi.FetchArtifactsResponse, error) {
+func (fakeHostService) FetchArtifacts(req *hoapi.FetchArtifactsRequest, creds string) (*hoapi.FetchArtifactsResponse, error) {
 	return &hoapi.FetchArtifactsResponse{AndroidCIBundle: &hoapi.AndroidCIBundle{}}, nil
 }
 
-func (fakeHostService) CreateCVD(req *hoapi.CreateCVDRequest) (*hoapi.CreateCVDResponse, error) {
+func (fakeHostService) CreateCVD(req *hoapi.CreateCVDRequest, creds string) (*hoapi.CreateCVDResponse, error) {
 	return &hoapi.CreateCVDResponse{CVDs: []*hoapi.CVD{{Name: "cvd-1"}}}, nil
 }
 
@@ -146,6 +146,10 @@ func (fakeHostService) UploadFiles(uploadDir string, filenames []string) error {
 	return nil
 }
 
+func (fakeHostService) UploadFilesWithOptions(uploadDir string, filenames []string, options client.UploadOptions) error {
+	return nil
+}
+
 func (fakeHostService) DownloadRuntimeArtifacts(dst io.Writer) error {
 	return nil
 }

pkg/cli/cvd.go

@@ -148,7 +148,7 @@ func (c *cvdCreator) createCVDFromLocalBuild() ([]*hoapi.CVD, error) {
 		},
 		AdditionalInstancesNum: c.opts.AdditionalInstancesNum(),
 	}
-	res, err := c.service.HostService(c.opts.Host).CreateCVD(&req)
+	res, err := c.service.HostService(c.opts.Host).CreateCVD(&req, client.InjectedCredentials)
 	if err != nil {
 		return nil, err
 	}
@@ -173,7 +173,7 @@ func (c *cvdCreator) createWithCanonicalConfig() ([]*hoapi.CVD, error) {
 		EnvConfig: c.opts.EnvConfig,
 	}
 	c.statePrinter.Print(stateMsgFetchAndStart)
-	res, err := c.service.HostService(c.opts.Host).CreateCVD(createReq)
+	res, err := c.service.HostService(c.opts.Host).CreateCVD(createReq, client.InjectedCredentials)
 	c.statePrinter.PrintDone(stateMsgFetchAndStart, err)
 	if err != nil {
 		return nil, err
@@ -197,7 +197,7 @@ func (c *cvdCreator) createWithOpts() ([]*hoapi.CVD, error) {
 		AndroidCIBundle: &hoapi.AndroidCIBundle{Build: mainBuild, Type: hoapi.MainBundleType},
 	}
 	c.statePrinter.Print(stateMsgFetchMainBundle)
-	fetchMainBuildRes, err := c.service.HostService(c.opts.Host).FetchArtifacts(fetchReq)
+	fetchMainBuildRes, err := c.service.HostService(c.opts.Host).FetchArtifacts(fetchReq, client.InjectedCredentials)
 	c.statePrinter.PrintDone(stateMsgFetchMainBundle, err)
 	if err != nil {
 		return nil, err
@@ -216,7 +216,7 @@ func (c *cvdCreator) createWithOpts() ([]*hoapi.CVD, error) {
 		AdditionalInstancesNum: c.opts.AdditionalInstancesNum(),
 	}
 	c.statePrinter.Print(stateMsgStartCVD)
-	res, err := c.service.HostService(c.opts.Host).CreateCVD(createReq)
+	res, err := c.service.HostService(c.opts.Host).CreateCVD(createReq, client.InjectedCredentials)
 	c.statePrinter.PrintDone(stateMsgStartCVD, err)
 	if err != nil {
 		return nil, err

pkg/client/client.go

@@ -28,11 +28,11 @@ import (
 	"github.com/hashicorp/go-multierror"
 )
 
-type OpTimeoutError string
-
-func (s OpTimeoutError) Error() string {
-	return fmt.Sprintf("waiting for operation %q timed out", string(s))
-}
+const (
+	// Value to pass as credentials to the Host Orchestrator service endpoints. Any non-empty value is enough.
+	InjectedCredentials             = "inject"
+	headerNameCOInjectBuildAPICreds = "X-Cutf-Cloud-Orchestrator-Inject-BuildAPI-Creds"
+)
 
 type ApiCallError struct {
 	Code     int    `json:"code,omitempty"`
@@ -54,14 +54,13 @@ func (e *ApiCallError) Is(target error) bool {
 }
 
 type ServiceOptions struct {
-	RootEndpoint           string
-	ProxyURL               string
-	DumpOut                io.Writer
-	ErrOut                 io.Writer
-	RetryAttempts          int
-	RetryDelay             time.Duration
-	ChunkSizeBytes         int64
-	ChunkUploadBackOffOpts BackOffOpts
+	RootEndpoint   string
+	ProxyURL       string
+	DumpOut        io.Writer
+	ErrOut         io.Writer
+	RetryAttempts  int
+	RetryDelay     time.Duration
+	ChunkSizeBytes int64
 }
 
 type Service interface {
@@ -98,8 +97,6 @@ func NewService(opts *ServiceOptions) (Service, error) {
 		httpHelper: HTTPHelper{
 			Client:       httpClient,
 			RootEndpoint: opts.RootEndpoint,
-			Retries:      uint(opts.RetryAttempts),
-			RetryDelay:   opts.RetryDelay,
 			Dumpster:     opts.DumpOut,
 		},
 	}, nil
@@ -114,6 +111,20 @@ func (c *serviceImpl) CreateHost(req *apiv1.CreateHostRequest) (*apiv1.HostInsta
 	if err := c.waitForOperation(&op, ins); err != nil {
 		return nil, err
 	}
+
+	// There is a short delay between the creation of the host and the availability of the host
+	// orchestrator. This call ensures the host orchestrator had time to start before returning
+	// from the this function.
+	retryOpts := RetryOptions{
+		StatusCodes: []int{http.StatusBadGateway},
+		NumRetries:  3,
+		RetryDelay:  5 * time.Second,
+	}
+	hostPath := fmt.Sprintf("/hosts/%s/", ins.Name)
+	if err := c.httpHelper.NewGetRequest(hostPath).DoWithRetries(nil, retryOpts); err != nil {
+		return nil, fmt.Errorf("Unable to communicate with host orchestrator: %w", err)
+	}
+
 	return ins, nil
 }
 
@@ -146,22 +157,26 @@ func (c *serviceImpl) DeleteHosts(names []string) error {
 
 func (c *serviceImpl) waitForOperation(op *apiv1.Operation, res any) error {
 	path := "/operations/" + op.Name + "/:wait"
-	return c.httpHelper.NewPostRequest(path, nil).Do(res)
+	retryOpts := RetryOptions{
+		[]int{http.StatusServiceUnavailable},
+		uint(c.ServiceOptions.RetryAttempts),
+		c.RetryDelay,
+	}
+	return c.httpHelper.NewPostRequest(path, nil).DoWithRetries(res, retryOpts)
 }
 
-const headerNameCOInjectBuildAPICreds = "X-Cutf-Cloud-Orchestrator-Inject-BuildAPI-Creds"
-
 func (s *serviceImpl) RootURI() string {
 	return s.RootEndpoint
 }
 
 func (s *serviceImpl) HostService(host string) HostOrchestratorService {
-	hs := &hostOrchestratorServiceImpl{
-		httpHelper:             s.httpHelper,
-		ChunkSizeBytes:         s.ChunkSizeBytes,
-		ChunkUploadBackOffOpts: s.ChunkUploadBackOffOpts,
+	hs := &HostOrchestratorServiceImpl{
+		HTTPHelper:                s.httpHelper,
+		WaitRetries:               uint(s.ServiceOptions.RetryAttempts),
+		WaitRetryDelay:            s.ServiceOptions.RetryDelay,
+		BuildAPICredentialsHeader: headerNameCOInjectBuildAPICreds,
 	}
-	hs.httpHelper.RootEndpoint = s.httpHelper.RootEndpoint + "/hosts/" + host
+	hs.HTTPHelper.RootEndpoint = s.httpHelper.RootEndpoint + "/hosts/" + host
 	return hs
 }
 

pkg/client/client_test.go

@@ -44,6 +44,8 @@ func TestRetryLogic(t *testing.T) {
 				return
 			}
 			writeOK(w, &apiv1.HostInstance{Name: "foo"})
+		case "GET /hosts/foo/":
+			writeOK(w, make(map[string]any))
 		default:
 			t.Fatal("unexpected endpoint: " + ep)
 		}