diff --git a/api/util/apps.go b/api/util/apps.go index 32a84a9..53d7947 100644 --- a/api/util/apps.go +++ b/api/util/apps.go @@ -1,11 +1,15 @@ package util import ( + "context" + "encoding/pem" "fmt" + "os" "sort" "strings" apps "github.com/ninech/apis/apps/v1alpha1" + "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -128,6 +132,14 @@ type GitAuth struct { SSHPrivateKey *string } +func (git GitAuth) HasPrivateKey() bool { + return git.SSHPrivateKey != nil +} + +func (git GitAuth) HasBasicAuth() bool { + return git.Username != nil && git.Password != nil +} + func (git GitAuth) Secret(app *apps.Application) *corev1.Secret { data := map[string][]byte{} @@ -171,8 +183,8 @@ func (git GitAuth) UpdateSecret(secret *corev1.Secret) { } func (git GitAuth) Enabled() bool { - if git.Username != nil || - git.Password != nil || + if (git.Username != nil && + git.Password != nil) || git.SSHPrivateKey != nil { return true } @@ -229,3 +241,51 @@ func GatherDNSDetails(items []apps.Application) []DNSDetail { } return result } + +// ValidatePEM validates if the passed content is in valid PEM format, errors +// out if the content is empty +func ValidatePEM(content string) (*string, error) { + if content == "" { + return nil, fmt.Errorf("the SSH private key cannot be empty") + } + + content = strings.TrimSpace(content) + b, rest := pem.Decode([]byte(content)) + if b == nil || len(rest) > 0 { + return nil, fmt.Errorf("no valid PEM formatted data found") + } + return &content, nil +} + +// TestRepositoryAccess tests if the given git repository can be accessed. +func TestRepositoryAccess(ctx context.Context, client *GitInformationClient, git *apps.ApplicationGitConfig, auth GitAuth) error { + repoInfo, err := client.RepositoryInformation(ctx, git.URL, auth) + if err != nil { + // we are not returning a detailed error here as it might be + // too technical. The full error can still be seen by using + // a different RetryLog function in the client. + return errors.New( + "communication issue with git information service " + + "(use --skip-repo-access-check to skip this check)", + ) + } + if len(repoInfo.Warnings) > 0 { + fmt.Fprintf(os.Stderr, "warning: %s\n", strings.Join(repoInfo.Warnings, ".")) + } + if repoInfo.Error != "" { + return errors.New(repoInfo.Error) + } + if !(containsBranch(git.Revision, repoInfo) || + containsTag(git.Revision, repoInfo)) { + return fmt.Errorf( + "can not find specified git revision (%q) in repository", + git.Revision, + ) + } + // it is possible to set a git URL without a proper scheme. In that + // case, HTTPS is used as a default. If the access check succeeds we + // need to overwrite the URL in the application as it will otherwise be + // denied by the webhook. + git.URL = repoInfo.RepositoryInfo.URL + return nil +} diff --git a/api/util/git_information_client.go b/api/util/git_information_client.go new file mode 100644 index 0000000..5139064 --- /dev/null +++ b/api/util/git_information_client.go @@ -0,0 +1,206 @@ +package util + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "time" + + apps "github.com/ninech/apis/apps/v1alpha1" + "github.com/pkg/errors" +) + +type GitInformationClient struct { + token string + url *url.URL + explorePath string + client *http.Client + logRetryFunc func(retry int, maxRetries int, err error) + retryBackoffs []time.Duration +} + +// NewGitInformationClient returns a client which can be used to retrieve +// metadata information about a given git repository +func NewGitInformationClient(address string, token string) (*GitInformationClient, error) { + u, err := url.Parse(address) + if err != nil { + return nil, fmt.Errorf("can not parse git information service URL: %w", err) + } + return defaultGitInformationClient(setURLDefaults(u), token), nil +} + +func setURLDefaults(u *url.URL) *url.URL { + newURL := u.JoinPath("explore") + if u.Scheme == "" { + newURL.Scheme = "https" + } + return newURL +} + +func defaultGitInformationClient(url *url.URL, token string) *GitInformationClient { + return &GitInformationClient{ + token: token, + url: url, + client: http.DefaultClient, + retryBackoffs: []time.Duration{time.Second, 2 * time.Second}, + } +} + +func (g *GitInformationClient) logError(format string, v ...any) { + fmt.Fprintf(os.Stderr, format, v...) +} + +func (g *GitInformationClient) logRetry(retry int, maxRetry int, err error) { + if g.logRetryFunc != nil { + g.logRetryFunc(retry, maxRetry, err) + return + } + if retry < maxRetry { + g.logError("Retrying because of error: %v\n", err) + } +} + +// SetLogRetryFunc allows to set the function which logs retries when doing +// requests to the git information service +func (g *GitInformationClient) SetLogRetryFunc(f func(retry int, maxRetries int, err error)) { + g.logRetryFunc = f +} + +// SetRetryBackoffs sets the durations which the client will wait between retries +func (g *GitInformationClient) SetRetryBackoffs(backoffs []time.Duration) { + g.retryBackoffs = backoffs +} + +func (g *GitInformationClient) repositoryInformation(ctx context.Context, url string, auth GitAuth) (*apps.GitExploreResponse, error) { + req := apps.GitExploreRequest{ + Repository: url, + } + if auth.Enabled() { + req.Auth = &apps.Auth{} + if auth.HasBasicAuth() { + req.Auth.BasicAuth = &apps.BasicAuth{ + Username: *auth.Username, + Password: *auth.Password, + } + } + if auth.HasPrivateKey() { + req.Auth.PrivateKey = []byte(*auth.SSHPrivateKey) + } + } + return g.sendRequest(ctx, req) +} + +// RepositoryInformation returns information about a given git repository. It retries on client connection issues. +func (g *GitInformationClient) RepositoryInformation(ctx context.Context, url string, auth GitAuth) (*apps.GitExploreResponse, error) { + var repoInfo *apps.GitExploreResponse + var err error + repoInfo, err = g.repositoryInformation(ctx, url, auth) + if err == nil { + return repoInfo, err + } + g.logRetry(0, len(g.retryBackoffs), err) + for retry, waitTime := range g.retryBackoffs { + time.Sleep(waitTime) + repoInfo, err = g.repositoryInformation(ctx, url, auth) + if err == nil { + break + } + g.logRetry(retry+1, len(g.retryBackoffs), err) + } + return repoInfo, err +} + +func (g *GitInformationClient) sendRequest(ctx context.Context, req apps.GitExploreRequest) (*apps.GitExploreResponse, error) { + data, err := json.Marshal(req) + if err != nil { + return nil, fmt.Errorf("can not JSON marshal request: %w", err) + } + + httpReq, err := http.NewRequest("POST", g.url.String(), bytes.NewReader(data)) + if err != nil { + return nil, fmt.Errorf("can not create HTTP request: %w", err) + } + if g.token != "" { + httpReq.Header.Set("Authorization", fmt.Sprintf("Bearer %s", g.token)) + } + httpReq = httpReq.WithContext(ctx) + + resp, err := g.client.Do(httpReq) + if err != nil { + return nil, err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("can not read response body: %w", err) + } + + exploreResponse := &apps.GitExploreResponse{} + if err := unmarshalResponse(bytes.NewReader(body), exploreResponse); err != nil { + return nil, fmt.Errorf( + "can not unmarshal response %q with status code %d: %w", + string(body), + resp.StatusCode, + err, + ) + } + return exploreResponse, nil +} + +func unmarshalResponse(data io.Reader, response *apps.GitExploreResponse) error { + decoder := json.NewDecoder(data) + err := decoder.Decode(response) + if err != nil { + return err + } + _, err = decoder.Token() + if err == nil || !errors.Is(err, io.EOF) { + return errors.New("invalid data after top-level JSON value") + } + return nil +} + +// containsTag returns true of the given tag exists in the git explore response +func containsTag(tag string, response *apps.GitExploreResponse) bool { + if response.RepositoryInfo == nil { + return false + } + for _, item := range response.RepositoryInfo.Tags { + if item == tag { + return true + } + } + return false +} + +// containsBranch returns true of the given branch exists in the git explore response +func containsBranch(branch string, response *apps.GitExploreResponse) bool { + if response.RepositoryInfo == nil { + return false + } + for _, item := range response.RepositoryInfo.Branches { + if item == branch { + return true + } + } + return false +} + +// RetryLogFunc returns a retry log function depending on the given showErrors +// parameter. If it is set to true, exact errors are shown when retrying to +// connect to the git information service. Otherwise they are not shown. +func RetryLogFunc(showErrors bool) func(retry int, maxRetries int, err error) { + return func(retry int, maxRetries int, err error) { + if showErrors { + fmt.Fprintf(os.Stderr, "got error: %v\n", err) + } + if retry < maxRetries { + fmt.Fprintln(os.Stderr, "Retrying...") + } + } +} diff --git a/api/util/git_information_client_test.go b/api/util/git_information_client_test.go new file mode 100644 index 0000000..d6cb2c0 --- /dev/null +++ b/api/util/git_information_client_test.go @@ -0,0 +1,125 @@ +package util_test + +import ( + "context" + "net/http" + "testing" + "time" + + apps "github.com/ninech/apis/apps/v1alpha1" + "github.com/ninech/nctl/api/util" + "github.com/ninech/nctl/internal/test" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/utils/pointer" +) + +func TestRepositoryInformation(t *testing.T) { + ctx := context.Background() + gitInfo := test.NewGitInformationService() + gitInfo.Start() + defer gitInfo.Close() + + dummyPrivateKey, err := test.GenerateRSAPrivateKey() + require.NoError(t, err) + + for name, testCase := range map[string]struct { + url string + token string + auth util.GitAuth + verifyRequest func(t *testing.T) func(p test.GitInfoServiceParsed, err error) + setResponse *test.GitInformationServiceResponse + expectedResponse *apps.GitExploreResponse + expectedRetries int + backoffs []time.Duration + errorExpected bool + }{ + "validate request": { + url: "https://github.com/ninech/deploio-examples", + token: "fake", + auth: util.GitAuth{ + Username: pointer.String("fake"), + Password: pointer.String("fakePass"), + SSHPrivateKey: &dummyPrivateKey, + }, + setResponse: &test.GitInformationServiceResponse{ + Code: http.StatusOK, + Content: apps.GitExploreResponse{ + RepositoryInfo: &apps.RepositoryInfo{ + URL: "https://github.com/ninech/deploio-examples", + Branches: []string{"main"}, + Tags: []string{"v1.0"}, + }, + }, + }, + expectedResponse: &apps.GitExploreResponse{ + RepositoryInfo: &apps.RepositoryInfo{ + URL: "https://github.com/ninech/deploio-examples", + Branches: []string{"main"}, + Tags: []string{"v1.0"}, + }, + }, + verifyRequest: func(t *testing.T) func(p test.GitInfoServiceParsed, err error) { + return func(p test.GitInfoServiceParsed, err error) { + is := assert.New(t) + is.NoError(err) + is.Equal("https://github.com/ninech/deploio-examples", p.Request.Repository) + is.Equal("fake", p.Token) + is.Equal("POST", p.Method) + is.NotNil(p.Request.Auth) + is.NotNil(p.Request.Auth.BasicAuth) + is.Equal("fake", p.Request.Auth.BasicAuth.Username) + is.Equal("fakePass", p.Request.Auth.BasicAuth.Password) + is.Equal(dummyPrivateKey, string(p.Request.Auth.PrivateKey)) + } + }, + }, + "we retry on server errors": { + url: "https://github.com/ninech/deploio-examples", + token: "fake", + expectedRetries: 2, + backoffs: []time.Duration{100 * time.Millisecond, 150 * time.Millisecond}, + setResponse: &test.GitInformationServiceResponse{ + Code: http.StatusBadGateway, + Raw: pointer.String("currently unavailable"), + }, + errorExpected: true, + }, + } { + testCase := testCase + t.Run(name, func(t *testing.T) { + if testCase.setResponse != nil { + gitInfo.SetResponse(*testCase.setResponse) + } + + c, err := util.NewGitInformationClient(gitInfo.URL(), testCase.token) + require.NoError(t, err) + + // we count the retries of the request + retries := 0 + c.SetLogRetryFunc(func(retry int, maxRetries int, err error) { + retries = retry + }) + // we lower the default backoff to have faster tests + if len(testCase.backoffs) > 0 { + c.SetRetryBackoffs(testCase.backoffs) + } + + response, err := c.RepositoryInformation(ctx, testCase.url, testCase.auth) + if testCase.errorExpected { + require.Error(t, err) + } else { + require.NoError(t, err) + } + + require.Equal(t, testCase.expectedRetries, retries) + if testCase.expectedResponse != nil { + require.Equal(t, *testCase.expectedResponse, *response) + } + if testCase.verifyRequest != nil { + data, err := gitInfo.Request() + testCase.verifyRequest(t)(data, err) + } + }) + } +} diff --git a/create/application.go b/create/application.go index e439fed..bf5dbcf 100644 --- a/create/application.go +++ b/create/application.go @@ -2,7 +2,6 @@ package create import ( "context" - "encoding/pem" "errors" "fmt" "os" @@ -32,18 +31,21 @@ import ( // note: when adding/changing fields here also make sure to carry it over to // update/application.go. type applicationCmd struct { - Name string `arg:"" default:"" help:"Name of the app. A random name is generated if omitted."` - Wait bool `default:"true" help:"Wait until the app is fully created."` - WaitTimeout time.Duration `default:"15m" help:"Duration to wait for the app getting ready. Only relevant if wait is set."` - Git gitConfig `embed:"" prefix:"git-"` - Size *string `help:"Size of the app (defaults to \"${app_default_size}\")." placeholder:"${app_default_size}"` - Port *int32 `help:"Port the app is listening on (defaults to ${app_default_port})." placeholder:"${app_default_port}"` - Replicas *int32 `help:"Amount of replicas of the running app (defaults to ${app_default_replicas})." placeholder:"${app_default_replicas}"` - Hosts []string `help:"Host names where the app can be accessed. If empty, the app will just be accessible on a generated host name on the deploio.app domain."` - BasicAuth *bool `help:"Enable/Disable basic authentication for the app (defaults to ${app_default_basic_auth})." placeholder:"${app_default_basic_auth}"` - Env map[string]string `help:"Environment variables which are passed to the app at runtime."` - BuildEnv map[string]string `help:"Environment variables which are passed to the app build process."` - DeployJob deployJob `embed:"" prefix:"deploy-job-"` + Name string `arg:"" default:"" help:"Name of the app. A random name is generated if omitted."` + Wait bool `default:"true" help:"Wait until the app is fully created."` + WaitTimeout time.Duration `default:"15m" help:"Duration to wait for the app getting ready. Only relevant if wait is set."` + Git gitConfig `embed:"" prefix:"git-"` + Size *string `help:"Size of the app (defaults to \"${app_default_size}\")." placeholder:"${app_default_size}"` + Port *int32 `help:"Port the app is listening on (defaults to ${app_default_port})." placeholder:"${app_default_port}"` + Replicas *int32 `help:"Amount of replicas of the running app (defaults to ${app_default_replicas})." placeholder:"${app_default_replicas}"` + Hosts []string `help:"Host names where the app can be accessed. If empty, the app will just be accessible on a generated host name on the deploio.app domain."` + BasicAuth *bool `help:"Enable/Disable basic authentication for the app (defaults to ${app_default_basic_auth})." placeholder:"${app_default_basic_auth}"` + Env map[string]string `help:"Environment variables which are passed to the app at runtime."` + BuildEnv map[string]string `help:"Environment variables which are passed to the app build process."` + DeployJob deployJob `embed:"" prefix:"deploy-job-"` + GitInformationServiceURL string `help:"URL of the git information service." default:"https://git-info.deplo.io" env:"GIT_INFORMATION_SERVICE_URL"` + SkipRepoAccessCheck bool `help:"Skip the git repository access check" default:"false"` + Debug bool `help:"Enable debug messages" default:"false"` } type gitConfig struct { @@ -65,7 +67,7 @@ type deployJob struct { func (g gitConfig) sshPrivateKey() (*string, error) { if g.SSHPrivateKey != nil { - return validatePEM(*g.SSHPrivateKey) + return util.ValidatePEM(*g.SSHPrivateKey) } if g.SSHPrivateKeyFromFile == nil { return nil, nil @@ -74,7 +76,7 @@ func (g gitConfig) sshPrivateKey() (*string, error) { if err != nil { return nil, err } - return validatePEM(string(content)) + return util.ValidatePEM(string(content)) } const ( @@ -89,7 +91,7 @@ const ( ) func (app *applicationCmd) Run(ctx context.Context, client *api.Client) error { - fmt.Println("Creating a new application") + fmt.Println("Creating new application") newApp := app.newApplication(client.Project) sshPrivateKey, err := app.Git.sshPrivateKey() @@ -102,6 +104,20 @@ func (app *applicationCmd) Run(ctx context.Context, client *api.Client) error { SSHPrivateKey: sshPrivateKey, } + if !app.SkipRepoAccessCheck { + // check if the repository is reachable via the git information service + gitInfoClient, err := util.NewGitInformationClient(app.GitInformationServiceURL, client.Token) + if err != nil { + return err + } + gitInfoClient.SetLogRetryFunc(util.RetryLogFunc(app.Debug)) + fmt.Println("Testing repository access") + if err := util.TestRepositoryAccess(ctx, gitInfoClient, &newApp.Spec.ForProvider.Git, auth); err != nil { + return err + } + fmt.Println("Repository access successfully tested") + } + if auth.Enabled() { if err := auth.Valid(); err != nil { return fmt.Errorf("the credentials are given but they are empty: %w", err) @@ -509,21 +525,6 @@ func errorLogQuery(queryString string) log.Query { } } -// validatePEM validates if the passed content is in valid PEM format, errors -// out if the content is empty -func validatePEM(content string) (*string, error) { - if content == "" { - return nil, fmt.Errorf("the SSH private key cannot be empty") - } - - content = strings.TrimSpace(content) - b, rest := pem.Decode([]byte(content)) - if b == nil || len(rest) > 0 { - return nil, fmt.Errorf("no valid PEM formatted data found") - } - return &content, nil -} - // ApplicationKongVars returns all variables which are used in the application // create command func ApplicationKongVars() (kong.Vars, error) { diff --git a/create/application_test.go b/create/application_test.go index 81052e4..7b86128 100644 --- a/create/application_test.go +++ b/create/application_test.go @@ -22,59 +22,10 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/pointer" "k8s.io/utils/ptr" ) -const ( - // dummySSHRSAPrivateKey is a dummy private RSA SSH key with some - // whitespace around - dummySSHRSAPrivateKey = ` - ------BEGIN RSA PRIVATE KEY----- -MIIEogIBAAKCAQEAgv9MjQEnssfXn8OCcVMZQUS0iP9Cpo643RkS2ENvNlnXFTgy -mLX35RgkuHoAeQIlCFgYKA9bneJNRVDLcKQg3dElsMSdx6e2W879LChKrlhu904v -XYv09Txm6+MHd69agQEU8STWXrw39Fpdk8a36MMAEe+4SzSSoh0b/2wuwRLZmapS -gQF3HmqzxlfwupPUCbVtiWf6okJFO39TCI5vWD1bVUG5WqemVD+WY+AHjFrk41LM -+i+gwlnn392FUB+NrCYlx6dKXhGTr1IMX15l6JVtgDp3AvlvNGG6JrA/CLqoOf9f -Hv8pMLqPquVnDVNB/t3U0m7x/ZV2MUetklX6KwIDAQABAoIBAFjRZoLYPKVgEBe3 -xLK3iBET12BnyjYJ4NewD3HoTvhH86fkgZG/F0QSmZsmxTlGtfsxV7eZqiGjdYbA -4B8QeWRMUUTIGr5rPR6Eem29J92MAjjVnxHLOhwohxP6y25fy3paVGun8V0sOrgH -qRjwDHPZ+ysuIQOEssMN/5SwMgcflXFgbLMjdNJxiP2TnJcjuEEHzDmXs7KAsch2 -8dE6Wj+0W3FH1HRPxnlLfqALGxB+7I8CngXaRExbHYpWyFr+ke4PAGcLuHZ7eyZ0 -86jqoo5ekyj0TTaflJVR851CQkRk9DWeFWEfQOeM5d17VWYqibHRs3r3jASMggus -61rmYRECgYEA4/blEB9CG6VRY0WMFljZrPtkr0zh0s01q3yUY95xizUOZknVLsk5 -bOvf9Ovw5DL3YMMQ8a09MVGcUFIRq6KlNdoh87hYIipEii8lRB536r3yyNHGAGVo -BGon+iOZc/ma4U7pBewooUZGF5RgSlrSlGTwusgZRADUi7ncavU81tkCgYEAkxuL -Od6HaQkZP6OtUsf/pd+Y6xLjWm6Pf+xM6eu5PIyBsvWvcTBnit765Wy/VLDK1mzr -vNgSueIi6k41MjBJdCf91R6U3WulEV7xj9uPBeQbDMFDPoZPqEeyOqlb07D++Bk9 -IJN6mVJWM/cOiQdJAhrXwqrk4vAsfeaiRKjx3qMCgYBPthE6pfNzv0bKM5NcbQ0Q -U4dNVNDR6TePEyzADxQc3Rx/3+lPRsVxtLjG54mAAeJGT28pUq5HBIZn/4p2PZUP -U4rzsc3/hFAbEYkyXIUJ7Als9w0JLmxEvunjqXcK+oiRqAoLLBy4592yeQuCdGeV -xAX5CebrxG6NvRu5uq7fYQKBgDd6j8tHTTIjqE4D4H3zx0o7RWSCPxP/1kacS3V8 -3OMk6lUfqwa5BpOs/FpB5PZ/pj+v3EfgBU/tJNXQoOdIpqsT2friCapnylz+vYNP -fmTuXfU1fbK63JfOUj0lWehAPCg8/HyooffowXHfnq+2+6W7kdtsr92WTnE85b2X -KYCZAoGAZ8hdRurgNcmaBzQfRF/lYQVvlmBkCy00YmTeSrwLerWlFHsh7T8icBDT -k2dECAM99MLPJKkOwI/E0v1pAncQunLkWDJpWwb3egr+3Az+LE2TBTaDkP4kLgOw -sMVLxmbNrxvMSjJZlSiw3jYVOnXW2jZe+ceIN4LKRwW06ifnBpg= ------END RSA PRIVATE KEY----- - -` - - // dummySSHED25519PrivateKey is a dummy SSH private key in ed25519 format with - // some whitespace around it - dummySSHED25519PrivateKey = ` - ------BEGIN OPENSSH PRIVATE KEY----- -b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW -QyNTUxOQAAACBSKbcOHZTe121IZz0EyMZMyvKPxRs8Rq1LTr+Uftr4zQAAAJDfP9T53z/U -+QAAAAtzc2gtZWQyNTUxOQAAACBSKbcOHZTe121IZz0EyMZMyvKPxRs8Rq1LTr+Uftr4zQ -AAAEDlLk0cOZ375YeCqvnfoTYl0pbFEGDaAAF4BwHqn6WqG1Iptw4dlN7XbUhnPQTIxkzK -8o/FGzxGrUtOv5R+2vjNAAAABm5vbmFtZQECAwQFBgc= ------END OPENSSH PRIVATE KEY----- - -` -) - func createTempKeyFile(content string) (string, error) { file, err := os.CreateTemp("", "temp-private-ssh-key.*.pem") if err != nil { @@ -94,22 +45,35 @@ func TestApplication(t *testing.T) { ctx := context.Background() - filenameRSAKey, err := createTempKeyFile(dummySSHRSAPrivateKey) + dummyRSAKey, err := test.GenerateRSAPrivateKey() + if err != nil { + t.Fatal(err) + } + filenameRSAKey, err := createTempKeyFile(" " + dummyRSAKey + " ") if err != nil { t.Fatal(err) } defer os.Remove(filenameRSAKey) - filenameED25519Key, err := createTempKeyFile(dummySSHED25519PrivateKey) + dummyED25519Key, err := test.GenerateED25519PrivateKey() + if err != nil { + t.Fatal(err) + } + filenameED25519Key, err := createTempKeyFile(dummyED25519Key) if err != nil { t.Fatal(err) } defer os.Remove(filenameED25519Key) + gitInfoService := test.NewGitInformationService() + gitInfoService.Start() + defer gitInfoService.Close() + cases := map[string]struct { - cmd applicationCmd - checkApp func(t *testing.T, cmd applicationCmd, app *apps.Application) - errorExpected bool + cmd applicationCmd + checkApp func(t *testing.T, cmd applicationCmd, app *apps.Application) + gitInformationServiceResponse test.GitInformationServiceResponse + errorExpected bool }{ "without git auth": { cmd: applicationCmd{ @@ -118,16 +82,17 @@ func TestApplication(t *testing.T) { SubPath: "/my/app", Revision: "superbug", }, - Wait: false, - Name: "custom-name", - Size: ptr.To("mini"), - Hosts: []string{"custom.example.org", "custom2.example.org"}, - Port: ptr.To(int32(1337)), - Replicas: ptr.To(int32(42)), - BasicAuth: ptr.To(false), - Env: map[string]string{"hello": "world"}, - BuildEnv: map[string]string{"BP_GO_TARGETS": "./cmd/web-server"}, - DeployJob: deployJob{Command: "date", Name: "print-date", Retries: 2, Timeout: time.Minute}, + Wait: false, + Name: "custom-name", + Size: ptr.To("mini"), + Hosts: []string{"custom.example.org", "custom2.example.org"}, + Port: ptr.To(int32(1337)), + Replicas: ptr.To(int32(42)), + BasicAuth: ptr.To(false), + Env: map[string]string{"hello": "world"}, + BuildEnv: map[string]string{"BP_GO_TARGETS": "./cmd/web-server"}, + DeployJob: deployJob{Command: "date", Name: "print-date", Retries: 2, Timeout: time.Minute}, + SkipRepoAccessCheck: true, }, checkApp: func(t *testing.T, cmd applicationCmd, app *apps.Application) { assert.Equal(t, cmd.Name, app.Name) @@ -150,10 +115,11 @@ func TestApplication(t *testing.T) { }, "with basic auth": { cmd: applicationCmd{ - Wait: false, - Name: "basic-auth", - Size: ptr.To("mini"), - BasicAuth: ptr.To(true), + Wait: false, + Name: "basic-auth", + Size: ptr.To("mini"), + BasicAuth: ptr.To(true), + SkipRepoAccessCheck: true, }, checkApp: func(t *testing.T, cmd applicationCmd, app *apps.Application) { assert.Equal(t, cmd.Name, app.Name) @@ -168,8 +134,9 @@ func TestApplication(t *testing.T) { Username: ptr.To("deploy"), Password: ptr.To("hunter2"), }, - Wait: false, - Name: "user-pass-auth", + Wait: false, + Name: "user-pass-auth", + SkipRepoAccessCheck: true, }, checkApp: func(t *testing.T, cmd applicationCmd, app *apps.Application) { auth := util.GitAuth{Username: cmd.Git.Username, Password: cmd.Git.Password} @@ -187,11 +154,12 @@ func TestApplication(t *testing.T) { cmd: applicationCmd{ Git: gitConfig{ URL: "https://github.com/ninech/doesnotexist.git", - SSHPrivateKey: ptr.To(dummySSHRSAPrivateKey), + SSHPrivateKey: &dummyRSAKey, }, - Wait: false, - Name: "ssh-key-auth", - Size: ptr.To("mini"), + Wait: false, + Name: "ssh-key-auth", + Size: ptr.To("mini"), + SkipRepoAccessCheck: true, }, checkApp: func(t *testing.T, cmd applicationCmd, app *apps.Application) { auth := util.GitAuth{SSHPrivateKey: cmd.Git.SSHPrivateKey} @@ -208,11 +176,12 @@ func TestApplication(t *testing.T) { cmd: applicationCmd{ Git: gitConfig{ URL: "https://github.com/ninech/doesnotexist.git", - SSHPrivateKey: ptr.To(dummySSHED25519PrivateKey), + SSHPrivateKey: &dummyED25519Key, }, - Wait: false, - Name: "ssh-key-auth-ed25519", - Size: ptr.To("mini"), + Wait: false, + Name: "ssh-key-auth-ed25519", + Size: ptr.To("mini"), + SkipRepoAccessCheck: true, }, checkApp: func(t *testing.T, cmd applicationCmd, app *apps.Application) { auth := util.GitAuth{SSHPrivateKey: cmd.Git.SSHPrivateKey} @@ -231,9 +200,10 @@ func TestApplication(t *testing.T) { URL: "https://github.com/ninech/doesnotexist.git", SSHPrivateKeyFromFile: ptr.To(filenameRSAKey), }, - Wait: false, - Name: "ssh-key-auth-from-file", - Size: ptr.To("mini"), + Wait: false, + Name: "ssh-key-auth-from-file", + Size: ptr.To("mini"), + SkipRepoAccessCheck: true, }, checkApp: func(t *testing.T, cmd applicationCmd, app *apps.Application) { auth := util.GitAuth{SSHPrivateKey: ptr.To("notused")} @@ -242,7 +212,7 @@ func TestApplication(t *testing.T) { t.Fatal(err) } - assert.Equal(t, strings.TrimSpace(dummySSHRSAPrivateKey), string(authSecret.Data[util.PrivateKeySecretKey])) + assert.Equal(t, dummyRSAKey, string(authSecret.Data[util.PrivateKeySecretKey])) assert.Equal(t, authSecret.Annotations[util.ManagedByAnnotation], util.NctlName) }, }, @@ -252,9 +222,10 @@ func TestApplication(t *testing.T) { URL: "https://github.com/ninech/doesnotexist.git", SSHPrivateKeyFromFile: ptr.To(filenameED25519Key), }, - Wait: false, - Name: "ssh-key-auth-from-file-ed25519", - Size: ptr.To("mini"), + Wait: false, + Name: "ssh-key-auth-from-file-ed25519", + Size: ptr.To("mini"), + SkipRepoAccessCheck: true, }, checkApp: func(t *testing.T, cmd applicationCmd, app *apps.Application) { auth := util.GitAuth{SSHPrivateKey: ptr.To("notused")} @@ -263,7 +234,7 @@ func TestApplication(t *testing.T) { t.Fatal(err) } - assert.Equal(t, strings.TrimSpace(dummySSHED25519PrivateKey), string(authSecret.Data[util.PrivateKeySecretKey])) + assert.Equal(t, strings.TrimSpace(dummyED25519Key), string(authSecret.Data[util.PrivateKeySecretKey])) assert.Equal(t, authSecret.Annotations[util.ManagedByAnnotation], util.NctlName) }, }, @@ -273,9 +244,10 @@ func TestApplication(t *testing.T) { URL: "https://github.com/ninech/doesnotexist.git", SSHPrivateKey: ptr.To("not valid"), }, - Wait: false, - Name: "ssh-key-auth-non-valid", - Size: ptr.To("mini"), + Wait: false, + Name: "ssh-key-auth-non-valid", + Size: ptr.To("mini"), + SkipRepoAccessCheck: true, }, errorExpected: true, }, @@ -284,10 +256,11 @@ func TestApplication(t *testing.T) { Git: gitConfig{ URL: "https://github.com/ninech/doesnotexist.git", }, - Wait: false, - Name: "deploy-job-empty-command", - Size: ptr.To("mini"), - DeployJob: deployJob{Command: "", Name: "print-date", Retries: 2, Timeout: time.Minute}, + Wait: false, + Name: "deploy-job-empty-command", + Size: ptr.To("mini"), + DeployJob: deployJob{Command: "", Name: "print-date", Retries: 2, Timeout: time.Minute}, + SkipRepoAccessCheck: true, }, checkApp: func(t *testing.T, cmd applicationCmd, app *apps.Application) { assert.Nil(t, app.Spec.ForProvider.Config.DeployJob) @@ -298,20 +271,143 @@ func TestApplication(t *testing.T) { Git: gitConfig{ URL: "https://github.com/ninech/doesnotexist.git", }, - Wait: false, - Name: "deploy-job-empty-name", - Size: ptr.To("mini"), - DeployJob: deployJob{Command: "date", Name: "", Retries: 2, Timeout: time.Minute}, + Wait: false, + Name: "deploy-job-empty-name", + Size: ptr.To("mini"), + DeployJob: deployJob{Command: "date", Name: "", Retries: 2, Timeout: time.Minute}, + SkipRepoAccessCheck: true, }, checkApp: func(t *testing.T, cmd applicationCmd, app *apps.Application) { assert.Nil(t, app.Spec.ForProvider.Config.DeployJob) }, }, + "git-information-service happy path": { + cmd: applicationCmd{ + Git: gitConfig{ + URL: "https://github.com/ninech/doesnotexist.git", + SubPath: "/my/app", + Revision: "superbug", + }, + Wait: false, + Name: "git-information-happy-path", + Size: ptr.To("mini"), + }, + gitInformationServiceResponse: test.GitInformationServiceResponse{ + Code: 200, + Content: apps.GitExploreResponse{ + RepositoryInfo: &apps.RepositoryInfo{ + URL: "https://github.com/ninech/doesnotexist.git", + Branches: []string{"main"}, + Tags: []string{"superbug"}, + }, + }, + }, + checkApp: func(t *testing.T, cmd applicationCmd, app *apps.Application) { + assert.Equal(t, cmd.Name, app.Name) + assert.Equal(t, cmd.Git.URL, app.Spec.ForProvider.Git.URL) + assert.Equal(t, cmd.Git.SubPath, app.Spec.ForProvider.Git.SubPath) + assert.Equal(t, cmd.Git.Revision, app.Spec.ForProvider.Git.Revision) + assert.Equal(t, apps.ApplicationSize(*cmd.Size), app.Spec.ForProvider.Config.Size) + assert.Nil(t, app.Spec.ForProvider.Git.Auth) + }, + }, + "git-information-service received errors": { + cmd: applicationCmd{ + Git: gitConfig{ + URL: "https://github.com/ninech/doesnotexist.git", + SubPath: "/my/app", + Revision: "superbug", + }, + Wait: false, + Name: "git-information-errors", + Size: ptr.To("mini"), + }, + gitInformationServiceResponse: test.GitInformationServiceResponse{ + Code: 200, + Content: apps.GitExploreResponse{ + Error: "repository does not exist", + }, + }, + errorExpected: true, + }, + "git-information-service revision unknown": { + cmd: applicationCmd{ + Git: gitConfig{ + URL: "https://github.com/ninech/doesnotexist.git", + SubPath: "/my/app", + Revision: "notexistent", + }, + Wait: false, + Name: "git-information-unknown-revision", + Size: ptr.To("mini"), + }, + gitInformationServiceResponse: test.GitInformationServiceResponse{ + Code: 200, + Content: apps.GitExploreResponse{ + RepositoryInfo: &apps.RepositoryInfo{ + URL: "https://github.com/ninech/doesnotexist.git", + Branches: []string{"main"}, + Tags: []string{"v1.0"}, + }, + }, + }, + errorExpected: true, + }, + "git-information-service has issues": { + cmd: applicationCmd{ + Git: gitConfig{ + URL: "https://github.com/ninech/doesnotexist.git", + SubPath: "/my/app", + Revision: "notexistent", + }, + Wait: false, + Name: "git-information-unknown-revision", + Size: ptr.To("mini"), + }, + gitInformationServiceResponse: test.GitInformationServiceResponse{ + Code: 501, + Raw: pointer.String("maintenance mode - we will be back soon"), + }, + errorExpected: true, + }, + "git URL without proper scheme should be updated to HTTPS on success": { + cmd: applicationCmd{ + Git: gitConfig{ + URL: "github.com/ninech/doesnotexist.git", + SubPath: "/my/app", + Revision: "main", + }, + Wait: false, + Name: "git-information-update-url-to-https", + Size: ptr.To("mini"), + }, + gitInformationServiceResponse: test.GitInformationServiceResponse{ + Code: 200, + Content: apps.GitExploreResponse{ + RepositoryInfo: &apps.RepositoryInfo{ + URL: "https://github.com/ninech/doesnotexist.git", + Branches: []string{"main"}, + }, + }, + }, + checkApp: func(t *testing.T, cmd applicationCmd, app *apps.Application) { + assert.Equal(t, cmd.Name, app.Name) + assert.Equal(t, "https://github.com/ninech/doesnotexist.git", app.Spec.ForProvider.Git.URL) + assert.Equal(t, cmd.Git.SubPath, app.Spec.ForProvider.Git.SubPath) + assert.Equal(t, cmd.Git.Revision, app.Spec.ForProvider.Git.Revision) + assert.Equal(t, apps.ApplicationSize(*cmd.Size), app.Spec.ForProvider.Config.Size) + assert.Nil(t, app.Spec.ForProvider.Git.Auth) + }, + }, } for name, tc := range cases { tc := tc t.Run(name, func(t *testing.T) { + if tc.cmd.GitInformationServiceURL == "" { + tc.cmd.GitInformationServiceURL = gitInfoService.URL() + } + gitInfoService.SetResponse(tc.gitInformationServiceResponse) app := tc.cmd.newApplication("default") err := tc.cmd.Run(ctx, apiClient) @@ -331,10 +427,11 @@ func TestApplication(t *testing.T) { func TestApplicationWait(t *testing.T) { cmd := applicationCmd{ - Wait: true, - WaitTimeout: time.Second * 5, - Name: "some-name", - BasicAuth: ptr.To(true), + Wait: true, + WaitTimeout: time.Second * 5, + Name: "some-name", + BasicAuth: ptr.To(true), + SkipRepoAccessCheck: true, } project := "default" @@ -470,9 +567,10 @@ func TestApplicationWait(t *testing.T) { func TestApplicationBuildFail(t *testing.T) { cmd := applicationCmd{ - Wait: true, - WaitTimeout: time.Second * 5, - Name: "some-name", + Wait: true, + WaitTimeout: time.Second * 5, + Name: "some-name", + SkipRepoAccessCheck: true, } project := "default" diff --git a/internal/test/apps.go b/internal/test/apps.go new file mode 100644 index 0000000..498a010 --- /dev/null +++ b/internal/test/apps.go @@ -0,0 +1,131 @@ +package test + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "fmt" + "strings" + + mathrand "math/rand" + + "golang.org/x/crypto/ed25519" + "golang.org/x/crypto/ssh" +) + +const ( + keySize = 4096 +) + +// GenerateRSAPrivateKey generates an RSA private key +func GenerateRSAPrivateKey() (string, error) { + // Private Key generation + privateKey, err := rsa.GenerateKey(rand.Reader, keySize) + if err != nil { + return "", err + } + // Validate Private Key + err = privateKey.Validate() + if err != nil { + return "", err + } + // Get ASN.1 DER format + privDER := x509.MarshalPKCS1PrivateKey(privateKey) + // pem.Block + privBlock := pem.Block{ + Type: "RSA PRIVATE KEY", + Headers: nil, + Bytes: privDER, + } + // Private key in PEM format + return strings.TrimSpace(string(pem.EncodeToMemory(&privBlock))), nil +} + +// GenerateED25519PrivateKey generates an ED25519 private key +func GenerateED25519PrivateKey() (string, error) { + publicKey, privateKey, err := ed25519.GenerateKey(nil) + if err != nil { + return "", fmt.Errorf("can not create ed25519 key: %w", err) + } + pemKey := &pem.Block{ + Type: "OPENSSH PRIVATE KEY", + Bytes: marshalPrivateED25519Key(privateKey, publicKey), + } + + // Private key in PEM format + return strings.TrimSpace(string(pem.EncodeToMemory(pemKey))), nil +} + +// Writes ed25519 private keys into the new OpenSSH private key format. +// (Function taken and slightly modified from: https://github.com/mikesmitty/edkey/blob/master/edkey.go) +func marshalPrivateED25519Key(privateKey ed25519.PrivateKey, publicKey ed25519.PublicKey) []byte { + // Add our key header (followed by a null byte) + magic := append([]byte("openssh-key-v1"), 0) + + var w struct { + CipherName string + KdfName string + KdfOpts string + NumKeys uint32 + PubKey []byte + PrivKeyBlock []byte + } + + // Fill out the private key fields + pk1 := struct { + Check1 uint32 + Check2 uint32 + Keytype string + Pub []byte + Priv []byte + Comment string + Pad []byte `ssh:"rest"` + }{} + + // Set our check ints + ci := mathrand.Uint32() + pk1.Check1 = ci + pk1.Check2 = ci + + // Set our key type + pk1.Keytype = ssh.KeyAlgoED25519 + + // Add the pubkey to the optionally-encrypted block + pk1.Pub = []byte(publicKey) + + // Add our private key + pk1.Priv = []byte(privateKey) + + // Might be useful to put something in here at some point + pk1.Comment = "" + + // Add some padding to match the encryption block size within PrivKeyBlock (without Pad field) + // 8 doesn't match the documentation, but that's what ssh-keygen uses for unencrypted keys. *shrug* + bs := 8 + blockLen := len(ssh.Marshal(pk1)) + padLen := (bs - (blockLen % bs)) % bs + pk1.Pad = make([]byte, padLen) + + // Padding is a sequence of bytes like: 1, 2, 3... + for i := 0; i < padLen; i++ { + pk1.Pad[i] = byte(i + 1) + } + + // Generate the pubkey prefix "\0\0\0\nssh-ed25519\0\0\0 " + prefix := []byte{0x0, 0x0, 0x0, 0x0b} + prefix = append(prefix, []byte(ssh.KeyAlgoED25519)...) + prefix = append(prefix, []byte{0x0, 0x0, 0x0, 0x20}...) + + // Only going to support unencrypted keys for now + w.CipherName = "none" + w.KdfName = "none" + w.KdfOpts = "" + w.NumKeys = 1 + w.PubKey = append(prefix, []byte(publicKey)...) + w.PrivKeyBlock = ssh.Marshal(pk1) + + magic = append(magic, ssh.Marshal(w)...) + + return magic +} diff --git a/internal/test/git_information_service.go b/internal/test/git_information_service.go new file mode 100644 index 0000000..964bb83 --- /dev/null +++ b/internal/test/git_information_service.go @@ -0,0 +1,145 @@ +package test + +import ( + "encoding/json" + "errors" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync" + + apps "github.com/ninech/apis/apps/v1alpha1" + "k8s.io/utils/pointer" +) + +// GitInformationServiceResponse describes a response of the git information service +type GitInformationServiceResponse struct { + // Code is the status code to be set + Code int + // Content describes the GitExploreResponse to be returned. + Content apps.GitExploreResponse + // Raw allows to return any text instead of a real GitExploreResponse. + // If it is set, it has precedence over the Content field. + Raw *string +} + +type GitInfoServiceParsed struct { + Token string + Method string + Request apps.GitExploreRequest +} + +// VerifyRequestFunc can be used to verify the parsed request which was sent to +// the git information service +type VerifyRequestFunc func(p GitInfoServiceParsed, err error) + +type gitInformationService struct { + sync.Mutex + server *httptest.Server + logger *slog.Logger + response GitInformationServiceResponse + request struct { + data GitInfoServiceParsed + err error + } +} + +func defaultResponse() GitInformationServiceResponse { + return GitInformationServiceResponse{ + Code: 404, + Raw: pointer.String("no response set"), + } +} + +// NewGitInformationService returns a new git information service mock. It can +// be used to verify requests sent to it and also to just return with a +// previously set response. +func NewGitInformationService() *gitInformationService { + g := &gitInformationService{ + logger: slog.New(slog.NewJSONHandler(os.Stdout, nil)), + response: defaultResponse(), + } + mux := http.NewServeMux() + mux.Handle("/explore", g) + g.server = httptest.NewUnstartedServer(mux) + return g +} + +// SetResponse sets the response to be returned +func (g *gitInformationService) SetResponse(r GitInformationServiceResponse) { + g.Lock() + defer g.Unlock() + g.response = r +} + +func (g *gitInformationService) Start() { + g.server.Start() +} + +func (g *gitInformationService) Close() { + if g.server != nil { + g.server.Close() + } +} + +func (g *gitInformationService) URL() string { + return g.server.URL +} + +// Request returns the parsed last request to the service or an eventual error which occured during parsing +func (g *gitInformationService) Request() (GitInfoServiceParsed, error) { + return g.request.data, g.request.err +} + +func (g *gitInformationService) ServeHTTP(w http.ResponseWriter, r *http.Request) { + g.request.data, g.request.err = parseRequest(r) + + w.WriteHeader(g.response.Code) + if g.response.Raw != nil { + _, err := w.Write([]byte(*g.response.Raw)) + if err != nil { + g.logger.Error(err.Error()) + } + return + } + content, err := json.Marshal(g.response.Content) + if err != nil { + g.logger.Error("error when marshaling response", "error", err.Error()) + } + _, err = w.Write(content) + if err != nil { + g.logger.Error(err.Error()) + } +} + +func parseRequest(r *http.Request) (GitInfoServiceParsed, error) { + p := GitInfoServiceParsed{} + p.Token = r.Header.Get("Authorization") + if strings.HasPrefix(p.Token, "Bearer") { + p.Token = strings.TrimSpace(p.Token[len("Bearer"):]) + } + + exploreRequest := apps.GitExploreRequest{} + if err := unmarshalRequest(r.Body, &exploreRequest); err != nil { + return p, err + } + p.Request = exploreRequest + p.Method = r.Method + return p, nil +} + +func unmarshalRequest(data io.Reader, request *apps.GitExploreRequest) error { + decoder := json.NewDecoder(data) + err := decoder.Decode(request) + if err != nil { + return err + } + _, err = decoder.Token() + if err == nil || !errors.Is(err, io.EOF) { + return errors.New("invalid data after top-level JSON value") + } + return nil +} diff --git a/update/application.go b/update/application.go index 844585a..3ab74ea 100644 --- a/update/application.go +++ b/update/application.go @@ -3,6 +3,7 @@ package update import ( "context" "fmt" + "os" "time" "github.com/crossplane/crossplane-runtime/pkg/resource" @@ -19,28 +20,46 @@ const BuildTrigger = "BUILD_TRIGGER" // all fields need to be pointers so we can detect if they have been set by // the user. type applicationCmd struct { - Name *string `arg:"" help:"Name of the application."` - Git *gitConfig `embed:"" prefix:"git-"` - Size *string `help:"Size of the app."` - Port *int32 `help:"Port the app is listening on."` - Replicas *int32 `help:"Amount of replicas of the running app."` - Hosts *[]string `help:"Host names where the application can be accessed. If empty, the application will just be accessible on a generated host name on the deploio.app domain."` - BasicAuth *bool `help:"Enable/Disable basic authentication for the application."` - Env *map[string]string `help:"Environment variables which are passed to the app at runtime."` - DeleteEnv *[]string `help:"Runtime environment variables names which are to be deleted."` - BuildEnv *map[string]string `help:"Environment variables names which are passed to the app build process."` - DeleteBuildEnv *[]string `help:"Build environment variables which are to be deleted."` - DeployJob *deployJob `embed:"" prefix:"deploy-job-"` - RetryBuild *bool `help:"Retries build for the application if set to true." placeholder:"false"` + Name *string `arg:"" help:"Name of the application."` + Git *gitConfig `embed:"" prefix:"git-"` + Size *string `help:"Size of the app."` + Port *int32 `help:"Port the app is listening on."` + Replicas *int32 `help:"Amount of replicas of the running app."` + Hosts *[]string `help:"Host names where the application can be accessed. If empty, the application will just be accessible on a generated host name on the deploio.app domain."` + BasicAuth *bool `help:"Enable/Disable basic authentication for the application."` + Env *map[string]string `help:"Environment variables which are passed to the app at runtime."` + DeleteEnv *[]string `help:"Runtime environment variables names which are to be deleted."` + BuildEnv *map[string]string `help:"Environment variables names which are passed to the app build process."` + DeleteBuildEnv *[]string `help:"Build environment variables which are to be deleted."` + DeployJob *deployJob `embed:"" prefix:"deploy-job-"` + RetryBuild *bool `help:"Retries build for the application if set to true." placeholder:"false"` + GitInformationServiceURL string `help:"URL of the git information service." default:"https://git-info.deplo.io" env:"GIT_INFORMATION_SERVICE_URL"` + SkipRepoAccessCheck bool `help:"Skip the git repository access check" default:"false"` + Debug bool `help:"Enable debug messages" default:"false"` } type gitConfig struct { - URL *string `help:"URL to the Git repository containing the application source. Both HTTPS and SSH formats are supported."` - SubPath *string `help:"SubPath is a path in the git repo which contains the application code. If not given, the root directory of the git repo will be used."` - Revision *string `help:"Revision defines the revision of the source to deploy the application to. This can be a commit, tag or branch."` - Username *string `help:"Username to use when authenticating to the git repository over HTTPS." env:"GIT_USERNAME"` - Password *string `help:"Password to use when authenticating to the git repository over HTTPS. In case of GitHub or GitLab, this can also be an access token." env:"GIT_PASSWORD"` - SSHPrivateKey *string `help:"Private key in x509 format to connect to the git repository via SSH." env:"GIT_SSH_PRIVATE_KEY"` + URL *string `help:"URL to the Git repository containing the application source. Both HTTPS and SSH formats are supported."` + SubPath *string `help:"SubPath is a path in the git repo which contains the application code. If not given, the root directory of the git repo will be used."` + Revision *string `help:"Revision defines the revision of the source to deploy the application to. This can be a commit, tag or branch."` + Username *string `help:"Username to use when authenticating to the git repository over HTTPS." env:"GIT_USERNAME"` + Password *string `help:"Password to use when authenticating to the git repository over HTTPS. In case of GitHub or GitLab, this can also be an access token." env:"GIT_PASSWORD"` + SSHPrivateKey *string `help:"Private key in x509 format to connect to the git repository via SSH." env:"GIT_SSH_PRIVATE_KEY"` + SSHPrivateKeyFromFile *string `help:"Path to a file containing a private key in PEM format to connect to the git repository via SSH." env:"GIT_SSH_PRIVATE_KEY_FROM_FILE" xor:"SSH_KEY"` +} + +func (g gitConfig) sshPrivateKey() (*string, error) { + if g.SSHPrivateKey != nil { + return util.ValidatePEM(*g.SSHPrivateKey) + } + if g.SSHPrivateKeyFromFile == nil { + return nil, nil + } + content, err := os.ReadFile(*g.SSHPrivateKeyFromFile) + if err != nil { + return nil, err + } + return util.ValidatePEM(string(content)) } type deployJob struct { @@ -71,35 +90,55 @@ func (cmd *applicationCmd) Run(ctx context.Context, client *api.Client) error { if !ok { return fmt.Errorf("resource is of type %T, expected %T", current, apps.Application{}) } - cmd.applyUpdates(app) - if cmd.Git != nil { - auth := util.GitAuth{ - Username: cmd.Git.Username, - Password: cmd.Git.Password, - SSHPrivateKey: cmd.Git.SSHPrivateKey, - } + // if there was no change in the git config, we don't have + // anything to do anymore + if cmd.Git == nil { + return nil + } - if auth.Enabled() { - secret := auth.Secret(app) - if err := client.Get(ctx, client.Name(secret.Name), secret); err != nil { - if errors.IsNotFound(err) { - auth.UpdateSecret(secret) - if err := client.Create(ctx, secret); err != nil { - return err - } + sshPrivateKey, err := cmd.Git.sshPrivateKey() + if err != nil { + return fmt.Errorf("error when reading SSH private key: %w", err) + } + auth := util.GitAuth{ + Username: cmd.Git.Username, + Password: cmd.Git.Password, + SSHPrivateKey: sshPrivateKey, + } + if !cmd.SkipRepoAccessCheck { + // check if the repository is reachable via the git information service + gitInfoClient, err := util.NewGitInformationClient(cmd.GitInformationServiceURL, client.Token) + if err != nil { + return err + } + gitInfoClient.SetLogRetryFunc(util.RetryLogFunc(cmd.Debug)) + fmt.Println("Testing repository access") + if err := util.TestRepositoryAccess(ctx, gitInfoClient, &app.Spec.ForProvider.Git, auth); err != nil { + return err + } + fmt.Println("Repository access successfully tested") + } - return nil + if auth.Enabled() { + secret := auth.Secret(app) + if err := client.Get(ctx, client.Name(secret.Name), secret); err != nil { + if errors.IsNotFound(err) { + auth.UpdateSecret(secret) + if err := client.Create(ctx, secret); err != nil { + return err } - return err + return nil } - auth.UpdateSecret(secret) - if err := client.Update(ctx, secret); err != nil { - return err - } + return err + } + + auth.UpdateSecret(secret) + if err := client.Update(ctx, secret); err != nil { + return err } } diff --git a/update/application_test.go b/update/application_test.go index 0f28091..480ab94 100644 --- a/update/application_test.go +++ b/update/application_test.go @@ -2,6 +2,7 @@ package update import ( "context" + "strings" "testing" "time" @@ -9,10 +10,12 @@ import ( apps "github.com/ninech/apis/apps/v1alpha1" "github.com/ninech/nctl/api" "github.com/ninech/nctl/api/util" + "github.com/ninech/nctl/internal/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -25,6 +28,16 @@ func TestApplication(t *testing.T) { } initialSize := apps.ApplicationSize("micro") + + dummyRSAKey, err := test.GenerateRSAPrivateKey() + if err != nil { + t.Fatal(err) + } + + gitInfoService := test.NewGitInformationService() + gitInfoService.Start() + defer gitInfoService.Close() + existingApp := &apps.Application{ ObjectMeta: metav1.ObjectMeta{ Name: "some-name", @@ -63,11 +76,13 @@ func TestApplication(t *testing.T) { } cases := map[string]struct { - orig *apps.Application - gitAuth *util.GitAuth - cmd applicationCmd - checkApp func(t *testing.T, cmd applicationCmd, orig, updated *apps.Application) - checkSecret func(t *testing.T, cmd applicationCmd, authSecret *corev1.Secret) + orig *apps.Application + gitAuth *util.GitAuth + cmd applicationCmd + checkApp func(t *testing.T, cmd applicationCmd, orig, updated *apps.Application) + checkSecret func(t *testing.T, cmd applicationCmd, authSecret *corev1.Secret) + gitInformationServiceResponse test.GitInformationServiceResponse + errorExpected bool }{ "change port": { orig: existingApp, @@ -110,6 +125,7 @@ func TestApplication(t *testing.T) { Command: ptr.To("exit 0"), Name: ptr.To("exit"), Retries: ptr.To(int32(1)), Timeout: ptr.To(time.Minute * 5), }, + SkipRepoAccessCheck: true, }, checkApp: func(t *testing.T, cmd applicationCmd, orig, updated *apps.Application) { assert.Equal(t, *cmd.Git.URL, updated.Spec.ForProvider.Git.URL) @@ -141,6 +157,18 @@ func TestApplication(t *testing.T) { assert.NotEmpty(t, updated.Spec.ForProvider.BuildEnv) }, }, + "change multiple env variables at once": { + orig: existingApp, + cmd: applicationCmd{ + Name: ptr.To(existingApp.Name), + Env: &map[string]string{"bar1": "zoo", "bar2": "foo"}, + }, + checkApp: func(t *testing.T, cmd applicationCmd, orig, updated *apps.Application) { + assert.Contains(t, updated.Spec.ForProvider.Config.Env, apps.EnvVar{Name: "bar1", Value: "zoo"}) + assert.Contains(t, updated.Spec.ForProvider.Config.Env, apps.EnvVar{Name: "bar2", Value: "foo"}) + assert.Contains(t, updated.Spec.ForProvider.Config.Env, apps.EnvVar{Name: "foo", Value: "bar"}) + }, + }, "reset build env variable": { orig: existingApp, cmd: applicationCmd{ @@ -165,6 +193,15 @@ func TestApplication(t *testing.T) { Password: ptr.To("new-pass"), }, }, + gitInformationServiceResponse: test.GitInformationServiceResponse{ + Code: 200, + Content: apps.GitExploreResponse{ + RepositoryInfo: &apps.RepositoryInfo{ + URL: existingApp.Spec.ForProvider.Git.URL, + Branches: []string{existingApp.Spec.ForProvider.Git.Revision}, + }, + }, + }, checkSecret: func(t *testing.T, cmd applicationCmd, authSecret *corev1.Secret) { assert.Equal(t, *cmd.Git.Username, string(authSecret.Data[util.UsernameSecretKey])) assert.Equal(t, *cmd.Git.Password, string(authSecret.Data[util.PasswordSecretKey])) @@ -179,11 +216,20 @@ func TestApplication(t *testing.T) { cmd: applicationCmd{ Name: ptr.To(existingApp.Name), Git: &gitConfig{ - SSHPrivateKey: ptr.To("newfakekey"), + SSHPrivateKey: &dummyRSAKey, + }, + }, + gitInformationServiceResponse: test.GitInformationServiceResponse{ + Code: 200, + Content: apps.GitExploreResponse{ + RepositoryInfo: &apps.RepositoryInfo{ + URL: existingApp.Spec.ForProvider.Git.URL, + Branches: []string{existingApp.Spec.ForProvider.Git.Revision}, + }, }, }, checkSecret: func(t *testing.T, cmd applicationCmd, authSecret *corev1.Secret) { - assert.Equal(t, *cmd.Git.SSHPrivateKey, string(authSecret.Data[util.PrivateKeySecretKey])) + assert.Equal(t, strings.TrimSpace(*cmd.Git.SSHPrivateKey), string(authSecret.Data[util.PrivateKeySecretKey])) assert.Equal(t, authSecret.Annotations[util.ManagedByAnnotation], util.NctlName) }, }, @@ -197,6 +243,15 @@ func TestApplication(t *testing.T) { Password: ptr.To("new-pass"), }, }, + gitInformationServiceResponse: test.GitInformationServiceResponse{ + Code: 200, + Content: apps.GitExploreResponse{ + RepositoryInfo: &apps.RepositoryInfo{ + URL: existingApp.Spec.ForProvider.Git.URL, + Branches: []string{existingApp.Spec.ForProvider.Git.Revision}, + }, + }, + }, checkSecret: func(t *testing.T, cmd applicationCmd, authSecret *corev1.Secret) { assert.Equal(t, *cmd.Git.Username, string(authSecret.Data[util.UsernameSecretKey])) assert.Equal(t, *cmd.Git.Password, string(authSecret.Data[util.PasswordSecretKey])) @@ -214,6 +269,15 @@ func TestApplication(t *testing.T) { URL: ptr.To("https://newgit.example.org"), }, }, + gitInformationServiceResponse: test.GitInformationServiceResponse{ + Code: 200, + Content: apps.GitExploreResponse{ + RepositoryInfo: &apps.RepositoryInfo{ + URL: "https://newgit.example.org", + Branches: []string{existingApp.Spec.ForProvider.Git.Revision}, + }, + }, + }, checkApp: func(t *testing.T, cmd applicationCmd, orig, updated *apps.Application) { assert.Equal(t, *cmd.Git.URL, updated.Spec.ForProvider.Git.URL) }, @@ -234,6 +298,15 @@ func TestApplication(t *testing.T) { }, DeployJob: &deployJob{Enabled: ptr.To(false)}, }, + gitInformationServiceResponse: test.GitInformationServiceResponse{ + Code: 200, + Content: apps.GitExploreResponse{ + RepositoryInfo: &apps.RepositoryInfo{ + URL: "https://newgit.example.org", + Branches: []string{existingApp.Spec.ForProvider.Git.Revision}, + }, + }, + }, checkApp: func(t *testing.T, cmd applicationCmd, orig, updated *apps.Application) { assert.Nil(t, updated.Spec.ForProvider.Config.DeployJob) }, @@ -258,12 +331,95 @@ func TestApplication(t *testing.T) { assert.Nil(t, util.EnvVarByName(updated.Spec.ForProvider.BuildEnv, BuildTrigger)) }, }, + "disabling the git repo check works": { + orig: existingApp, + cmd: applicationCmd{ + Name: ptr.To(existingApp.Name), + Git: &gitConfig{ + URL: ptr.To("https://newgit.example.org"), + }, + SkipRepoAccessCheck: true, + }, + gitInformationServiceResponse: test.GitInformationServiceResponse{ + Code: 200, + Content: apps.GitExploreResponse{ + Error: "repository can not be accessed", + }, + }, + checkApp: func(t *testing.T, cmd applicationCmd, orig, updated *apps.Application) { + assert.Equal(t, *cmd.Git.URL, updated.Spec.ForProvider.Git.URL) + }, + }, + "an error on the git repo check will lead to an error shown to the user": { + orig: existingApp, + cmd: applicationCmd{ + Name: ptr.To(existingApp.Name), + Git: &gitConfig{ + URL: ptr.To("https://newgit.example.org"), + }, + }, + gitInformationServiceResponse: test.GitInformationServiceResponse{ + Code: 200, + Content: apps.GitExploreResponse{ + Error: "repository can not be accessed", + }, + }, + errorExpected: true, + }, + "specifying a non existing branch/tag will be detected": { + orig: existingApp, + cmd: applicationCmd{ + Name: ptr.To(existingApp.Name), + Git: &gitConfig{ + URL: ptr.To("https://newgit.example.org"), + Revision: pointer.String("not-existent"), + }, + }, + gitInformationServiceResponse: test.GitInformationServiceResponse{ + Code: 200, + Content: apps.GitExploreResponse{ + RepositoryInfo: &apps.RepositoryInfo{ + URL: "https://newgit.example.org", + Branches: []string{"main"}, + }, + }, + }, + errorExpected: true, + }, + "defaulting to HTTPS when not specifying a scheme in a git URL works": { + orig: existingApp, + cmd: applicationCmd{ + Name: ptr.To(existingApp.Name), + Git: &gitConfig{ + URL: ptr.To("github.com/ninech/new-repo"), + Revision: pointer.String("main"), + }, + }, + gitInformationServiceResponse: test.GitInformationServiceResponse{ + Code: 200, + Content: apps.GitExploreResponse{ + RepositoryInfo: &apps.RepositoryInfo{ + URL: "https://github.com/ninech/new-repo", + Branches: []string{"main"}, + }, + }, + }, + checkApp: func(t *testing.T, cmd applicationCmd, orig, updated *apps.Application) { + assert.Equal(t, "https://github.com/ninech/new-repo", updated.Spec.ForProvider.Git.URL) + assert.Equal(t, "main", updated.Spec.ForProvider.Git.Revision) + }, + }, } for name, tc := range cases { tc := tc t.Run(name, func(t *testing.T) { + if tc.cmd.GitInformationServiceURL == "" { + tc.cmd.GitInformationServiceURL = gitInfoService.URL() + } + gitInfoService.SetResponse(tc.gitInformationServiceResponse) + objects := []client.Object{tc.orig} if tc.gitAuth != nil { objects = append(objects, tc.gitAuth.Secret(tc.orig)) @@ -273,7 +429,11 @@ func TestApplication(t *testing.T) { ctx := context.Background() if err := tc.cmd.Run(ctx, apiClient); err != nil { - t.Fatal(err) + if tc.errorExpected { + require.Error(t, err) + } else { + require.NoError(t, err) + } } updated := &apps.Application{}