diff --git a/README.md b/README.md index 5bd1c1c..d005ea7 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ Builder is great at guessing what to do with most repos it's given, for the othe - only necessary argument is a github repo url - `builder config`: user defined (user created builder.yaml) project build that creates artifact with metadata and logs - only necessary argument is a github repo url -- `builder docker`: generates a docker image from dockerfile which utilizes the Builder binary +- `builder docker`: generates a docker image from dockerfile - Use the `--release` or `-r` flag to push to a remote registry and save it for future builds - `builder push `: Pushes build metadata and logs JSON to the specified URL upon build completion - Optionally, you can omit the URL at the end of the command if the push URL is provided in `builder.yaml`. diff --git a/artifact/artifactDir.go b/artifact/artifactDir.go index 670b4a2..63e1ad5 100644 --- a/artifact/artifactDir.go +++ b/artifact/artifactDir.go @@ -37,7 +37,7 @@ func ArtifactDir() { //check workspace env exists, if not, create it _, present := os.LookupEnv("BUILDER_ARTIFACT_DIR") - if !present { + if !present || os.Getenv("BUILDER_ARTIFACT_DIR") == "" { os.Setenv("BUILDER_ARTIFACT_DIR", artifactDir) } } diff --git a/cmd/docker.go b/cmd/docker.go index 19d5e5e..042e592 100644 --- a/cmd/docker.go +++ b/cmd/docker.go @@ -7,6 +7,7 @@ import ( "Builder/utils" "Builder/utils/log" "Builder/yaml" + "crypto/sha256" "bufio" "encoding/json" @@ -19,51 +20,11 @@ import ( goYaml "gopkg.in/yaml.v2" ) -var closeLocalLogger func() - -// BuildMetadata holds the data gathered from the build done in the docker container -type BuildMetadata struct { - ProjectName string `json:"ProjectName"` - ProjectType string `json:"ProjectType"` - ArtifactName string `json:"ArtifactName"` - ArtifactChecksums string `json:"ArtifactChecksums"` - ArtifactLocation string `json:"ArtifactLocation"` - LogsLocation string `json:"LogsLocation"` - UserName string `json:"UserName"` - HomeDir string `json:"HomeDir"` - IP string `json:"IP"` - StartTime string `json:"StartTime"` - EndTime string `json:"EndTime"` - GitURL string `json:"GitURL"` - MasterGitHash string `json:"MasterGitHash"` - BranchName string `json:"BranchName"` - BuildID string `json:"BuildID"` -} - -// DockerMetadata holds the data gathered during the docker image build -type DockerMetadata struct { - ProjectName string `json:"ProjectName"` - ProjectType string `json:"ProjectType"` - ArtifactName string `json:"ArtifactName"` - ArtifactChecksums string `json:"ArtifactChecksums"` - ArtifactLocation string `json:"ArtifactLocation"` - LogsLocation string `json:"LogsLocation"` - UserName string `json:"UserName"` - HomeDir string `json:"HomeDir"` - IP string `json:"IP"` - StartTime string `json:"StartTime"` - EndTime string `json:"EndTime"` - GitURL string `json:"GitURL"` - MasterGitHash string `json:"MasterGitHash"` - BranchName string `json:"BranchName"` -} +//var closeLocalLogger func() // AllDockerMetaData holds the data for docker metadata.json file type AllDockerMetadata struct { - DockerBuild BuildMetadata - DockerImageTag DockerMetadata ProjectName string - ProjectType string UserName string StartTime string EndTime string @@ -82,14 +43,6 @@ func Docker() { // Start loading spinner spinner.Spinner.Start() - // If builder_data folder already exists, remove it - if _, err := os.Stat(path + "/" + "builder_data"); err == nil { - e := os.RemoveAll(path + "/" + "builder_data") - if e != nil { - spinner.LogMessage("Couldn't remove builder data from previous build: "+e.Error(), "error") - } - } - //checks if yaml file exists in path, if it does, continue if _, err := os.Stat(path + "/" + "builder.yaml"); err == nil { //parse builder.yaml @@ -145,7 +98,7 @@ func Docker() { //Set up local logger localPath, _ := os.LookupEnv("BUILDER_LOGS_DIR") - locallogger, closeLocalLogger := log.NewLogger("docker_logs", localPath) + locallogger, closeLocalLogger := log.NewLogger("logs", localPath) // Use the scanner to scan the output line by line and log it // It's running in a goroutine so that it doesn't block @@ -157,7 +110,7 @@ func Docker() { spinner.Spinner.Stop() locallogger.Info(line) - if strings.Contains(line, "docker"){ + if strings.Contains(line, "docker") { if strings.Contains(line, "permission denied") { spinner.LogMessage("Docker might not have the right permissions to run this command. For Mac users, try running `sudo chown -R $(whoami) ~/.docker` and for Linux users, try adding your user to the docker group with `sudo usermod -aG docker $(whoami)`", "fatal") } @@ -171,157 +124,39 @@ func Docker() { }() - if err := cmd.Start(); err != nil { - spinner.LogMessage(err.Error(), "fatal") - } - - // Wait for all output to be processed - <-done - - // Wait for cmd to finish - if err := cmd.Wait(); err != nil { - spinner.LogMessage(err.Error(), "fatal") - } - - copyPath := os.Getenv("BUILDER_WORKSPACE_DIR") - - // Create a container from the image to grab build info - if err := exec.Command("docker", "create", "--name", name+"_"+fmt.Sprint(startTime.Unix()), name+"_"+fmt.Sprint(startTime.Unix())+":latest").Run(); err != nil { - spinner.LogMessage(err.Error(), "fatal") - } - - // Copy build info from created container - cmd = exec.Command("docker", "cp", name+"_"+fmt.Sprint(startTime.Unix())+":/root/.builder/builds.json", "./builder_builds.json") - cmd.Dir = copyPath - - //run cmd, check for err, log cmd - spinner.LogMessage("retrieving build info from container...", "info") - - stdout, pipeErr = cmd.StdoutPipe() - if pipeErr != nil { - spinner.LogMessage(pipeErr.Error(), "fatal") - } - - cmd.Stderr = cmd.Stdout - - // Make a new channel which will be used to ensure we get all output - copyMetadataFileFromContainer := make(chan struct{}) - - scanner = bufio.NewScanner(stdout) - - // Use the scanner to scan the output line by line and log it - // It's running in a goroutine so that it doesn't block - go func() { - // Read line by line and process it - for scanner.Scan() { - line := scanner.Text() - // Have to stop spinner or it will get printed with log to console - spinner.Spinner.Stop() - locallogger.Info(line) - spinner.Spinner.Start() - } - - // We're all done, unblock the channel - copyMetadataFileFromContainer <- struct{}{} - - }() - - if err := cmd.Start(); err != nil { - spinner.LogMessage(err.Error(), "fatal") - } - - // Wait for all output to be processed - <-copyMetadataFileFromContainer - - // Wait for cmd to finish - if err := cmd.Wait(); err != nil { - spinner.LogMessage(err.Error(), "fatal") - } - - // Retrieve build info from file copied from container - data, err := os.ReadFile(copyPath + "/builder_builds.json") - if err != nil { - spinner.LogMessage("Can't open file copied from container: "+err.Error(), "fatal") - } - - // Decode json into interface - var metadata BuildMetadata - json.NewDecoder(strings.NewReader(string(data))).Decode(&metadata) - - gatheredStartTime, err := time.Parse(time.RFC850, metadata.StartTime) - if err != nil { - spinner.LogMessage("Couldn't parse time: "+err.Error(), "fatal") - } - os.Setenv("BUILD_START_TIME", metadata.StartTime) - os.Setenv("BUILD_END_TIME", metadata.EndTime) - - // Retrieve logs file from container - cmd = exec.Command("docker", "cp", name+"_"+fmt.Sprint(startTime.Unix())+":"+metadata.LogsLocation, "./builder_logs.json") - cmd.Dir = copyPath - - //run cmd, check for err, log cmd - spinner.LogMessage("retrieving build logs from container...", "info") - - stdout, pipeErr = cmd.StdoutPipe() - if pipeErr != nil { - spinner.LogMessage(pipeErr.Error(), "fatal") - } - - cmd.Stderr = cmd.Stdout - - // Make a new channel which will be used to ensure we get all output - copyLogsFileFromContainer := make(chan struct{}) - - scanner = bufio.NewScanner(stdout) - - // Use the scanner to scan the output line by line and log it - // It's running in a goroutine so that it doesn't block - go func() { - // Read line by line and process it - for scanner.Scan() { - line := scanner.Text() - // Have to stop spinner or it will get printed with log to console - spinner.Spinner.Stop() - locallogger.Info(line) - spinner.Spinner.Start() - } - - // We're all done, unblock the channel - copyLogsFileFromContainer <- struct{}{} - - }() + dockerBuildStartTime := time.Now() + os.Setenv("BUILD_START_TIME", dockerBuildStartTime.Format(time.RFC850)) if err := cmd.Start(); err != nil { spinner.LogMessage(err.Error(), "fatal") } // Wait for all output to be processed - <-copyLogsFileFromContainer + <-done // Wait for cmd to finish if err := cmd.Wait(); err != nil { spinner.LogMessage(err.Error(), "fatal") } - // Retrieve build logs from file copied from container - logsJSON, err := os.ReadFile(copyPath + "/builder_logs.json") - if err != nil { - spinner.LogMessage("Can't open file copied from container: "+err.Error(), "fatal") - } + os.Setenv("BUILD_END_TIME", time.Now().Format(time.RFC850)) - logsDir := os.Getenv("BUILDER_LOGS_DIR") + // Close log file + closeLocalLogger() - // Save build logs with docker logs to file - SaveBuildLogs(logsJSON, logsDir) + // Rename parent dir + parentDirPath := os.Getenv("BUILDER_DIR_PATH") + parentDirPath = directory.UpdateParentDirName(parentDirPath) - // Create dir for metadata - artifact.ArtifactDir() + buildID := GetDockerBuildID() + userName := utils.GetUserData().Username - // Close log file - closeLocalLogger() + // Remove any data prior to slash if included in user's name + userArray := strings.Split(userName, "\\") + userName = userArray[len(userArray)-1] // Rename docker container to same name as build completed in container - if err := exec.Command("docker", "tag", name+"_"+fmt.Sprint(startTime.Unix())+":latest", metadata.ProjectName+":"+fmt.Sprint(gatheredStartTime.Unix())).Run(); err != nil { + if err := exec.Command("docker", "tag", name+"_"+fmt.Sprint(startTime.Unix())+":latest", name+":"+fmt.Sprint(dockerBuildStartTime.Unix())).Run(); err != nil { spinner.LogMessage(err.Error(), "fatal") } @@ -330,17 +165,12 @@ func Docker() { spinner.LogMessage(err.Error(), "fatal") } - // Remove temp container - if err := exec.Command("docker", "rm", name+"_"+fmt.Sprint(startTime.Unix())).Run(); err != nil { - spinner.LogMessage(err.Error(), "fatal") - } - // Get additional tags for docker image - runningTag := fmt.Sprint(gatheredStartTime.Unix()) + "_" + metadata.BuildID + "_" + metadata.UserName + runningTag := fmt.Sprint(dockerBuildStartTime.Unix()) + "_" + buildID + "_" + userName tags := []string{ "latest", - GetHumanReadableStartTimeTag(gatheredStartTime), - fmt.Sprint(gatheredStartTime.Unix()) + "_" + metadata.BuildID, + GetHumanReadableStartTimeTag(dockerBuildStartTime), + fmt.Sprint(dockerBuildStartTime.Unix()) + "_" + buildID, runningTag, } _, masterGitHash := utils.GitMasterNameAndHash() @@ -356,9 +186,6 @@ func Docker() { tags = append(tags, version, runningTag) } - // Update parent directory name - copyPath = directory.UpdateParentDirName(copyPath) - // If Docker registry provided in the builder.yaml tag and push the image to it dockerRegistryProvided := false var remoteDockerRepo string @@ -367,21 +194,21 @@ func Docker() { spinner.LogMessage("Tagging and pushing docker image...", "info") dockerRegistry := os.Getenv("BUILDER_DOCKER_REGISTRY") - remoteDockerRepo = dockerRegistry + "/" + metadata.ProjectName + remoteDockerRepo = dockerRegistry + "/" + name // Re-tag docker image to include remote registry - if err := exec.Command("docker", "tag", metadata.ProjectName+":"+fmt.Sprint(gatheredStartTime.Unix()), remoteDockerRepo+":"+fmt.Sprint(gatheredStartTime.Unix())).Run(); err != nil { + if err := exec.Command("docker", "tag", name+":"+fmt.Sprint(dockerBuildStartTime.Unix()), remoteDockerRepo+":"+fmt.Sprint(dockerBuildStartTime.Unix())).Run(); err != nil { spinner.LogMessage("Could not re-tag docker image to include registry: "+err.Error(), "fatal") } // Remove previously tagged image - if err := exec.Command("docker", "rmi", "-f", metadata.ProjectName+":"+fmt.Sprint(gatheredStartTime.Unix())).Run(); err != nil { + if err := exec.Command("docker", "rmi", "-f", name+":"+fmt.Sprint(dockerBuildStartTime.Unix())).Run(); err != nil { spinner.LogMessage(err.Error(), "fatal") } // Add more tags to new remote image for _, tag := range tags { - if err := exec.Command("docker", "tag", remoteDockerRepo+":"+fmt.Sprint(gatheredStartTime.Unix()), remoteDockerRepo+":"+tag).Run(); err != nil { - spinner.LogMessage("Could not re-tag docker image to include registry: "+err.Error(), "fatal") + if err := exec.Command("docker", "tag", remoteDockerRepo+":"+fmt.Sprint(dockerBuildStartTime.Unix()), remoteDockerRepo+":"+tag).Run(); err != nil { + spinner.LogMessage("Could not re-tag docker image to include tag: "+tag, "fatal") } } @@ -393,7 +220,7 @@ func Docker() { spinner.LogMessage("Docker image successfully tagged and pushed to provided registry.", "info") } else { // If not using builder docker release, add the extra tags to local docker image: for _, tag := range tags { - if err := exec.Command("docker", "tag", metadata.ProjectName+":"+fmt.Sprint(gatheredStartTime.Unix()), metadata.ProjectName+":"+tag).Run(); err != nil { + if err := exec.Command("docker", "tag", name+":"+fmt.Sprint(dockerBuildStartTime.Unix()), name+":"+tag).Run(); err != nil { spinner.LogMessage("Could not re-tag docker image to include additional tag: "+err.Error(), "fatal") } } @@ -404,23 +231,19 @@ func Docker() { os.Setenv("BUILDER_DOCKER_REPO", remoteDockerRepo) os.Setenv("BUILDER_DOCKER_REPO_TAG", remoteDockerRepo+":"+runningTag) } else { - os.Setenv("BUILDER_DOCKER_REPO", metadata.ProjectName) - os.Setenv("BUILDER_DOCKER_REPO_TAG", metadata.ProjectName+":"+runningTag) + os.Setenv("BUILDER_DOCKER_REPO", name) + os.Setenv("BUILDER_DOCKER_REPO_TAG", name+":"+runningTag) } os.Setenv("BUILDER_DOCKER_TAGS", fmt.Sprintf("%+v", tags)) + // Create dir for metadata + artifact.ArtifactDir() + // Create metadata for docker image build artifactDir := os.Getenv("BUILDER_ARTIFACT_DIR") - OutputDockerMetadata(metadata, tags, artifactDir) + OutputDockerMetadata(buildID, tags, artifactDir) spinner.LogMessage("Metadata saved successfully.", "info") - // Remove unneeded docker_logs.json file - logsDir = os.Getenv("BUILDER_LOGS_DIR") - e := os.Remove(logsDir + "/docker_logs.json") - if e != nil { - spinner.LogMessage("Couldn't delete old docker_logs.json file: "+e.Error(), "error") - } - // Re-create builder.yaml to include any new vars yaml.UpdateBuilderYaml(path) @@ -453,37 +276,19 @@ func GetHumanReadableStartTimeTag(startTime time.Time) string { return finalTag } -func OutputDockerMetadata(buildMetadata BuildMetadata, imageTags []string, path string) { - workspaceDir := os.Getenv("BUILDER_WORKSPACE_DIR") - utils.Metadata(workspaceDir) // Round up all metadata around building docker image - - // Retrieve docker metadata from file created by utils.Metadata() - data, err := os.ReadFile(workspaceDir + "/metadata.json") - if err != nil { - spinner.LogMessage("Can't open metadata file from docker image build: "+err.Error(), "fatal") - } - - // Decode json into interface - var dockerMetadata DockerMetadata - json.NewDecoder(strings.NewReader(string(data))).Decode(&dockerMetadata) - +func OutputDockerMetadata(buildID string, imageTags []string, path string) { // Get master git hash of current project _, masterGitHash := utils.GitMasterNameAndHash() - // Retrieve logs - // Store all docker metadata we want to output to metadata.json allDockerMetadata := AllDockerMetadata{ - DockerBuild: buildMetadata, - DockerImageTag: dockerMetadata, - ProjectName: buildMetadata.ProjectName, - ProjectType: buildMetadata.ProjectType, + ProjectName: os.Getenv("BUILDER_DIR_NAME"), UserName: utils.GetUserData().Username, - StartTime: buildMetadata.StartTime, - EndTime: buildMetadata.EndTime, + StartTime: os.Getenv("BUILD_START_TIME"), + EndTime: os.Getenv("BUILD_END_TIME"), GitURL: utils.GetRepoURL(), MasterGitHash: masterGitHash, - BuildID: buildMetadata.BuildID, + BuildID: buildID, DockerRepositoryTag: os.Getenv("BUILDER_DOCKER_REPO_TAG"), DockerRepository: os.Getenv("BUILDER_DOCKER_REPO"), DockerTags: imageTags, @@ -502,50 +307,17 @@ func OutputDockerMetadata(buildMetadata BuildMetadata, imageTags []string, path if yamlErr != nil { spinner.LogMessage("YAML Metadata creation unsuccessful: "+yamlErr.Error(), "fatal") } - - // Delete build metadata file grabbed from container to workspace dir - removeErr := os.Remove(workspaceDir + "/builder_builds.json") - if removeErr != nil { - spinner.LogMessage("Couldn't delete temporary log file in workspace: "+removeErr.Error(), "error") - } - - // Delete docker metadata file created by utils.Metadata() - removeErr2 := os.Remove(workspaceDir + "/metadata.json") - if removeErr2 != nil { - spinner.LogMessage("Couldn't delete temporary log file in workspace: "+removeErr2.Error(), "error") - } } -func SaveBuildLogs(buildLogsJSON []byte, path string) { - dockerLogsJSON, err := os.ReadFile(path + "/docker_logs.json") - - f, err := os.OpenFile(path+"/logs.json", os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666) - if err != nil { - spinner.LogMessage("Cannot create logs.json file: "+err.Error(), "fatal") - } +func GetDockerBuildID() string { + _, masterGitHash := utils.GitMasterNameAndHash() - if _, err = f.WriteString(string(buildLogsJSON)); err != nil { - spinner.LogMessage("unsuccessful write to logs.json file.", "fatal") - } - if _, err = f.WriteString("{\"level\":\"info\",\"timestamp\":\"\",\"caller\":\"cmd/docker.go:580\",\"msg\":\"======= Build End =======\"}\n"); err != nil { - spinner.LogMessage("unsuccessful write to logs.json file.", "fatal") - } - if _, err = f.WriteString("{\"level\":\"info\",\"timestamp\":\"\",\"caller\":\"cmd/docker.go:583\",\"msg\":\"====== Docker Start ======\"}\n"); err != nil { - spinner.LogMessage("unsuccessful write to logs.json file.", "fatal") - } - if _, err = f.WriteString(string(buildLogsJSON)); err != nil { - spinner.LogMessage("unsuccessful write to logs.json file.", "fatal") - } - if _, err = f.WriteString(string(dockerLogsJSON)); err != nil { - spinner.LogMessage("unsuccessful write to logs.json file.", "fatal") - } + data := os.Getenv("BUILDER_DIR_NAME") + utils.GetUserData().Username + os.Getenv("BUILD_START_TIME") + + os.Getenv("BUILD_END_TIME") + utils.GetRepoURL() + masterGitHash - f.Close() + sum := sha256.Sum256([]byte(data)) + checksum := fmt.Sprintf("%x", sum) - // Delete build logs file grabbed from container to workspace dir - workspaceDir := os.Getenv("BUILDER_WORKSPACE_DIR") - e := os.Remove(workspaceDir + "/builder_logs.json") - if e != nil { - spinner.LogMessage("Couldn't delete temporary log file in workspace: "+e.Error(), "error") - } + // Only return first 10 char of sum + return checksum[0:9] } diff --git a/directory/parent.go b/directory/parent.go index 1f31031..c05229e 100644 --- a/directory/parent.go +++ b/directory/parent.go @@ -86,8 +86,6 @@ func MakeDirs() { if os.Getenv("BUILDER_DOCKER_COMMAND") == "true" { MakeParentDir(path) - MakeWorkspaceDir(path) - MakeLogsDir(path) MakeBuilderDir() } else { diff --git a/utils/help.go b/utils/help.go index dc67162..2ea3a96 100644 --- a/utils/help.go +++ b/utils/help.go @@ -40,6 +40,8 @@ func Help() { - ex: builder init * builder config: build project w/ a builder.yaml (repo needed) - ex: builder config +* builder docker: build a Docker image given a Dockerfile and a builder.yaml file + - ex: builder docker * builder push [optional flags]: pushes build metadata and logs JSON to provided url (url needed in line or in builder.yaml) - optional flag: '--save' to automate "push" process for future builds - ex: builder push