From 3ed317abb071d8a5d29d370909d92b4191aa7591 Mon Sep 17 00:00:00 2001 From: Devashish Date: Tue, 5 Mar 2024 14:43:51 -0500 Subject: [PATCH 1/3] Packer tracks Version and Plugins Metadata (#12860) --- .../types.build.hcp_packer_registry_test.go | 3 ++ hcl2template/types.build_test.go | 34 ++++++++++------ hcl2template/types.datasource_test.go | 2 + hcl2template/types.packer_config.go | 1 + hcl2template/types.packer_config_test.go | 10 +++-- hcl2template/types.source_test.go | 1 + hcl2template/types.variables_test.go | 13 ++++-- internal/hcp/registry/hcl.go | 12 ++++++ internal/hcp/registry/json.go | 15 ++++++- packer/build.go | 40 +++++++++++++++++++ packer/plugin.go | 29 ++++++++++++++ 11 files changed, 139 insertions(+), 21 deletions(-) diff --git a/hcl2template/types.build.hcp_packer_registry_test.go b/hcl2template/types.build.hcp_packer_registry_test.go index b27b80e5b34..b43566d3eba 100644 --- a/hcl2template/types.build.hcp_packer_registry_test.go +++ b/hcl2template/types.build.hcp_packer_registry_test.go @@ -57,6 +57,7 @@ func Test_ParseHCPPackerRegistryBlock(t *testing.T) { Builder: emptyMockBuilder, Provisioners: []packer.CoreBuildProvisioner{}, PostProcessors: [][]packer.CoreBuildPostProcessor{}, + BuilderType: "virtualbox-iso", }, }, false, @@ -110,6 +111,7 @@ func Test_ParseHCPPackerRegistryBlock(t *testing.T) { Builder: emptyMockBuilder, Provisioners: []packer.CoreBuildProvisioner{}, PostProcessors: [][]packer.CoreBuildPostProcessor{}, + BuilderType: "virtualbox-iso", }, }, false, @@ -241,6 +243,7 @@ func Test_ParseHCPPackerRegistryBlock(t *testing.T) { Provisioners: []packer.CoreBuildProvisioner{}, PostProcessors: [][]packer.CoreBuildPostProcessor{}, Prepared: true, + BuilderType: "null", }, }, false, diff --git a/hcl2template/types.build_test.go b/hcl2template/types.build_test.go index 8647821dd18..d94dee4f82f 100644 --- a/hcl2template/types.build_test.go +++ b/hcl2template/types.build_test.go @@ -283,6 +283,7 @@ func TestParse_build(t *testing.T) { []packersdk.Build{ &packer.CoreBuild{ Type: "virtualbox-iso.ubuntu-1204", + BuilderType: "virtualbox-iso", Prepared: true, Builder: emptyMockBuilder, Provisioners: []packer.CoreBuildProvisioner{}, @@ -317,6 +318,7 @@ func TestParse_build(t *testing.T) { }, &packer.CoreBuild{ Type: "amazon-ebs.aws-ubuntu-16.04", + BuilderType: "amazon-ebs", Prepared: true, Builder: emptyMockBuilder, Provisioners: []packer.CoreBuildProvisioner{}, @@ -397,9 +399,10 @@ func TestParse_build(t *testing.T) { false, false, []packersdk.Build{ &packer.CoreBuild{ - Type: "virtualbox-iso.ubuntu-1204", - Prepared: true, - Builder: emptyMockBuilder, + Type: "virtualbox-iso.ubuntu-1204", + BuilderType: "virtualbox-iso", + Prepared: true, + Builder: emptyMockBuilder, Provisioners: []packer.CoreBuildProvisioner{ { PType: "shell", @@ -427,9 +430,10 @@ func TestParse_build(t *testing.T) { PostProcessors: [][]packer.CoreBuildPostProcessor{}, }, &packer.CoreBuild{ - Type: "amazon-ebs.aws-ubuntu-16.04", - Prepared: true, - Builder: emptyMockBuilder, + Type: "amazon-ebs.aws-ubuntu-16.04", + BuilderType: "amazon-ebs", + Prepared: true, + Builder: emptyMockBuilder, Provisioners: []packer.CoreBuildProvisioner{ { PType: "file", @@ -486,9 +490,10 @@ func TestParse_build(t *testing.T) { false, false, []packersdk.Build{ &packer.CoreBuild{ - Type: "virtualbox-iso.ubuntu-1204", - Prepared: true, - Builder: emptyMockBuilder, + Type: "virtualbox-iso.ubuntu-1204", + BuilderType: "virtualbox-iso", + Prepared: true, + Builder: emptyMockBuilder, Provisioners: []packer.CoreBuildProvisioner{ { PType: "shell", @@ -550,6 +555,7 @@ func TestParse_build(t *testing.T) { &packer.CoreBuild{ BuildName: "build-name", Type: "virtualbox-iso.ubuntu-1204", + BuilderType: "virtualbox-iso", Prepared: true, Builder: emptyMockBuilder, Provisioners: []packer.CoreBuildProvisioner{}, @@ -604,6 +610,7 @@ func TestParse_build(t *testing.T) { &packer.CoreBuild{ BuildName: "test-build", Type: "virtualbox-iso.ubuntu-1204", + BuilderType: "virtualbox-iso", Prepared: true, Builder: emptyMockBuilder, Provisioners: []packer.CoreBuildProvisioner{}, @@ -659,10 +666,11 @@ func TestParse_build(t *testing.T) { false, false, []packersdk.Build{ &packer.CoreBuild{ - BuildName: "build-name-test", - Type: "virtualbox-iso.ubuntu-1204", - Prepared: true, - Builder: emptyMockBuilder, + BuildName: "build-name-test", + Type: "virtualbox-iso.ubuntu-1204", + BuilderType: "virtualbox-iso", + Prepared: true, + Builder: emptyMockBuilder, Provisioners: []packer.CoreBuildProvisioner{ { PName: "build-name-test", diff --git a/hcl2template/types.datasource_test.go b/hcl2template/types.datasource_test.go index d12fcafd076..69f6541389c 100644 --- a/hcl2template/types.datasource_test.go +++ b/hcl2template/types.datasource_test.go @@ -57,6 +57,7 @@ func TestParse_datasource(t *testing.T) { []packersdk.Build{ &packer.CoreBuild{ Type: "null.test", + BuilderType: "null", Builder: &null.Builder{}, Provisioners: []packer.CoreBuildProvisioner{}, PostProcessors: [][]packer.CoreBuildPostProcessor{}, @@ -134,6 +135,7 @@ func TestParse_datasource(t *testing.T) { []packersdk.Build{ &packer.CoreBuild{ Type: "null.test", + BuilderType: "null", Builder: &null.Builder{}, Provisioners: []packer.CoreBuildProvisioner{}, PostProcessors: [][]packer.CoreBuildPostProcessor{}, diff --git a/hcl2template/types.packer_config.go b/hcl2template/types.packer_config.go index 3ead0b6b5f1..d21d10edeae 100644 --- a/hcl2template/types.packer_config.go +++ b/hcl2template/types.packer_config.go @@ -622,6 +622,7 @@ func (cfg *PackerConfig) GetBuilds(opts packer.GetBuildsOptions) ([]packersdk.Bu decoded, _ := decodeHCL2Spec(srcUsage.Body, cfg.EvalContext(BuildContext, nil), builder) pcb.HCLConfig = decoded + pcb.BuilderType = srcUsage.Type // If the builder has provided a list of to-be-generated variables that // should be made accessible to provisioners, pass that list into diff --git a/hcl2template/types.packer_config_test.go b/hcl2template/types.packer_config_test.go index 69bc2da2c1c..391899e9234 100644 --- a/hcl2template/types.packer_config_test.go +++ b/hcl2template/types.packer_config_test.go @@ -208,8 +208,9 @@ func TestParser_complete(t *testing.T) { false, false, []packersdk.Build{ &packer.CoreBuild{ - Type: "virtualbox-iso.ubuntu-1204", - Prepared: true, + Type: "virtualbox-iso.ubuntu-1204", + BuilderType: "virtualbox-iso", + Prepared: true, Builder: &MockBuilder{ Config: MockConfig{ NestedMockConfig: NestedMockConfig{ @@ -319,8 +320,9 @@ func TestParser_complete(t *testing.T) { }, }, &packer.CoreBuild{ - Type: "amazon-ebs.ubuntu-1604", - Prepared: true, + Type: "amazon-ebs.ubuntu-1604", + BuilderType: "amazon-ebs", + Prepared: true, Builder: &MockBuilder{ Config: MockConfig{ NestedMockConfig: NestedMockConfig{ diff --git a/hcl2template/types.source_test.go b/hcl2template/types.source_test.go index 47caeec5a75..e319163682d 100644 --- a/hcl2template/types.source_test.go +++ b/hcl2template/types.source_test.go @@ -55,6 +55,7 @@ func TestParse_source(t *testing.T) { []packersdk.Build{ &packer.CoreBuild{ Type: "null.test", + BuilderType: "null", Builder: &null.Builder{}, Provisioners: []packer.CoreBuildProvisioner{}, PostProcessors: [][]packer.CoreBuildPostProcessor{}, diff --git a/hcl2template/types.variables_test.go b/hcl2template/types.variables_test.go index a19b80bd990..ecff3677ca2 100644 --- a/hcl2template/types.variables_test.go +++ b/hcl2template/types.variables_test.go @@ -129,6 +129,7 @@ func TestParse_variables(t *testing.T) { []packersdk.Build{ &packer.CoreBuild{ Type: "null.test", + BuilderType: "null", Builder: &null.Builder{}, Provisioners: []packer.CoreBuildProvisioner{}, PostProcessors: [][]packer.CoreBuildPostProcessor{}, @@ -293,6 +294,7 @@ func TestParse_variables(t *testing.T) { []packersdk.Build{ &packer.CoreBuild{ Type: "null", + BuilderType: "null", Builder: &null.Builder{}, Provisioners: []packer.CoreBuildProvisioner{}, PostProcessors: [][]packer.CoreBuildPostProcessor{}, @@ -379,6 +381,7 @@ func TestParse_variables(t *testing.T) { []packersdk.Build{ &packer.CoreBuild{ Type: "null.test", + BuilderType: "null", Builder: &null.Builder{}, Provisioners: []packer.CoreBuildProvisioner{}, PostProcessors: [][]packer.CoreBuildPostProcessor{}, @@ -442,6 +445,7 @@ func TestParse_variables(t *testing.T) { []packersdk.Build{ &packer.CoreBuild{ Type: "null.test", + BuilderType: "null", Builder: &null.Builder{}, Provisioners: []packer.CoreBuildProvisioner{}, PostProcessors: [][]packer.CoreBuildPostProcessor{}, @@ -483,6 +487,7 @@ func TestParse_variables(t *testing.T) { []packersdk.Build{ &packer.CoreBuild{ Type: "null.test", + BuilderType: "null", Builder: &null.Builder{}, Provisioners: []packer.CoreBuildProvisioner{}, PostProcessors: [][]packer.CoreBuildPostProcessor{}, @@ -538,9 +543,10 @@ func TestParse_variables(t *testing.T) { }, false, false, []packersdk.Build{&packer.CoreBuild{ - Type: "null.null-builder", - Prepared: true, - Builder: &null.Builder{}, + Type: "null.null-builder", + BuilderType: "null", + Prepared: true, + Builder: &null.Builder{}, Provisioners: []packer.CoreBuildProvisioner{ { PType: "shell", @@ -627,6 +633,7 @@ func TestParse_variables(t *testing.T) { []packersdk.Build{ &packer.CoreBuild{ Type: "null.test", + BuilderType: "null", Builder: &null.Builder{}, Provisioners: []packer.CoreBuildProvisioner{}, PostProcessors: [][]packer.CoreBuildPostProcessor{}, diff --git a/internal/hcp/registry/hcl.go b/internal/hcp/registry/hcl.go index 23c5af7ccd9..ce6fd68eaa6 100644 --- a/internal/hcp/registry/hcl.go +++ b/internal/hcp/registry/hcl.go @@ -85,6 +85,18 @@ func (h *HCLRegistry) CompleteBuild( if ok { name = cb.Type } + + metadata := cb.GetMetadata() + log.Printf( + "[TRACE] HCL 'Packer Version' Metadata for build name %q: %q\n", + name, metadata.PackerVersion, + ) + for k, pluginDetails := range metadata.Plugins { + log.Printf( + "[TRACE] HCL 'Plugin' Metadata for build name %q: %q -- %q\n", + name, k, pluginDetails.Description.Version, + ) + } return h.bucket.completeBuild(ctx, name, artifacts, buildErr) } diff --git a/internal/hcp/registry/json.go b/internal/hcp/registry/json.go index b35f2ea49ed..b86cfd45273 100644 --- a/internal/hcp/registry/json.go +++ b/internal/hcp/registry/json.go @@ -93,7 +93,20 @@ func (h *JSONRegistry) CompleteBuild( artifacts []sdkpacker.Artifact, buildErr error, ) ([]sdkpacker.Artifact, error) { - return h.bucket.completeBuild(ctx, build.Name(), artifacts, buildErr) + name := build.Name() + + metadata := build.(*packer.CoreBuild).GetMetadata() + log.Printf( + "[TRACE] JSON 'Packer Version' Metadata for build name %q: %q\n", + name, metadata.PackerVersion, + ) + for k, pluginDetails := range metadata.Plugins { + log.Printf( + "[TRACE] JSON 'Plugin' Metadata for build name %q: %q -- %q\n", + name, k, pluginDetails.Description.Version, + ) + } + return h.bucket.completeBuild(ctx, name, artifacts, buildErr) } // VersionStatusSummary prints a status report in the UI if the version is not yet done diff --git a/packer/build.go b/packer/build.go index e51e71d00c9..9dca9610ac8 100644 --- a/packer/build.go +++ b/packer/build.go @@ -52,6 +52,46 @@ type CoreBuild struct { prepareCalled bool } +type BuildMetadata struct { + PackerVersion string + Plugins map[string]PluginDetails +} + +func (b *CoreBuild) getPluginsMetadata() map[string]PluginDetails { + resp := map[string]PluginDetails{} + + builderPlugin, builderPluginOk := PluginsDetailsStorage[fmt.Sprintf("%q-%q", PluginComponentBuilder, b.BuilderType)] + if builderPluginOk { + resp[builderPlugin.Name] = builderPlugin + } + + for _, pp := range b.PostProcessors { + for _, p := range pp { + postprocessorsPlugin, postprocessorsPluginOk := PluginsDetailsStorage[fmt.Sprintf("%q-%q", PluginComponentPostProcessor, p.PType)] + if postprocessorsPluginOk { + resp[postprocessorsPlugin.Name] = postprocessorsPlugin + } + } + } + + for _, pv := range b.Provisioners { + provisionerPlugin, provisionerPluginOk := PluginsDetailsStorage[fmt.Sprintf("%q-%q", PluginComponentProvisioner, pv.PType)] + if provisionerPluginOk { + resp[provisionerPlugin.Name] = provisionerPlugin + } + } + + return resp +} + +func (b *CoreBuild) GetMetadata() BuildMetadata { + metadata := BuildMetadata{ + PackerVersion: version.FormattedVersion(), + Plugins: b.getPluginsMetadata(), + } + return metadata +} + // CoreBuildPostProcessor Keeps track of the post-processor and the // configuration of the post-processor used within a build. type CoreBuildPostProcessor struct { diff --git a/packer/plugin.go b/packer/plugin.go index 30261d7e05c..513694f639f 100644 --- a/packer/plugin.go +++ b/packer/plugin.go @@ -6,6 +6,7 @@ package packer import ( "crypto/sha256" "encoding/json" + "fmt" "log" "os" "os/exec" @@ -135,6 +136,11 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error } pluginPrefix := pluginName + "-" + pluginDetails := PluginDetails{ + Name: pluginName, + Description: desc, + PluginPath: pluginPath, + } for _, builderName := range desc.Builders { builderName := builderName // copy to avoid pointer overwrite issue @@ -145,6 +151,8 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error c.Builders.Set(key, func() (packersdk.Builder, error) { return c.Client(pluginPath, "start", "builder", builderName).Builder() }) + PluginsDetailsStorage[fmt.Sprintf("%q-%q", PluginComponentBuilder, key)] = pluginDetails + } if len(desc.Builders) > 0 { @@ -160,6 +168,7 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error c.PostProcessors.Set(key, func() (packersdk.PostProcessor, error) { return c.Client(pluginPath, "start", "post-processor", postProcessorName).PostProcessor() }) + PluginsDetailsStorage[fmt.Sprintf("%q-%q", PluginComponentPostProcessor, key)] = pluginDetails } if len(desc.PostProcessors) > 0 { @@ -175,6 +184,8 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error c.Provisioners.Set(key, func() (packersdk.Provisioner, error) { return c.Client(pluginPath, "start", "provisioner", provisionerName).Provisioner() }) + PluginsDetailsStorage[fmt.Sprintf("%q-%q", PluginComponentProvisioner, key)] = pluginDetails + } if len(desc.Provisioners) > 0 { log.Printf("found external %v provisioner from %s plugin", desc.Provisioners, pluginName) @@ -189,6 +200,7 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error c.DataSources.Set(key, func() (packersdk.Datasource, error) { return c.Client(pluginPath, "start", "datasource", datasourceName).Datasource() }) + PluginsDetailsStorage[fmt.Sprintf("%q-%q", PluginComponentDataSource, key)] = pluginDetails } if len(desc.Datasources) > 0 { log.Printf("found external %v datasource from %s plugin", desc.Datasources, pluginName) @@ -240,3 +252,20 @@ func (c *PluginConfig) Client(path string, args ...string) *PluginClient { config.MaxPort = c.PluginMaxPort return NewClient(&config) } + +type PluginComponentType string + +const ( + PluginComponentBuilder PluginComponentType = "builder" + PluginComponentPostProcessor PluginComponentType = "post-processor" + PluginComponentProvisioner PluginComponentType = "provisioner" + PluginComponentDataSource PluginComponentType = "data-source" +) + +type PluginDetails struct { + Name string + Description pluginsdk.SetDescription + PluginPath string +} + +var PluginsDetailsStorage = map[string]PluginDetails{} From 2e35c44bad04cfcee0db4cf8a4d12adfd69dcdce Mon Sep 17 00:00:00 2001 From: Devashish Date: Tue, 26 Mar 2024 10:58:48 -0400 Subject: [PATCH 2/3] HPR-1537: Packer Core sends metadata to HCP Packer in UpdateBuild API (#12891) --- go.mod | 2 ++ go.sum | 6 +++--- internal/hcp/api/service_build.go | 2 ++ internal/hcp/registry/hcl.go | 10 +++++++++ internal/hcp/registry/json.go | 14 ++++++++++++- internal/hcp/registry/types.bucket.go | 30 +++++++++++++++++++++++++++ internal/hcp/registry/types.builds.go | 1 + 7 files changed, 61 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 596b48a9e6f..2c89433899b 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,7 @@ module github.com/hashicorp/packer +replace github.com/hashicorp/hcp-sdk-go => github.com/hashicorp/hcp-sdk-go-internal v0.0.0-20240325194734-0bb6bd5bbc19 + require ( cloud.google.com/go v0.110.8 // indirect github.com/biogo/hts v1.4.3 diff --git a/go.sum b/go.sum index 006aa84bf8c..1bc75fa168c 100644 --- a/go.sum +++ b/go.sum @@ -313,8 +313,8 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl/v2 v2.19.1 h1://i05Jqznmb2EXqa39Nsvyan2o5XyMowW5fnCKW5RPI= github.com/hashicorp/hcl/v2 v2.19.1/go.mod h1:ThLC89FV4p9MPW804KVbe/cEXoQ8NZEh+JtMeeGErHE= -github.com/hashicorp/hcp-sdk-go v0.90.0 h1:dYN7PB9bBYKamjMHPBQnK6d8zphyexHGXw4ddynK2ck= -github.com/hashicorp/hcp-sdk-go v0.90.0/go.mod h1:vQ4fzdL1AmhIAbCw+4zmFe5Hbpajj3NvRWkJoVuxmAk= +github.com/hashicorp/hcp-sdk-go-internal v0.0.0-20240325194734-0bb6bd5bbc19 h1:QFacHWwhQbVqA9p/nX5CC7wPVQUuT8T2pN7TKI15BQw= +github.com/hashicorp/hcp-sdk-go-internal v0.0.0-20240325194734-0bb6bd5bbc19/go.mod h1:vsLMHasv6H3Qf7wIlRSzHXIOp81G+ad1FP31JfdTKYc= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= github.com/hashicorp/memberlist v0.5.0 h1:EtYPN8DpAURiapus508I4n9CzHs2W+8NZGbmmR/prTM= @@ -784,8 +784,8 @@ gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/hcp/api/service_build.go b/internal/hcp/api/service_build.go index 69ce13b2b34..946d8d08e42 100644 --- a/internal/hcp/api/service_build.go +++ b/internal/hcp/api/service_build.go @@ -58,6 +58,7 @@ func (c *Client) UpdateBuild( buildLabels map[string]string, buildStatus hcpPackerModels.HashicorpCloudPacker20230101BuildStatus, artifacts []*hcpPackerModels.HashicorpCloudPacker20230101ArtifactCreateBody, + metadata *hcpPackerModels.HashicorpCloudPacker20230101BuildMetadata, ) (string, error) { params := hcpPackerAPI.NewPackerServiceUpdateBuildParamsWithContext(ctx) @@ -76,6 +77,7 @@ func (c *Client) UpdateBuild( Platform: platform, SourceExternalIdentifier: sourceExternalIdentifier, Status: &buildStatus, + Metadata: metadata, } resp, err := c.Packer.PackerServiceUpdateBuild(params, nil) diff --git a/internal/hcp/registry/hcl.go b/internal/hcp/registry/hcl.go index ce6fd68eaa6..332c19df953 100644 --- a/internal/hcp/registry/hcl.go +++ b/internal/hcp/registry/hcl.go @@ -70,6 +70,12 @@ func (h *HCLRegistry) StartBuild(ctx context.Context, build sdkpacker.Build) err if ok { name = cb.Type } + + metadata := cb.GetMetadata() + err := h.bucket.AddMetadataToBuild(ctx, name, metadata) + if err != nil { + return err + } return h.bucket.startBuild(ctx, name) } @@ -97,6 +103,10 @@ func (h *HCLRegistry) CompleteBuild( name, k, pluginDetails.Description.Version, ) } + err := h.bucket.AddMetadataToBuild(ctx, name, metadata) + if err != nil { + return nil, err + } return h.bucket.completeBuild(ctx, name, artifacts, buildErr) } diff --git a/internal/hcp/registry/json.go b/internal/hcp/registry/json.go index b86cfd45273..785ccc4a92f 100644 --- a/internal/hcp/registry/json.go +++ b/internal/hcp/registry/json.go @@ -83,7 +83,14 @@ func (h *JSONRegistry) PopulateVersion(ctx context.Context) error { // StartBuild is invoked when one build for the configuration is starting to be processed func (h *JSONRegistry) StartBuild(ctx context.Context, build sdkpacker.Build) error { - return h.bucket.startBuild(ctx, build.Name()) + name := build.Name() + + metadata := build.(*packer.CoreBuild).GetMetadata() + err := h.bucket.AddMetadataToBuild(ctx, name, metadata) + if err != nil { + return err + } + return h.bucket.startBuild(ctx, name) } // CompleteBuild is invoked when one build for the configuration has finished @@ -106,6 +113,11 @@ func (h *JSONRegistry) CompleteBuild( name, k, pluginDetails.Description.Version, ) } + + err := h.bucket.AddMetadataToBuild(ctx, name, metadata) + if err != nil { + return nil, err + } return h.bucket.completeBuild(ctx, name, artifacts, buildErr) } diff --git a/internal/hcp/registry/types.bucket.go b/internal/hcp/registry/types.bucket.go index 6eda3b484bf..f0a7c4b4353 100644 --- a/internal/hcp/registry/types.bucket.go +++ b/internal/hcp/registry/types.bucket.go @@ -19,6 +19,7 @@ import ( "github.com/hashicorp/packer/hcl2template" hcpPackerAPI "github.com/hashicorp/packer/internal/hcp/api" "github.com/hashicorp/packer/internal/hcp/env" + "github.com/hashicorp/packer/packer" "github.com/mitchellh/mapstructure" "google.golang.org/grpc/codes" ) @@ -213,6 +214,7 @@ func (bucket *Bucket) UpdateBuildStatus( nil, status, nil, + &buildToUpdate.Metadata, ) if err != nil { return err @@ -285,6 +287,7 @@ func (bucket *Bucket) markBuildComplete(ctx context.Context, name string) error buildToUpdate.Labels, status, artifacts, + &buildToUpdate.Metadata, ) if err != nil { return err @@ -535,6 +538,7 @@ func (bucket *Bucket) HeartbeatBuild(ctx context.Context, build string) (func(), nil, hcpPackerModels.HashicorpCloudPacker20230101BuildStatusBUILDRUNNING, nil, + nil, ) if err != nil { log.Printf("[ERROR] failed to send heartbeat for build %q: %s", build, err) @@ -599,6 +603,32 @@ type NotAHCPArtifactError struct { error } +// AddMetadataToBuild adds metadata to a build in the HCP Packer registry. +func (bucket *Bucket) AddMetadataToBuild( + ctx context.Context, buildName string, metadata packer.BuildMetadata, +) error { + buildToUpdate, err := bucket.Version.Build(buildName) + if err != nil { + return err + } + + packerMetadata := make(map[string]interface{}) + packerMetadata["version"] = metadata.PackerVersion + + var pluginsMetadata []map[string]interface{} + for _, plugin := range metadata.Plugins { + pluginMetadata := map[string]interface{}{ + "version": plugin.Description.Version, + "name": plugin.Name, + } + pluginsMetadata = append(pluginsMetadata, pluginMetadata) + } + packerMetadata["plugins"] = pluginsMetadata + + buildToUpdate.Metadata.Packer = packerMetadata + return nil +} + func (bucket *Bucket) completeBuild( ctx context.Context, buildName string, diff --git a/internal/hcp/registry/types.builds.go b/internal/hcp/registry/types.builds.go index a55e9b0b9f6..dc7e132762c 100644 --- a/internal/hcp/registry/types.builds.go +++ b/internal/hcp/registry/types.builds.go @@ -19,6 +19,7 @@ type Build struct { Labels map[string]string Artifacts map[string]packerSDKRegistry.Image Status hcpPackerModels.HashicorpCloudPacker20230101BuildStatus + Metadata hcpPackerModels.HashicorpCloudPacker20230101BuildMetadata } // NewBuildFromCloudPackerBuild converts a HashicorpCloudPackerBuild to a local build that can be tracked and From 9ba3547d08e2487ba0fd9396789cd43566540348 Mon Sep 17 00:00:00 2001 From: Devashish Date: Wed, 27 Mar 2024 15:23:06 -0400 Subject: [PATCH 3/3] Use latest version of hcp-sdk-go Remove loggers and move AddMetadataToBuild method to Version --- go.mod | 2 -- go.sum | 6 +++--- internal/hcp/registry/hcl.go | 22 ++++++-------------- internal/hcp/registry/json.go | 22 +++++--------------- internal/hcp/registry/types.bucket.go | 27 ------------------------- internal/hcp/registry/types.version.go | 28 ++++++++++++++++++++++++++ 6 files changed, 42 insertions(+), 65 deletions(-) diff --git a/go.mod b/go.mod index 2c89433899b..596b48a9e6f 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,5 @@ module github.com/hashicorp/packer -replace github.com/hashicorp/hcp-sdk-go => github.com/hashicorp/hcp-sdk-go-internal v0.0.0-20240325194734-0bb6bd5bbc19 - require ( cloud.google.com/go v0.110.8 // indirect github.com/biogo/hts v1.4.3 diff --git a/go.sum b/go.sum index 1bc75fa168c..006aa84bf8c 100644 --- a/go.sum +++ b/go.sum @@ -313,8 +313,8 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl/v2 v2.19.1 h1://i05Jqznmb2EXqa39Nsvyan2o5XyMowW5fnCKW5RPI= github.com/hashicorp/hcl/v2 v2.19.1/go.mod h1:ThLC89FV4p9MPW804KVbe/cEXoQ8NZEh+JtMeeGErHE= -github.com/hashicorp/hcp-sdk-go-internal v0.0.0-20240325194734-0bb6bd5bbc19 h1:QFacHWwhQbVqA9p/nX5CC7wPVQUuT8T2pN7TKI15BQw= -github.com/hashicorp/hcp-sdk-go-internal v0.0.0-20240325194734-0bb6bd5bbc19/go.mod h1:vsLMHasv6H3Qf7wIlRSzHXIOp81G+ad1FP31JfdTKYc= +github.com/hashicorp/hcp-sdk-go v0.90.0 h1:dYN7PB9bBYKamjMHPBQnK6d8zphyexHGXw4ddynK2ck= +github.com/hashicorp/hcp-sdk-go v0.90.0/go.mod h1:vQ4fzdL1AmhIAbCw+4zmFe5Hbpajj3NvRWkJoVuxmAk= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= github.com/hashicorp/memberlist v0.5.0 h1:EtYPN8DpAURiapus508I4n9CzHs2W+8NZGbmmR/prTM= @@ -784,8 +784,8 @@ gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/hcp/registry/hcl.go b/internal/hcp/registry/hcl.go index 332c19df953..897e2afc0ec 100644 --- a/internal/hcp/registry/hcl.go +++ b/internal/hcp/registry/hcl.go @@ -72,7 +72,7 @@ func (h *HCLRegistry) StartBuild(ctx context.Context, build sdkpacker.Build) err } metadata := cb.GetMetadata() - err := h.bucket.AddMetadataToBuild(ctx, name, metadata) + err := h.bucket.Version.AddMetadataToBuild(ctx, name, metadata) if err != nil { return err } @@ -86,28 +86,18 @@ func (h *HCLRegistry) CompleteBuild( artifacts []sdkpacker.Artifact, buildErr error, ) ([]sdkpacker.Artifact, error) { - name := build.Name() + buildName := build.Name() cb, ok := build.(*packer.CoreBuild) if ok { - name = cb.Type + buildName = cb.Type } - metadata := cb.GetMetadata() - log.Printf( - "[TRACE] HCL 'Packer Version' Metadata for build name %q: %q\n", - name, metadata.PackerVersion, - ) - for k, pluginDetails := range metadata.Plugins { - log.Printf( - "[TRACE] HCL 'Plugin' Metadata for build name %q: %q -- %q\n", - name, k, pluginDetails.Description.Version, - ) - } - err := h.bucket.AddMetadataToBuild(ctx, name, metadata) + buildMetadata := cb.GetMetadata() + err := h.bucket.Version.AddMetadataToBuild(ctx, buildName, buildMetadata) if err != nil { return nil, err } - return h.bucket.completeBuild(ctx, name, artifacts, buildErr) + return h.bucket.completeBuild(ctx, buildName, artifacts, buildErr) } // VersionStatusSummary prints a status report in the UI if the version is not yet done diff --git a/internal/hcp/registry/json.go b/internal/hcp/registry/json.go index 785ccc4a92f..407ea40a062 100644 --- a/internal/hcp/registry/json.go +++ b/internal/hcp/registry/json.go @@ -86,7 +86,7 @@ func (h *JSONRegistry) StartBuild(ctx context.Context, build sdkpacker.Build) er name := build.Name() metadata := build.(*packer.CoreBuild).GetMetadata() - err := h.bucket.AddMetadataToBuild(ctx, name, metadata) + err := h.bucket.Version.AddMetadataToBuild(ctx, name, metadata) if err != nil { return err } @@ -100,25 +100,13 @@ func (h *JSONRegistry) CompleteBuild( artifacts []sdkpacker.Artifact, buildErr error, ) ([]sdkpacker.Artifact, error) { - name := build.Name() - - metadata := build.(*packer.CoreBuild).GetMetadata() - log.Printf( - "[TRACE] JSON 'Packer Version' Metadata for build name %q: %q\n", - name, metadata.PackerVersion, - ) - for k, pluginDetails := range metadata.Plugins { - log.Printf( - "[TRACE] JSON 'Plugin' Metadata for build name %q: %q -- %q\n", - name, k, pluginDetails.Description.Version, - ) - } - - err := h.bucket.AddMetadataToBuild(ctx, name, metadata) + buildName := build.Name() + buildMetadata := build.(*packer.CoreBuild).GetMetadata() + err := h.bucket.Version.AddMetadataToBuild(ctx, buildName, buildMetadata) if err != nil { return nil, err } - return h.bucket.completeBuild(ctx, name, artifacts, buildErr) + return h.bucket.completeBuild(ctx, buildName, artifacts, buildErr) } // VersionStatusSummary prints a status report in the UI if the version is not yet done diff --git a/internal/hcp/registry/types.bucket.go b/internal/hcp/registry/types.bucket.go index f0a7c4b4353..1fe9dd3b2b3 100644 --- a/internal/hcp/registry/types.bucket.go +++ b/internal/hcp/registry/types.bucket.go @@ -19,7 +19,6 @@ import ( "github.com/hashicorp/packer/hcl2template" hcpPackerAPI "github.com/hashicorp/packer/internal/hcp/api" "github.com/hashicorp/packer/internal/hcp/env" - "github.com/hashicorp/packer/packer" "github.com/mitchellh/mapstructure" "google.golang.org/grpc/codes" ) @@ -603,32 +602,6 @@ type NotAHCPArtifactError struct { error } -// AddMetadataToBuild adds metadata to a build in the HCP Packer registry. -func (bucket *Bucket) AddMetadataToBuild( - ctx context.Context, buildName string, metadata packer.BuildMetadata, -) error { - buildToUpdate, err := bucket.Version.Build(buildName) - if err != nil { - return err - } - - packerMetadata := make(map[string]interface{}) - packerMetadata["version"] = metadata.PackerVersion - - var pluginsMetadata []map[string]interface{} - for _, plugin := range metadata.Plugins { - pluginMetadata := map[string]interface{}{ - "version": plugin.Description.Version, - "name": plugin.Name, - } - pluginsMetadata = append(pluginsMetadata, pluginMetadata) - } - packerMetadata["plugins"] = pluginsMetadata - - buildToUpdate.Metadata.Packer = packerMetadata - return nil -} - func (bucket *Bucket) completeBuild( ctx context.Context, buildName string, diff --git a/internal/hcp/registry/types.version.go b/internal/hcp/registry/types.version.go index ae8edb86931..91d81749702 100644 --- a/internal/hcp/registry/types.version.go +++ b/internal/hcp/registry/types.version.go @@ -4,6 +4,7 @@ package registry import ( + "context" "errors" "fmt" "math/rand" @@ -16,6 +17,7 @@ import ( sdkpacker "github.com/hashicorp/packer-plugin-sdk/packer" packerSDKRegistry "github.com/hashicorp/packer-plugin-sdk/packer/registry/image" "github.com/hashicorp/packer/internal/hcp/env" + "github.com/hashicorp/packer/packer" "github.com/oklog/ulid" ) @@ -174,3 +176,29 @@ func (version *Version) statusSummary(ui sdkpacker.Ui) { ui.Say(buf.String()) } + +// AddMetadataToBuild adds metadata to a build in the HCP Packer registry. +func (version *Version) AddMetadataToBuild( + ctx context.Context, buildName string, metadata packer.BuildMetadata, +) error { + buildToUpdate, err := version.Build(buildName) + if err != nil { + return err + } + + packerMetadata := make(map[string]interface{}) + packerMetadata["version"] = metadata.PackerVersion + + var pluginsMetadata []map[string]interface{} + for _, plugin := range metadata.Plugins { + pluginMetadata := map[string]interface{}{ + "version": plugin.Description.Version, + "name": plugin.Name, + } + pluginsMetadata = append(pluginsMetadata, pluginMetadata) + } + packerMetadata["plugins"] = pluginsMetadata + + buildToUpdate.Metadata.Packer = packerMetadata + return nil +}