From c1c1577073557b25b8084b243805223de4ae0a6c Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Thu, 24 Aug 2023 14:44:05 -0400 Subject: [PATCH] hcl2template: rework parsing logic In previous versions of Packer, the parser would load a HCL file (in either HCL or JSON format), and process it in phases. This process meant that if the file was not valid in the first place, we'd have some duplicate errors first. Another thing is that sources and build blocks were not being parsed at the beginning of the parsing phase at all, but were discovered later in the process, when the data sources have been executed and variables have been evaluated. This commit changes the parsing to happen all at once, so we parse the files with the parser, which will fail should one file be malformatted, then we visit the body once using the top-level schema for a Packer build template, handling misplaced blocks once in the process. Then, since the builds and sources may contain some dynamic blocks that need to be expanded once their context is ready (i.e. the variables/datasources they depend on), we can expand the build block for them, and populate their respective objects. This last step should also be adapted for datasources later, as they may need to support dynamic blocks, and be expanded lazily. --- hcl2template/common_test.go | 3 + hcl2template/parser.go | 396 +++++++++++++----- hcl2template/plugin.go | 16 + hcl2template/types.build.go | 83 ++-- .../types.build.hcp_packer_registry.go | 2 +- .../types.build.hcp_packer_registry_test.go | 10 + hcl2template/types.build.post-processor.go | 2 +- hcl2template/types.build.provisioners.go | 2 +- hcl2template/types.build.provisioners_test.go | 2 +- hcl2template/types.build_test.go | 28 +- hcl2template/types.datasource.go | 29 -- hcl2template/types.packer_config.go | 49 +-- hcl2template/types.packer_config_test.go | 15 +- hcl2template/types.required_plugins.go | 31 -- hcl2template/types.required_plugins_test.go | 2 +- hcl2template/types.source.go | 39 +- hcl2template/types.source_test.go | 6 +- hcl2template/types.variables.go | 29 -- hcl2template/types.variables_test.go | 5 +- 19 files changed, 438 insertions(+), 311 deletions(-) diff --git a/hcl2template/common_test.go b/hcl2template/common_test.go index 831ce5d3596..08c6ae647eb 100644 --- a/hcl2template/common_test.go +++ b/hcl2template/common_test.go @@ -12,6 +12,7 @@ import ( "github.com/hashicorp/go-version" "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hclparse" + "github.com/hashicorp/hcl/v2/hclsyntax" packersdk "github.com/hashicorp/packer-plugin-sdk/packer" "github.com/hashicorp/packer-plugin-sdk/template/config" "github.com/hashicorp/packer/builder/null" @@ -348,6 +349,7 @@ var cmpOpts = []cmp.Option{ cmpopts.IgnoreUnexported( PackerConfig{}, Variable{}, + BuildBlock{}, SourceBlock{}, DatasourceBlock{}, ProvisionerBlock{}, @@ -376,6 +378,7 @@ var cmpOpts = []cmp.Option{ cmpopts.IgnoreFields(packer.CoreBuildPostProcessor{}, "HCLConfig", ), + cmpopts.IgnoreTypes(hclsyntax.Body{}), cmpopts.IgnoreTypes(hcl2template.MockBuilder{}), cmpopts.IgnoreTypes(HCL2Ref{}), cmpopts.IgnoreTypes([]*LocalBlock{}), diff --git a/hcl2template/parser.go b/hcl2template/parser.go index c372a7733a9..2481f0a0304 100644 --- a/hcl2template/parser.go +++ b/hcl2template/parser.go @@ -11,10 +11,14 @@ import ( "github.com/hashicorp/go-version" "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/ext/dynblock" + "github.com/hashicorp/hcl/v2/gohcl" "github.com/hashicorp/hcl/v2/hclparse" + "github.com/hashicorp/hcl/v2/hclsyntax" packersdk "github.com/hashicorp/packer-plugin-sdk/packer" + pkrfunction "github.com/hashicorp/packer/hcl2template/function" "github.com/hashicorp/packer/packer" "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/function" ) const ( @@ -163,33 +167,8 @@ func (p *Parser) Parse(filename string, varFiles []string, argVars map[string]st return cfg, diags } - // Decode required_plugins blocks. - // - // Note: using `latest` ( or actually an empty string ) in a config file - // does not work and packer will ask you to pick a version - { - for _, file := range files { - diags = append(diags, cfg.decodeRequiredPluginsBlock(file)...) - } - } - - // Decode variable blocks so that they are available later on. Here locals - // can use input variables so we decode input variables first. - { - for _, file := range files { - diags = append(diags, cfg.decodeInputVariables(file)...) - } - - for _, file := range files { - morediags := p.decodeDatasources(file, cfg) - diags = append(diags, morediags...) - } - - for _, file := range files { - moreLocals, morediags := parseLocalVariableBlocks(file) - diags = append(diags, morediags...) - cfg.LocalBlocks = append(cfg.LocalBlocks, moreLocals...) - } + for _, file := range files { + diags = append(diags, cfg.decodeFile(file)...) } // parse var files @@ -293,111 +272,314 @@ func filterVarsFromLogs(inputOrLocal Variables) { } } -func (cfg *PackerConfig) Initialize(opts packer.InitializeOptions) hcl.Diagnostics { - diags := cfg.InputVariables.ValidateValues() - diags = append(diags, cfg.evaluateDatasources(opts.SkipDatasourcesExecution)...) - diags = append(diags, checkForDuplicateLocalDefinition(cfg.LocalBlocks)...) - diags = append(diags, cfg.evaluateLocalVariables(cfg.LocalBlocks)...) +// decodeFile attempts to decode the configuration from a HCL file, and starts +// populating the config from it. +func (cfg *PackerConfig) decodeFile(file *hcl.File) hcl.Diagnostics { + var diags hcl.Diagnostics + + content, moreDiags := file.Body.Content(configSchema) + diags = append(diags, moreDiags...) + // If basic parsing failed, we should not continue + if diags.HasErrors() { + return diags + } + + for _, block := range content.Blocks { + diags = append(diags, cfg.decodeBlock(block)...) + } + + return diags +} - filterVarsFromLogs(cfg.InputVariables) - filterVarsFromLogs(cfg.LocalVariables) +func (cfg *PackerConfig) decodeBlock(block *hcl.Block) hcl.Diagnostics { + switch block.Type { + case packerLabel: + return cfg.decodePackerBlock(block) + case dataSourceLabel: + return cfg.decodeDatasource(block) + case variableLabel: + return cfg.decodeVariableBlock(block) + case variablesLabel: + return cfg.decodeVariablesBlock(block) + case localLabel: + return cfg.decodeLocalBlock(block) + case localsLabel: + return cfg.decodeLocalsBlock(block) + // NOTE: Both build and source blocks can be dynamically expanded. + // + // Because of that, we only populate the config's top-level build and + // source lists, but we don't attempt to dynamically expand them now, + // since dynamic blocks can depend on other resources that are executed + // later in the process. + // + // The source and builds are only ready to use when we have called the + // finalizeDecode/finalizeDecodeSource respectively on the blocks. + case buildLabel: + return cfg.decodeBuildBlock(block) + case sourceLabel: + return cfg.decodeSourceBlock(block) + } - // parse the actual content // rest - for _, file := range cfg.files { - diags = append(diags, cfg.parser.parseConfig(file, cfg)...) + return hcl.Diagnostics{ + &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid block type", + Detail: fmt.Sprintf("The block %q is not a valid top-level block", block.Type), + Subject: &block.DefRange, + }, } +} - diags = append(diags, cfg.initializeBlocks()...) +func (cfg *PackerConfig) decodePackerBlock(block *hcl.Block) hcl.Diagnostics { + var diags hcl.Diagnostics + content, contentDiags := block.Body.Content(packerBlockSchema) + diags = append(diags, contentDiags...) + + // We ignore "packer_version"" here because + // sniffCoreVersionRequirements already dealt with that + for _, innerBlock := range content.Blocks { + switch innerBlock.Type { + case "required_plugins": + reqs, reqsDiags := decodeRequiredPluginsBlock(innerBlock) + diags = append(diags, reqsDiags...) + cfg.Packer.RequiredPlugins = append(cfg.Packer.RequiredPlugins, reqs) + default: + continue + } + + } return diags } -// parseConfig looks in the found blocks for everything that is not a variable -// block. -func (p *Parser) parseConfig(f *hcl.File, cfg *PackerConfig) hcl.Diagnostics { +func (cfg *PackerConfig) decodeDatasource(block *hcl.Block) hcl.Diagnostics { + datasource, diags := cfg.decodeDataBlock(block) + if diags.HasErrors() { + return diags + } + ref := datasource.Ref() + if existing, found := cfg.Datasources[ref]; found { + return append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Duplicate " + dataSourceLabel + " block", + Detail: fmt.Sprintf("This "+dataSourceLabel+" block has the "+ + "same data type and name as a previous block declared "+ + "at %s. Each "+dataSourceLabel+" must have a unique name per builder type.", + existing.block.DefRange.Ptr()), + Subject: datasource.block.DefRange.Ptr(), + }) + } + if cfg.Datasources == nil { + cfg.Datasources = Datasources{} + } + cfg.Datasources[ref] = *datasource + + return diags +} + +func (cfg *PackerConfig) decodeDataBlock(block *hcl.Block) (*DatasourceBlock, hcl.Diagnostics) { var diags hcl.Diagnostics + r := &DatasourceBlock{ + Type: block.Labels[0], + Name: block.Labels[1], + block: block, + } - body := f.Body - body = dynblock.Expand(body, cfg.EvalContext(DatasourceContext, nil)) - content, moreDiags := body.Content(configSchema) - diags = append(diags, moreDiags...) + if !hclsyntax.ValidIdentifier(r.Type) { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid data source name", + Detail: badIdentifierDetail, + Subject: &block.LabelRanges[0], + }) + } + if !hclsyntax.ValidIdentifier(r.Name) { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid data resource name", + Detail: badIdentifierDetail, + Subject: &block.LabelRanges[1], + }) + } - for _, block := range content.Blocks { - switch block.Type { - case sourceLabel: - source, moreDiags := p.decodeSource(block) - diags = append(diags, moreDiags...) - if moreDiags.HasErrors() { - continue - } + return r, diags +} - ref := source.Ref() - if existing, found := cfg.Sources[ref]; found { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Duplicate " + sourceLabel + " block", - Detail: fmt.Sprintf("This "+sourceLabel+" block has the "+ - "same builder type and name as a previous block declared "+ - "at %s. Each "+sourceLabel+" must have a unique name per builder type.", - existing.block.DefRange.Ptr()), - Subject: source.block.DefRange.Ptr(), - }) - continue - } +func (cfg *PackerConfig) decodeLocalBlock(block *hcl.Block) hcl.Diagnostics { + name := block.Labels[0] - if cfg.Sources == nil { - cfg.Sources = map[SourceRef]SourceBlock{} - } - cfg.Sources[ref] = source + content, diags := block.Body.Content(localBlockSchema) + if !hclsyntax.ValidIdentifier(name) { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid local name", + Detail: badIdentifierDetail, + Subject: &block.LabelRanges[0], + }) + } - case buildLabel: - build, moreDiags := p.decodeBuildConfig(block, cfg) - diags = append(diags, moreDiags...) - if moreDiags.HasErrors() { - continue - } + l := &LocalBlock{ + Name: name, + } - cfg.Builds = append(cfg.Builds, build) - } + if attr, exists := content.Attributes["sensitive"]; exists { + valDiags := gohcl.DecodeExpression(attr.Expr, nil, &l.Sensitive) + diags = append(diags, valDiags...) + } + + if def, ok := content.Attributes["expression"]; ok { + l.Expr = def.Expr + } + + cfg.LocalBlocks = append(cfg.LocalBlocks, l) + + return diags +} + +func (cfg *PackerConfig) decodeLocalsBlock(block *hcl.Block) hcl.Diagnostics { + attrs, diags := block.Body.JustAttributes() + + for name, attr := range attrs { + cfg.LocalBlocks = append(cfg.LocalBlocks, &LocalBlock{ + Name: name, + Expr: attr.Expr, + }) } return diags } -func (p *Parser) decodeDatasources(file *hcl.File, cfg *PackerConfig) hcl.Diagnostics { +func (cfg *PackerConfig) decodeVariableBlock(block *hcl.Block) hcl.Diagnostics { + // for input variables we allow to use env in the default value section. + ectx := &hcl.EvalContext{ + Functions: map[string]function.Function{ + "env": pkrfunction.EnvFunc, + }, + } + + return cfg.InputVariables.decodeVariableBlock(block, ectx) +} + +func (cfg *PackerConfig) decodeVariablesBlock(block *hcl.Block) hcl.Diagnostics { + // for input variables we allow to use env in the default value section. + ectx := &hcl.EvalContext{ + Functions: map[string]function.Function{ + "env": pkrfunction.EnvFunc, + }, + } + var diags hcl.Diagnostics - body := file.Body - content, moreDiags := body.Content(configSchema) + attrs, moreDiags := block.Body.JustAttributes() diags = append(diags, moreDiags...) + for key, attr := range attrs { + moreDiags = cfg.InputVariables.decodeVariable(key, attr, ectx) + diags = append(diags, moreDiags...) + } - for _, block := range content.Blocks { - switch block.Type { - case dataSourceLabel: - datasource, moreDiags := p.decodeDataBlock(block) - diags = append(diags, moreDiags...) - if moreDiags.HasErrors() { - continue - } - ref := datasource.Ref() - if existing, found := cfg.Datasources[ref]; found { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Duplicate " + dataSourceLabel + " block", - Detail: fmt.Sprintf("This "+dataSourceLabel+" block has the "+ - "same data type and name as a previous block declared "+ - "at %s. Each "+dataSourceLabel+" must have a unique name per builder type.", - existing.block.DefRange.Ptr()), - Subject: datasource.block.DefRange.Ptr(), - }) - continue - } - if cfg.Datasources == nil { - cfg.Datasources = Datasources{} - } - cfg.Datasources[ref] = *datasource - } + return diags +} + +// decodeBuildBlock shallowly decodes a build block from the config. +// +// The final decoding step (which requires an up-to-date context) will be done +// when we need it. +func (cfg *PackerConfig) decodeBuildBlock(block *hcl.Block) hcl.Diagnostics { + build := &BuildBlock{ + block: block, + } + + cfg.Builds = append(cfg.Builds, build) + + return nil +} + +func (cfg *PackerConfig) decodeSourceBlock(block *hcl.Block) hcl.Diagnostics { + source, diags := cfg.decodeSource(block) + if diags.HasErrors() { + return diags + } + + if cfg.Sources == nil { + cfg.Sources = map[SourceRef]SourceBlock{} } + ref := source.Ref() + if existing, found := cfg.Sources[ref]; found { + return append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Duplicate " + sourceLabel + " block", + Detail: fmt.Sprintf("This "+sourceLabel+" block has the "+ + "same builder type and name as a previous block declared "+ + "at %s. Each "+sourceLabel+" must have a unique name per builder type.", + existing.block.DefRange.Ptr()), + Subject: source.block.DefRange.Ptr(), + }) + } + + cfg.Sources[ref] = source + return diags } + +// decodeBuildSource reads a used source block from a build: +// +// build { +// source "type.example" { +// name = "local_name" +// } +// } +func (cfg *PackerConfig) decodeBuildSource(block *hcl.Block) (SourceUseBlock, hcl.Diagnostics) { + ref := sourceRefFromString(block.Labels[0]) + out := SourceUseBlock{SourceRef: ref} + var b struct { + Name string `hcl:"name,optional"` + Rest hcl.Body `hcl:",remain"` + } + diags := gohcl.DecodeBody(block.Body, nil, &b) + if diags.HasErrors() { + return out, diags + } + out.LocalName = b.Name + out.Body = b.Rest + return out, nil +} + +func (source *SourceBlock) finalizeDecode(cfg *PackerConfig) hcl.Diagnostics { + if source.Ready { + return nil + } + + dyn := dynblock.Expand(source.block.Body, cfg.EvalContext(DatasourceContext, nil)) + // Expand without a base schema since nothing is known in advance for a + // source, but we still want to expand dynamic blocks if any + _, rem, diags := dyn.PartialContent(&hcl.BodySchema{}) + + // Only try to expand once, regardless of whether the source succeeded + // to expand dynamic data or not. + source.Ready = true + if diags.HasErrors() { + return diags + } + + source.block = &hcl.Block{ + Labels: []string{ + source.Type, + source.Name, + }, + Body: rem, + } + + return diags +} + +func (cfg *PackerConfig) decodeSource(block *hcl.Block) (SourceBlock, hcl.Diagnostics) { + source := SourceBlock{ + Type: block.Labels[0], + Name: block.Labels[1], + block: block, + } + var diags hcl.Diagnostics + + return source, diags +} diff --git a/hcl2template/plugin.go b/hcl2template/plugin.go index 323263596f8..8575b83baec 100644 --- a/hcl2template/plugin.go +++ b/hcl2template/plugin.go @@ -128,6 +128,15 @@ func (cfg *PackerConfig) initializeBlocks() hcl.Diagnostics { var diags hcl.Diagnostics for _, build := range cfg.Builds { + // Since during parsing we only shallowly decoded the block, we + // need to finish this step, with an up-to-date context for + // both dynamic expansion, and expressions that rely on + // components that got executed before we do this step. + diags = append(diags, build.finalizeDecode(cfg)...) + if diags.HasErrors() { + continue + } + for i := range build.Sources { // here we grab a pointer to the source usage because we will set // its body. @@ -158,6 +167,13 @@ func (cfg *PackerConfig) initializeBlocks() hcl.Diagnostics { continue } + // Before attempting to use the body for merging, we + // finalise its decoding if necessary. + diags = append(diags, sourceDefinition.finalizeDecode(cfg)...) + if diags.HasErrors() { + continue + } + body := sourceDefinition.block.Body if srcUsage.Body != nil { // merge additions into source definition to get a new body. diff --git a/hcl2template/types.build.go b/hcl2template/types.build.go index 648305ee202..b581b10067f 100644 --- a/hcl2template/types.build.go +++ b/hcl2template/types.build.go @@ -7,6 +7,7 @@ import ( "fmt" "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/ext/dynblock" "github.com/hashicorp/hcl/v2/gohcl" "github.com/hashicorp/hcl/v2/hclsyntax" "github.com/zclconf/go-cty/cty" @@ -83,13 +84,27 @@ type BuildBlock struct { PostProcessorsLists [][]*PostProcessorBlock HCL2Ref HCL2Ref + + // Block is the raw hcl block lifted from the HCL file + block *hcl.Block + // ready marks whether or not there's any decoding left to do before + // using the data from the build block. + decoded bool } type Builds []*BuildBlock -// decodeBuildConfig is called when a 'build' block has been detected. It will -// load the references to the contents of the build block. -func (p *Parser) decodeBuildConfig(block *hcl.Block, cfg *PackerConfig) (*BuildBlock, hcl.Diagnostics) { +// finalizeDecode finalises decoding the build block. +// +// This is only called after we've finished evaluating the dependencies for the +// build, and will expand the dynamic block for it, if any were present at first. +func (build *BuildBlock) finalizeDecode(cfg *PackerConfig) hcl.Diagnostics { + if build.decoded { + return nil + } + + build.decoded = true + var b struct { Name string `hcl:"name,optional"` Description string `hcl:"description,optional"` @@ -97,25 +112,30 @@ func (p *Parser) decodeBuildConfig(block *hcl.Block, cfg *PackerConfig) (*BuildB Config hcl.Body `hcl:",remain"` } - body := block.Body - diags := gohcl.DecodeBody(body, cfg.EvalContext(LocalContext, nil), &b) - if diags.HasErrors() { - return nil, diags - } + var diags hcl.Diagnostics - build := &BuildBlock{ - HCL2Ref: newHCL2Ref(block, b.Config), - } + body := build.block.Body + // At this point we can discard this decode's diags since it has already + // been sucessfully done once during the initial pre-decoding phase (at + // parsing-time) + _ = gohcl.DecodeBody(body, cfg.EvalContext(LocalContext, nil), &b) + // Here we'll replace the base contents from what we re-extracted at the + // time, as some things may be derived from other components through expressions + // or interpolation. build.Name = b.Name build.Description = b.Description - build.HCL2Ref.DefRange = block.DefRange + build.HCL2Ref = newHCL2Ref(build.block, b.Config) - // Expose build.name during parsing of pps and provisioners ectx := cfg.EvalContext(BuildContext, nil) - ectx.Variables[buildAccessor] = cty.ObjectVal(map[string]cty.Value{ - "name": cty.StringVal(b.Name), - }) + // Expand dynamics: we wrap the config in a dynblock and request the final + // content. If something cannot be expanded for some reason here (invalid + // reference, unknown values, etc.), this will fail, as it should. + dyn := dynblock.Expand(b.Config, ectx) + content, expandDiags := dyn.Content(buildSchema) + if expandDiags.HasErrors() { + return append(diags, expandDiags...) + } // We rely on `hadSource` to determine which error to proc. // @@ -124,6 +144,11 @@ func (p *Parser) decodeBuildConfig(block *hcl.Block, cfg *PackerConfig) (*BuildB // source is processed. hadSource := false + // Expose build.name during parsing of pps and provisioners + ectx.Variables[buildAccessor] = cty.ObjectVal(map[string]cty.Value{ + "name": cty.StringVal(b.Name), + }) + for _, buildFrom := range b.FromSources { hadSource = true @@ -135,11 +160,11 @@ func (p *Parser) decodeBuildConfig(block *hcl.Block, cfg *PackerConfig) (*BuildB diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, Summary: "Invalid " + sourceLabel + " reference", - Detail: "A " + sourceLabel + " type is made of three parts that are" + + Detail: "A " + sourceLabel + " type is made of two to three parts that are" + "split by a dot `.`; each part must start with a letter and " + "may contain only letters, digits, underscores, and dashes." + "A valid source reference looks like: `source.type.name`", - Subject: block.DefRange.Ptr(), + Subject: build.block.DefRange.Ptr(), }) continue } @@ -148,12 +173,6 @@ func (p *Parser) decodeBuildConfig(block *hcl.Block, cfg *PackerConfig) (*BuildB build.Sources = append(build.Sources, SourceUseBlock{SourceRef: ref}) } - body = b.Config - content, moreDiags := body.Content(buildSchema) - diags = append(diags, moreDiags...) - if diags.HasErrors() { - return nil, diags - } for _, block := range content.Blocks { switch block.Type { case buildHCPPackerRegistryLabel: @@ -165,7 +184,7 @@ func (p *Parser) decodeBuildConfig(block *hcl.Block, cfg *PackerConfig) (*BuildB }) continue } - hcpPackerRegistry, moreDiags := p.decodeHCPRegistry(block, cfg) + hcpPackerRegistry, moreDiags := cfg.decodeHCPRegistry(block) diags = append(diags, moreDiags...) if moreDiags.HasErrors() { continue @@ -173,14 +192,14 @@ func (p *Parser) decodeBuildConfig(block *hcl.Block, cfg *PackerConfig) (*BuildB build.HCPPackerRegistry = hcpPackerRegistry case sourceLabel: hadSource = true - ref, moreDiags := p.decodeBuildSource(block) + ref, moreDiags := cfg.decodeBuildSource(block) diags = append(diags, moreDiags...) if moreDiags.HasErrors() { continue } build.Sources = append(build.Sources, ref) case buildProvisionerLabel: - p, moreDiags := p.decodeProvisioner(block, ectx) + p, moreDiags := cfg.decodeProvisioner(block, ectx) diags = append(diags, moreDiags...) if moreDiags.HasErrors() { continue @@ -195,14 +214,14 @@ func (p *Parser) decodeBuildConfig(block *hcl.Block, cfg *PackerConfig) (*BuildB }) continue } - p, moreDiags := p.decodeProvisioner(block, ectx) + p, moreDiags := cfg.decodeProvisioner(block, ectx) diags = append(diags, moreDiags...) if moreDiags.HasErrors() { continue } build.ErrorCleanupProvisionerBlock = p case buildPostProcessorLabel: - pp, moreDiags := p.decodePostProcessor(block, ectx) + pp, moreDiags := cfg.decodePostProcessor(block, ectx) diags = append(diags, moreDiags...) if moreDiags.HasErrors() { continue @@ -219,7 +238,7 @@ func (p *Parser) decodeBuildConfig(block *hcl.Block, cfg *PackerConfig) (*BuildB errored := false postProcessors := []*PostProcessorBlock{} for _, block := range content.Blocks { - pp, moreDiags := p.decodePostProcessor(block, ectx) + pp, moreDiags := cfg.decodePostProcessor(block, ectx) diags = append(diags, moreDiags...) if moreDiags.HasErrors() { errored = true @@ -238,9 +257,9 @@ func (p *Parser) decodeBuildConfig(block *hcl.Block, cfg *PackerConfig) (*BuildB Summary: "missing source reference", Detail: "a build block must reference at least one source to be built", Severity: hcl.DiagError, - Subject: block.DefRange.Ptr(), + Subject: build.block.DefRange.Ptr(), }) } - return build, diags + return diags } diff --git a/hcl2template/types.build.hcp_packer_registry.go b/hcl2template/types.build.hcp_packer_registry.go index b64b3ff4e55..b28621a034a 100644 --- a/hcl2template/types.build.hcp_packer_registry.go +++ b/hcl2template/types.build.hcp_packer_registry.go @@ -23,7 +23,7 @@ type HCPPackerRegistryBlock struct { HCL2Ref } -func (p *Parser) decodeHCPRegistry(block *hcl.Block, cfg *PackerConfig) (*HCPPackerRegistryBlock, hcl.Diagnostics) { +func (cfg *PackerConfig) decodeHCPRegistry(block *hcl.Block) (*HCPPackerRegistryBlock, hcl.Diagnostics) { par := &HCPPackerRegistryBlock{} body := block.Body diff --git a/hcl2template/types.build.hcp_packer_registry_test.go b/hcl2template/types.build.hcp_packer_registry_test.go index 3ada389c29d..cfef9527751 100644 --- a/hcl2template/types.build.hcp_packer_registry_test.go +++ b/hcl2template/types.build.hcp_packer_registry_test.go @@ -118,6 +118,11 @@ func Test_ParseHCPPackerRegistryBlock(t *testing.T) { &PackerConfig{ CorePackerVersionString: lockedVersion, Basedir: filepath.Join("testdata", "hcp_par"), + Builds: Builds{ + &BuildBlock{ + Name: "bucket-slug", + }, + }, }, true, true, nil, @@ -129,6 +134,11 @@ func Test_ParseHCPPackerRegistryBlock(t *testing.T) { &PackerConfig{ CorePackerVersionString: lockedVersion, Basedir: filepath.Join("testdata", "hcp_par"), + Builds: Builds{ + &BuildBlock{ + Name: "bucket-slug", + }, + }, }, true, true, nil, diff --git a/hcl2template/types.build.post-processor.go b/hcl2template/types.build.post-processor.go index 8844eadff11..1cf943b0bd6 100644 --- a/hcl2template/types.build.post-processor.go +++ b/hcl2template/types.build.post-processor.go @@ -26,7 +26,7 @@ func (p *PostProcessorBlock) String() string { return fmt.Sprintf(buildPostProcessorLabel+"-block %q %q", p.PType, p.PName) } -func (p *Parser) decodePostProcessor(block *hcl.Block, ectx *hcl.EvalContext) (*PostProcessorBlock, hcl.Diagnostics) { +func (cfg *PackerConfig) decodePostProcessor(block *hcl.Block, ectx *hcl.EvalContext) (*PostProcessorBlock, hcl.Diagnostics) { var b struct { Name string `hcl:"name,optional"` Only []string `hcl:"only,optional"` diff --git a/hcl2template/types.build.provisioners.go b/hcl2template/types.build.provisioners.go index b08eca59f63..dd6fb71ae7f 100644 --- a/hcl2template/types.build.provisioners.go +++ b/hcl2template/types.build.provisioners.go @@ -77,7 +77,7 @@ func (p *ProvisionerBlock) String() string { return fmt.Sprintf(buildProvisionerLabel+"-block %q %q", p.PType, p.PName) } -func (p *Parser) decodeProvisioner(block *hcl.Block, ectx *hcl.EvalContext) (*ProvisionerBlock, hcl.Diagnostics) { +func (cfg *PackerConfig) decodeProvisioner(block *hcl.Block, ectx *hcl.EvalContext) (*ProvisionerBlock, hcl.Diagnostics) { var b struct { Name string `hcl:"name,optional"` PauseBefore string `hcl:"pause_before,optional"` diff --git a/hcl2template/types.build.provisioners_test.go b/hcl2template/types.build.provisioners_test.go index 284651f2cbd..f4a1558a5d7 100644 --- a/hcl2template/types.build.provisioners_test.go +++ b/hcl2template/types.build.provisioners_test.go @@ -52,7 +52,7 @@ func TestPackerConfig_ParseProvisionerBlock(t *testing.T) { Column: 1, Byte: 0, }) - _, diags = cfg.parser.decodeProvisioner(provBlock, nil) + _, diags = cfg.decodeProvisioner(provBlock, nil) if !diags.HasErrors() { if !test.expectError { diff --git a/hcl2template/types.build_test.go b/hcl2template/types.build_test.go index 8647821dd18..808929ed83a 100644 --- a/hcl2template/types.build_test.go +++ b/hcl2template/types.build_test.go @@ -64,7 +64,9 @@ func TestParse_build(t *testing.T) { &PackerConfig{ CorePackerVersionString: lockedVersion, Basedir: filepath.Join("testdata", "build"), - Builds: nil, + Builds: Builds{ + &BuildBlock{}, + }, }, true, true, nil, @@ -118,6 +120,18 @@ func TestParse_build(t *testing.T) { Sources: map[SourceRef]SourceBlock{ refVBIsoUbuntu1204: {Type: "virtualbox-iso", Name: "ubuntu-1204"}, }, + Builds: Builds{ + &BuildBlock{ + Sources: []SourceUseBlock{ + { + SourceRef: refVBIsoUbuntu1204, + }, + }, + ErrorCleanupProvisionerBlock: &ProvisionerBlock{ + PType: "shell-local", + }, + }, + }, }, true, true, []packersdk.Build{&packer.CoreBuild{ @@ -142,7 +156,9 @@ func TestParse_build(t *testing.T) { &PackerConfig{ CorePackerVersionString: lockedVersion, Basedir: filepath.Join("testdata", "build"), - Builds: nil, + Builds: Builds{ + &BuildBlock{}, + }, }, true, true, []packersdk.Build{&packer.CoreBuild{}}, @@ -195,7 +211,9 @@ func TestParse_build(t *testing.T) { &PackerConfig{ CorePackerVersionString: lockedVersion, Basedir: filepath.Join("testdata", "build"), - Builds: nil, + Builds: Builds{ + &BuildBlock{}, + }, }, true, true, []packersdk.Build{}, @@ -565,7 +583,9 @@ func TestParse_build(t *testing.T) { CorePackerVersionString: lockedVersion, Basedir: filepath.Join("testdata", "build"), InputVariables: Variables{}, - Builds: nil, + Builds: Builds{ + &BuildBlock{}, + }, }, true, true, []packersdk.Build{}, diff --git a/hcl2template/types.datasource.go b/hcl2template/types.datasource.go index 348d06f452c..976470ebc46 100644 --- a/hcl2template/types.datasource.go +++ b/hcl2template/types.datasource.go @@ -7,7 +7,6 @@ import ( "fmt" "github.com/hashicorp/hcl/v2" - "github.com/hashicorp/hcl/v2/hclsyntax" packersdk "github.com/hashicorp/packer-plugin-sdk/packer" hcl2shim "github.com/hashicorp/packer/hcl2template/shim" "github.com/hashicorp/packer/packer" @@ -130,31 +129,3 @@ func (cfg *PackerConfig) startDatasource(dataSourceStore packer.DatasourceStore, } return datasource, diags } - -func (p *Parser) decodeDataBlock(block *hcl.Block) (*DatasourceBlock, hcl.Diagnostics) { - var diags hcl.Diagnostics - r := &DatasourceBlock{ - Type: block.Labels[0], - Name: block.Labels[1], - block: block, - } - - if !hclsyntax.ValidIdentifier(r.Type) { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Invalid data source name", - Detail: badIdentifierDetail, - Subject: &block.LabelRanges[0], - }) - } - if !hclsyntax.ValidIdentifier(r.Name) { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Invalid data resource name", - Detail: badIdentifierDetail, - Subject: &block.LabelRanges[1], - }) - } - - return r, diags -} diff --git a/hcl2template/types.packer_config.go b/hcl2template/types.packer_config.go index 819107e86b3..c77c5864b23 100644 --- a/hcl2template/types.packer_config.go +++ b/hcl2template/types.packer_config.go @@ -179,40 +179,6 @@ func (c *PackerConfig) decodeInputVariables(f *hcl.File) hcl.Diagnostics { return diags } -// parseLocalVariableBlocks looks in the AST for 'local' and 'locals' blocks and -// returns them all. -func parseLocalVariableBlocks(f *hcl.File) ([]*LocalBlock, hcl.Diagnostics) { - var diags hcl.Diagnostics - - content, moreDiags := f.Body.Content(configSchema) - diags = append(diags, moreDiags...) - - var locals []*LocalBlock - - for _, block := range content.Blocks { - switch block.Type { - case localLabel: - block, moreDiags := decodeLocalBlock(block) - diags = append(diags, moreDiags...) - if moreDiags.HasErrors() { - return locals, diags - } - locals = append(locals, block) - case localsLabel: - attrs, moreDiags := block.Body.JustAttributes() - diags = append(diags, moreDiags...) - for name, attr := range attrs { - locals = append(locals, &LocalBlock{ - Name: name, - Expr: attr.Expr, - }) - } - } - } - - return locals, diags -} - func (c *PackerConfig) evaluateAllLocalVariables(locals []*LocalBlock) hcl.Diagnostics { var diags hcl.Diagnostics @@ -880,3 +846,18 @@ func (p *PackerConfig) InspectConfig(opts packer.InspectConfigOptions) int { ui.Say(p.printBuilds()) return 0 } + +func (cfg *PackerConfig) Initialize(opts packer.InitializeOptions) hcl.Diagnostics { + diags := cfg.InputVariables.ValidateValues() + diags = append(diags, cfg.LocalVariables.ValidateValues()...) + diags = append(diags, cfg.evaluateDatasources(opts.SkipDatasourcesExecution)...) + diags = append(diags, checkForDuplicateLocalDefinition(cfg.LocalBlocks)...) + diags = append(diags, cfg.evaluateLocalVariables(cfg.LocalBlocks)...) + + filterVarsFromLogs(cfg.InputVariables) + filterVarsFromLogs(cfg.LocalVariables) + + diags = append(diags, cfg.initializeBlocks()...) + + return diags +} diff --git a/hcl2template/types.packer_config_test.go b/hcl2template/types.packer_config_test.go index 5eae6a2f21f..3af072164b7 100644 --- a/hcl2template/types.packer_config_test.go +++ b/hcl2template/types.packer_config_test.go @@ -575,8 +575,19 @@ func TestParser_no_init(t *testing.T) { Type: cty.List(cty.String), }, }, - Sources: nil, - Builds: nil, + Sources: map[SourceRef]SourceBlock{ + refAWSV3MyImage: { + Type: "amazon-v3-ebs", + Name: "my-image", + }, + refVBIsoUbuntu1204: { + Type: "virtualbox-iso", + Name: "ubuntu-1204", + }, + }, + Builds: Builds{ + &BuildBlock{}, + }, }, false, false, []packersdk.Build{}, diff --git a/hcl2template/types.required_plugins.go b/hcl2template/types.required_plugins.go index d5e010345ea..c7e585f7893 100644 --- a/hcl2template/types.required_plugins.go +++ b/hcl2template/types.required_plugins.go @@ -12,37 +12,6 @@ import ( "github.com/zclconf/go-cty/cty" ) -func (cfg *PackerConfig) decodeRequiredPluginsBlock(f *hcl.File) hcl.Diagnostics { - var diags hcl.Diagnostics - - content, moreDiags := f.Body.Content(configSchema) - diags = append(diags, moreDiags...) - - for _, block := range content.Blocks { - switch block.Type { - case packerLabel: - content, contentDiags := block.Body.Content(packerBlockSchema) - diags = append(diags, contentDiags...) - - // We ignore "packer_version"" here because - // sniffCoreVersionRequirements already dealt with that - - for _, innerBlock := range content.Blocks { - switch innerBlock.Type { - case "required_plugins": - reqs, reqsDiags := decodeRequiredPluginsBlock(innerBlock) - diags = append(diags, reqsDiags...) - cfg.Packer.RequiredPlugins = append(cfg.Packer.RequiredPlugins, reqs) - default: - continue - } - - } - } - } - return diags -} - // RequiredPlugin represents a declaration of a dependency on a particular // Plugin version or source. type RequiredPlugin struct { diff --git a/hcl2template/types.required_plugins_test.go b/hcl2template/types.required_plugins_test.go index 0329d1505ff..e9bd9c48833 100644 --- a/hcl2template/types.required_plugins_test.go +++ b/hcl2template/types.required_plugins_test.go @@ -138,7 +138,7 @@ func TestPackerConfig_required_plugin_parse(t *testing.T) { if len(diags) > 0 { t.Fatal(diags) } - if diags := cfg.decodeRequiredPluginsBlock(file); len(diags) > 0 { + if diags := cfg.decodeFile(file); len(diags) > 0 { t.Fatal(diags) } diff --git a/hcl2template/types.source.go b/hcl2template/types.source.go index 46b9caac98e..450c8fd12c8 100644 --- a/hcl2template/types.source.go +++ b/hcl2template/types.source.go @@ -9,7 +9,6 @@ import ( "strconv" "github.com/hashicorp/hcl/v2" - "github.com/hashicorp/hcl/v2/gohcl" packersdk "github.com/hashicorp/packer-plugin-sdk/packer" hcl2shim "github.com/hashicorp/packer/hcl2template/shim" "github.com/zclconf/go-cty/cty" @@ -23,6 +22,10 @@ type SourceBlock struct { // Given name; if any Name string + // ready signals that the source block is ready for use, i.e. it does not + // need some dynamic expansion before being used. + Ready bool + block *hcl.Block // LocalName can be set in a singular source block from a build block, it @@ -66,40 +69,6 @@ func (b *SourceUseBlock) ctyValues() map[string]cty.Value { } } -// decodeBuildSource reads a used source block from a build: -// -// build { -// source "type.example" { -// name = "local_name" -// } -// } -func (p *Parser) decodeBuildSource(block *hcl.Block) (SourceUseBlock, hcl.Diagnostics) { - ref := sourceRefFromString(block.Labels[0]) - out := SourceUseBlock{SourceRef: ref} - var b struct { - Name string `hcl:"name,optional"` - Rest hcl.Body `hcl:",remain"` - } - diags := gohcl.DecodeBody(block.Body, nil, &b) - if diags.HasErrors() { - return out, diags - } - out.LocalName = b.Name - out.Body = b.Rest - return out, nil -} - -func (p *Parser) decodeSource(block *hcl.Block) (SourceBlock, hcl.Diagnostics) { - source := SourceBlock{ - Type: block.Labels[0], - Name: block.Labels[1], - block: block, - } - var diags hcl.Diagnostics - - return source, diags -} - func (cfg *PackerConfig) startBuilder(source SourceUseBlock, ectx *hcl.EvalContext) (packersdk.Builder, hcl.Diagnostics, []string) { var diags hcl.Diagnostics diff --git a/hcl2template/types.source_test.go b/hcl2template/types.source_test.go index 47caeec5a75..3aa5cbd62fb 100644 --- a/hcl2template/types.source_test.go +++ b/hcl2template/types.source_test.go @@ -90,8 +90,10 @@ func TestParse_source(t *testing.T) { parseTestArgs{"testdata/sources/nonexistent.pkr.hcl", nil, nil}, &PackerConfig{ CorePackerVersionString: lockedVersion, - Builds: nil, - Basedir: filepath.Join("testdata", "sources"), + Builds: Builds{ + &BuildBlock{}, + }, + Basedir: filepath.Join("testdata", "sources"), Sources: map[SourceRef]SourceBlock{ {Type: "nonexistent", Name: "ubuntu-1204"}: {Type: "nonexistent", Name: "ubuntu-1204"}, }, diff --git a/hcl2template/types.variables.go b/hcl2template/types.variables.go index 33df28a84ee..f45aa72913c 100644 --- a/hcl2template/types.variables.go +++ b/hcl2template/types.variables.go @@ -278,35 +278,6 @@ var localBlockSchema = &hcl.BodySchema{ }, } -func decodeLocalBlock(block *hcl.Block) (*LocalBlock, hcl.Diagnostics) { - name := block.Labels[0] - - content, diags := block.Body.Content(localBlockSchema) - if !hclsyntax.ValidIdentifier(name) { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Invalid local name", - Detail: badIdentifierDetail, - Subject: &block.LabelRanges[0], - }) - } - - l := &LocalBlock{ - Name: name, - } - - if attr, exists := content.Attributes["sensitive"]; exists { - valDiags := gohcl.DecodeExpression(attr.Expr, nil, &l.Sensitive) - diags = append(diags, valDiags...) - } - - if def, ok := content.Attributes["expression"]; ok { - l.Expr = def.Expr - } - - return l, diags -} - // decodeVariableBlock decodes a "variable" block // ectx is passed only in the evaluation of the default value. func (variables *Variables) decodeVariableBlock(block *hcl.Block, ectx *hcl.EvalContext) hcl.Diagnostics { diff --git a/hcl2template/types.variables_test.go b/hcl2template/types.variables_test.go index a19b80bd990..37bfc779f0d 100644 --- a/hcl2template/types.variables_test.go +++ b/hcl2template/types.variables_test.go @@ -248,7 +248,10 @@ func TestParse_variables(t *testing.T) { parseTestArgs{"testdata/variables/unset_used_string_variable.pkr.hcl", nil, nil}, &PackerConfig{ CorePackerVersionString: lockedVersion, - Basedir: filepath.Join("testdata", "variables"), + Builds: Builds{ + &BuildBlock{}, + }, + Basedir: filepath.Join("testdata", "variables"), InputVariables: Variables{ "foo": &Variable{ Name: "foo",