From 1fb5b20e761ca061e44a3fcf78539fdfbfe6e232 Mon Sep 17 00:00:00 2001 From: Lucas Bajolet <105649352+lbajolet-hashicorp@users.noreply.github.com> Date: Wed, 24 Jul 2024 16:58:13 -0400 Subject: [PATCH] packer: pick protobuf/gob for serialisation (#13025) As we're trying to move away from gob for serialising data over the wire, this commit adds the capability for Packer to pick dynamically between gob or protobuf for the serialisation format to communicate with plugins. As it stands, if all the plugins discovered are compatible with protobuf, and we have not forced gob usage, protobuf will be the serialisation format picked. If any plugin is not compatible with protobuf, gob will be used for communicating with all the plugins that will be used over the course of a command. --- command/build.go | 4 +++ command/execute.go | 55 ++++++++++++++++++++++------ command/validate.go | 5 +++ config.go | 52 ++++++++++++++++++--------- go.mod | 6 ++-- go.sum | 15 ++++---- packer/plugin.go | 71 ++++++++++++++++++++++++++++++++++--- packer/plugin_client.go | 1 + scripts/generate-plugins.go | 57 +++++++++++++++++++++++------ 9 files changed, 218 insertions(+), 48 deletions(-) diff --git a/command/build.go b/command/build.go index a7503eeea3e..2a175dd1798 100644 --- a/command/build.go +++ b/command/build.go @@ -102,6 +102,10 @@ func (c *BuildCommand) RunContext(buildCtx context.Context, cla *BuildArgs) int return ret } + if packer.PackerUseProto { + log.Printf("[TRACE] Using protobuf for communication with plugins") + } + diags = packerStarter.Initialize(packer.InitializeOptions{}) ret = writeDiags(c.Ui, nil, diags) if ret != 0 { diff --git a/command/execute.go b/command/execute.go index 7ad74f314d4..ccecf28f8cf 100644 --- a/command/execute.go +++ b/command/execute.go @@ -5,8 +5,8 @@ package command import ( + "flag" "fmt" - "log" "regexp" "strings" @@ -75,18 +75,45 @@ var Datasources = map[string]packersdk.Datasource{ var pluginRegexp = regexp.MustCompile("packer-(builder|post-processor|provisioner|datasource)-(.+)") -func (c *ExecuteCommand) Run(args []string) int { - // This is an internal call (users should not call this directly) so we're - // not going to do much input validation. If there's a problem we'll often - // just crash. Error handling should be added to facilitate debugging. - log.Printf("args: %#v", args) +type ExecuteArgs struct { + UseProtobuf bool + CommandType string +} + +func (ea *ExecuteArgs) AddFlagSets(flags *flag.FlagSet) { + flags.BoolVar(&ea.UseProtobuf, "protobuf", false, "Use protobuf for serialising data over the wire instead of gob") +} + +func (c *ExecuteCommand) ParseArgs(args []string) (*ExecuteArgs, int) { + var cfg ExecuteArgs + flags := c.Meta.FlagSet("") + flags.Usage = func() { c.Ui.Say(c.Help()) } + cfg.AddFlagSets(flags) + if err := flags.Parse(args); err != nil { + return &cfg, 1 + } + + args = flags.Args() if len(args) != 1 { - c.Ui.Error(c.Help()) - return 1 + flags.Usage() + return &cfg, 1 } + cfg.CommandType = args[0] + return &cfg, 0 +} +func (c *ExecuteCommand) Run(args []string) int { + cfg, ret := c.ParseArgs(args) + if ret != 0 { + return ret + } + + return c.RunContext(cfg) +} + +func (c *ExecuteCommand) RunContext(args *ExecuteArgs) int { // Plugin will match something like "packer-builder-amazon-ebs" - parts := pluginRegexp.FindStringSubmatch(args[0]) + parts := pluginRegexp.FindStringSubmatch(args.CommandType) if len(parts) != 3 { c.Ui.Error(c.Help()) return 1 @@ -100,6 +127,10 @@ func (c *ExecuteCommand) Run(args []string) int { return 1 } + if args.UseProtobuf { + server.UseProto = true + } + switch pluginType { case "builder": builder, found := Builders[pluginName] @@ -138,11 +169,15 @@ func (c *ExecuteCommand) Run(args []string) int { func (*ExecuteCommand) Help() string { helpText := ` -Usage: packer execute PLUGIN +Usage: packer execute [options] PLUGIN Runs an internally-compiled version of a plugin from the packer binary. NOTE: this is an internal command and you should not call it yourself. + +Options: + + --protobuf: use protobuf for serialising data over-the-wire instead of gob. ` return strings.TrimSpace(helpText) diff --git a/command/validate.go b/command/validate.go index 47d88a901ba..f8a0effde76 100644 --- a/command/validate.go +++ b/command/validate.go @@ -5,6 +5,7 @@ package command import ( "context" + "log" "strings" "github.com/hashicorp/packer/packer" @@ -76,6 +77,10 @@ func (c *ValidateCommand) RunContext(ctx context.Context, cla *ValidateArgs) int return ret } + if packer.PackerUseProto { + log.Printf("[TRACE] Using protobuf for communication with plugins") + } + diags = packerStarter.Initialize(packer.InitializeOptions{ SkipDatasourcesExecution: !cla.EvaluateDatasources, }) diff --git a/config.go b/config.go index 35ce6acd927..96327d1d26f 100644 --- a/config.go +++ b/config.go @@ -16,10 +16,6 @@ import ( "github.com/hashicorp/packer/packer" ) -// PACKERSPACE is used to represent the spaces that separate args for a command -// without being confused with spaces in the path to the command itself. -const PACKERSPACE = "-PACKERSPACE-" - type config struct { DisableCheckpoint bool `json:"disable_checkpoint"` DisableCheckpointSignature bool `json:"disable_checkpoint_signature"` @@ -109,10 +105,16 @@ func (c *config) discoverInternalComponents() error { for builder := range command.Builders { builder := builder if !c.Plugins.Builders.Has(builder) { - bin := fmt.Sprintf("%s%sexecute%spacker-builder-%s", - packerPath, PACKERSPACE, PACKERSPACE, builder) c.Plugins.Builders.Set(builder, func() (packersdk.Builder, error) { - return c.Plugins.Client(bin).Builder() + args := []string{"execute"} + + if packer.PackerUseProto { + args = append(args, "--protobuf") + } + + args = append(args, fmt.Sprintf("packer-builder-%s", builder)) + + return c.Plugins.Client(packerPath, args...).Builder() }) } } @@ -120,10 +122,16 @@ func (c *config) discoverInternalComponents() error { for provisioner := range command.Provisioners { provisioner := provisioner if !c.Plugins.Provisioners.Has(provisioner) { - bin := fmt.Sprintf("%s%sexecute%spacker-provisioner-%s", - packerPath, PACKERSPACE, PACKERSPACE, provisioner) c.Plugins.Provisioners.Set(provisioner, func() (packersdk.Provisioner, error) { - return c.Plugins.Client(bin).Provisioner() + args := []string{"execute"} + + if packer.PackerUseProto { + args = append(args, "--protobuf") + } + + args = append(args, fmt.Sprintf("packer-provisioner-%s", provisioner)) + + return c.Plugins.Client(packerPath, args...).Provisioner() }) } } @@ -131,10 +139,16 @@ func (c *config) discoverInternalComponents() error { for postProcessor := range command.PostProcessors { postProcessor := postProcessor if !c.Plugins.PostProcessors.Has(postProcessor) { - bin := fmt.Sprintf("%s%sexecute%spacker-post-processor-%s", - packerPath, PACKERSPACE, PACKERSPACE, postProcessor) c.Plugins.PostProcessors.Set(postProcessor, func() (packersdk.PostProcessor, error) { - return c.Plugins.Client(bin).PostProcessor() + args := []string{"execute"} + + if packer.PackerUseProto { + args = append(args, "--protobuf") + } + + args = append(args, fmt.Sprintf("packer-post-processor-%s", postProcessor)) + + return c.Plugins.Client(packerPath, args...).PostProcessor() }) } } @@ -142,10 +156,16 @@ func (c *config) discoverInternalComponents() error { for dataSource := range command.Datasources { dataSource := dataSource if !c.Plugins.DataSources.Has(dataSource) { - bin := fmt.Sprintf("%s%sexecute%spacker-datasource-%s", - packerPath, PACKERSPACE, PACKERSPACE, dataSource) c.Plugins.DataSources.Set(dataSource, func() (packersdk.Datasource, error) { - return c.Plugins.Client(bin).Datasource() + args := []string{"execute"} + + if packer.PackerUseProto { + args = append(args, "--protobuf") + } + + args = append(args, fmt.Sprintf("packer-datasource-%s", dataSource)) + + return c.Plugins.Client(packerPath, args...).Datasource() }) } } diff --git a/go.mod b/go.mod index bd377d0f4c3..2f17178fdf3 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/hashicorp/hcl/v2 v2.19.1 github.com/hashicorp/hcp-sdk-go v0.105.0 github.com/hashicorp/packer-plugin-amazon v1.2.1 - github.com/hashicorp/packer-plugin-sdk v0.5.4 + github.com/hashicorp/packer-plugin-sdk v0.5.5-0.20240715190433-28e1cb00bc48 github.com/jehiah/go-strftime v0.0.0-20171201141054-1d33003b3869 github.com/klauspost/compress v1.13.6 // indirect github.com/klauspost/pgzip v1.2.5 @@ -84,7 +84,7 @@ require ( github.com/armon/go-metrics v0.4.1 // indirect github.com/armon/go-radix v1.0.0 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect - github.com/aws/aws-sdk-go v1.44.114 // indirect + github.com/aws/aws-sdk-go v1.45.6 // indirect github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect github.com/bgentry/speakeasy v0.1.0 // indirect github.com/bmatcuk/doublestar v1.1.5 // indirect @@ -169,6 +169,8 @@ require ( github.com/tklauser/go-sysconf v0.3.11 // indirect github.com/tklauser/numcpus v0.6.0 // indirect github.com/ugorji/go/codec v1.2.6 // indirect + github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect + github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect diff --git a/go.sum b/go.sum index f2f214db79f..05abe1fd237 100644 --- a/go.sum +++ b/go.sum @@ -64,8 +64,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkY github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go v1.31.9/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= -github.com/aws/aws-sdk-go v1.44.114 h1:plIkWc/RsHr3DXBj4MEw9sEW4CcL/e2ryokc+CKyq1I= -github.com/aws/aws-sdk-go v1.44.114/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go v1.45.6 h1:Y2isQQBZsnO15dzUQo9YQRThtHgrV200XCH05BRHVJI= +github.com/aws/aws-sdk-go v1.45.6/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= @@ -305,8 +305,8 @@ github.com/hashicorp/memberlist v0.5.0 h1:EtYPN8DpAURiapus508I4n9CzHs2W+8NZGbmmR github.com/hashicorp/memberlist v0.5.0/go.mod h1:yvyXLpo0QaGE59Y7hDTsTzDD25JYBZ4mHgHUZ8lrOI0= github.com/hashicorp/packer-plugin-amazon v1.2.1 h1:0Xqr8KsTJJhIo0vvjqPYrVMgyVxNRuYH4DeB5m/WAtw= github.com/hashicorp/packer-plugin-amazon v1.2.1/go.mod h1:qlp0h5TWVGgcPzN9mSxPiEAwOUOW3XU/zep0pGd0ZsM= -github.com/hashicorp/packer-plugin-sdk v0.5.4 h1:5Bl5DMEa//G4gBNcl842JopM9L4KSSsxpvB4W1lEwIA= -github.com/hashicorp/packer-plugin-sdk v0.5.4/go.mod h1:ALm0ZIK3c/F4iOqPNi7xVuHTgrR5dxzOK+DhFN5DHj4= +github.com/hashicorp/packer-plugin-sdk v0.5.5-0.20240715190433-28e1cb00bc48 h1:MxQWh/2TOwX7j+iAkDzvWV2+YlZokDr4zQfaEp7nnYM= +github.com/hashicorp/packer-plugin-sdk v0.5.5-0.20240715190433-28e1cb00bc48/go.mod h1:ALm0ZIK3c/F4iOqPNi7xVuHTgrR5dxzOK+DhFN5DHj4= github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY= github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4= github.com/hashicorp/vault/api v1.14.0 h1:Ah3CFLixD5jmjusOgm8grfN9M0d+Y8fVR2SW0K6pJLU= @@ -528,7 +528,9 @@ github.com/ugorji/go/codec v1.2.6/go.mod h1:V6TCNZ4PHqoHGFZuSG1W8nrCzzdgA2DozYxW github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= @@ -604,8 +606,8 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= @@ -651,13 +653,13 @@ golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -667,6 +669,7 @@ golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= diff --git a/packer/plugin.go b/packer/plugin.go index 79326f23af2..15d769014d3 100644 --- a/packer/plugin.go +++ b/packer/plugin.go @@ -30,6 +30,9 @@ type PluginConfig struct { PostProcessors PostProcessorSet DataSources DatasourceSet ReleasesOnly bool + // UseProtobuf is set if all the plugin candidates support protobuf, and + // the user has not forced usage of gob for serialisation. + UseProtobuf bool } // PACKERSPACE is used to represent the spaces that separate args for a command @@ -118,6 +121,10 @@ func (c *PluginConfig) Discover() error { return nil } +const ForceGobEnvvar = "PACKER_FORCE_GOB" + +var PackerUseProto = true + // DiscoverMultiPlugin takes the description from a multi-component plugin // binary and makes the plugins available to use in Packer. Each plugin found in the // binary will be addressable using `${pluginName}-${builderName}` for example. @@ -131,6 +138,18 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error return fmt.Errorf("failed to get plugin description from executable %q: %s", pluginPath, err) } + canProto := desc.ProtocolVersion == "v2" + if os.Getenv(ForceGobEnvvar) != "" && os.Getenv(ForceGobEnvvar) != "0" { + canProto = false + } + + // Keeps track of whether or not the plugin had components registered + // + // If no components are registered, we don't need to clamp usage of + // protobuf regardless if the plugin supports it or not, as we won't + // use it at all. + registered := false + pluginPrefix := pluginName + "-" pluginDetails := PluginDetails{ Name: pluginName, @@ -147,8 +166,17 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error if c.Builders.Has(key) { continue } + registered = true + c.Builders.Set(key, func() (packersdk.Builder, error) { - return c.Client(pluginPath, "start", "builder", builderName).Builder() + args := []string{"start", "builder"} + + if PackerUseProto { + args = append(args, "--protobuf") + } + args = append(args, builderName) + + return c.Client(pluginPath, args...).Builder() }) GlobalPluginsDetailsStore.SetBuilder(key, pluginDetails) } @@ -166,8 +194,17 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error if c.PostProcessors.Has(key) { continue } + registered = true + c.PostProcessors.Set(key, func() (packersdk.PostProcessor, error) { - return c.Client(pluginPath, "start", "post-processor", postProcessorName).PostProcessor() + args := []string{"start", "post-processor"} + + if PackerUseProto { + args = append(args, "--protobuf") + } + args = append(args, postProcessorName) + + return c.Client(pluginPath, args...).PostProcessor() }) GlobalPluginsDetailsStore.SetPostProcessor(key, pluginDetails) } @@ -185,8 +222,17 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error if c.Provisioners.Has(key) { continue } + registered = true + c.Provisioners.Set(key, func() (packersdk.Provisioner, error) { - return c.Client(pluginPath, "start", "provisioner", provisionerName).Provisioner() + args := []string{"start", "provisioner"} + + if PackerUseProto { + args = append(args, "--protobuf") + } + args = append(args, provisionerName) + + return c.Client(pluginPath, args...).Provisioner() }) GlobalPluginsDetailsStore.SetProvisioner(key, pluginDetails) @@ -204,8 +250,17 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error if c.DataSources.Has(key) { continue } + registered = true + c.DataSources.Set(key, func() (packersdk.Datasource, error) { - return c.Client(pluginPath, "start", "datasource", datasourceName).Datasource() + args := []string{"start", "datasource"} + + if PackerUseProto { + args = append(args, "--protobuf") + } + args = append(args, datasourceName) + + return c.Client(pluginPath, args...).Datasource() }) GlobalPluginsDetailsStore.SetDataSource(key, pluginDetails) } @@ -213,6 +268,14 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error log.Printf("found external %v datasource from %s plugin", desc.Datasources, pluginName) } + // Only print the log once, for the plugin that triggers that + // limitation in functionality. Otherwise this could be a bit + // verbose to print it for each non-compatible plugin. + if registered && !canProto && PackerUseProto { + log.Printf("plugin %q does not support Protobuf, forcing use of Gob", pluginPath) + PackerUseProto = false + } + return nil } diff --git a/packer/plugin_client.go b/packer/plugin_client.go index e230edddb33..dcd394bdf8e 100644 --- a/packer/plugin_client.go +++ b/packer/plugin_client.go @@ -417,6 +417,7 @@ func (c *PluginClient) Client() (*packerrpc.Client, error) { conn.Close() return nil, err } + client.UseProto = PackerUseProto return client, nil } diff --git a/scripts/generate-plugins.go b/scripts/generate-plugins.go index b31306ae12e..09b3cb27ba9 100755 --- a/scripts/generate-plugins.go +++ b/scripts/generate-plugins.go @@ -266,14 +266,15 @@ const source = `// package command import ( + "flag" "fmt" - "log" "regexp" "strings" "github.com/hashicorp/packer/packer" packersdk "github.com/hashicorp/packer-plugin-sdk/packer" "github.com/hashicorp/packer-plugin-sdk/plugin" + "github.com/hashicorp/packer-plugin-sdk/rpc" IMPORTS ) @@ -292,18 +293,46 @@ DATASOURCES var pluginRegexp = regexp.MustCompile("packer-(builder|post-processor|provisioner|datasource)-(.+)") -func (c *ExecuteCommand) Run(args []string) int { - // This is an internal call (users should not call this directly) so we're - // not going to do much input validation. If there's a problem we'll often - // just crash. Error handling should be added to facilitate debugging. - log.Printf("args: %#v", args) +type ExecuteArgs struct { + UseProtobuf bool + CommandType string +} + +func (ea *ExecuteArgs) AddFlagSets(flags *flag.FlagSet) { + flags.BoolVar(&ea.UseProtobuf, "protobuf", false, "Use protobuf for serialising data over the wire instead of gob") +} + +func (c *ExecuteCommand) ParseArgs(args []string) (*ExecuteArgs, int) { + var cfg ExecuteArgs + flags := c.Meta.FlagSet("") + flags.Usage = func() { c.Ui.Say(c.Help()) } + cfg.AddFlagSets(flags) + if err := flags.Parse(args); err != nil { + return &cfg, 1 + } + + args = flags.Args() if len(args) != 1 { - c.Ui.Error(c.Help()) - return 1 + flags.Usage() + return &cfg, 1 } + cfg.CommandType = args[0] + return &cfg, 0 +} +func (c *ExecuteCommand) Run(args []string) int { + cfg, ret := c.ParseArgs(args) + if ret != 0 { + return ret + } + + return c.RunContext(cfg) +} + + +func (c *ExecuteCommand) RunContext(args *ExecuteArgs) int { // Plugin will match something like "packer-builder-amazon-ebs" - parts := pluginRegexp.FindStringSubmatch(args[0]) + parts := pluginRegexp.FindStringSubmatch(args.CommandType) if len(parts) != 3 { c.Ui.Error(c.Help()) return 1 @@ -317,6 +346,10 @@ func (c *ExecuteCommand) Run(args []string) int { return 1 } + if args.UseProtobuf { + server.UseProto = true + } + switch pluginType { case "builder": builder, found := Builders[pluginName] @@ -355,11 +388,15 @@ func (c *ExecuteCommand) Run(args []string) int { func (*ExecuteCommand) Help() string { helpText := ` + "`" + ` -Usage: packer execute PLUGIN +Usage: packer execute [options] PLUGIN Runs an internally-compiled version of a plugin from the packer binary. NOTE: this is an internal command and you should not call it yourself. + +Options: + + --protobuf: use protobuf for serialising data over-the-wire instead of gob. ` + "`" + ` return strings.TrimSpace(helpText)