rpc: allow using both gob/protobuf for structs

As follow-up to the introduction of the protobufs for HCLSpec, we
introduce a new environment variable and code to use those structures,
so we don't use gob for serialising HCLSpecs.

This should make the plugins and packer able to transmit data
over-the-wire without using gob for the most part (the communicators
still use it, and will probably need some work to replace).

Makefile

@@ -59,7 +59,7 @@ generate: install-gen-deps ## Generate dynamically generated code
 	@find ./ -type f | xargs grep -l '^// Code generated' | xargs rm -f
 	PROJECT_ROOT="$(CURDIR)" go generate ./...
 	go fmt bootcommand/boot_command.go
-# 	go run ./cmd/generate-fixer-deprecations
+	protoc rpc/hcl_spec.proto --go_out=. --go_opt=paths=source_relative
 
 generate-check: generate ## Check go code generation is on par
 	@echo "==> Checking that auto-generated code is not changed..."

plugin/set.go

@@ -12,6 +12,7 @@ import (
 	"sort"
 
 	packersdk "github.com/hashicorp/packer-plugin-sdk/packer"
+	"github.com/hashicorp/packer-plugin-sdk/rpc"
 	pluginVersion "github.com/hashicorp/packer-plugin-sdk/version"
 )
 
@@ -106,11 +107,39 @@ func (i *Set) Run() error {
 	return i.RunCommand(args...)
 }
 
+func useProtobuf(args ...string) []string {
+	protobufPos := -1
+	for i, arg := range args {
+		if arg == "--protobuf" {
+			protobufPos = i
+			break
+		}
+	}
+
+	if protobufPos == -1 {
+		return args
+	}
+
+	rpc.UseProto = true
+
+	if protobufPos == 0 {
+		return args[1:]
+	}
+
+	if protobufPos == len(args)-1 {
+		return args[:len(args)-1]
+	}
+
+	return append(args[:protobufPos], args[protobufPos+1:]...)
+}
+
 func (i *Set) RunCommand(args ...string) error {
 	if len(args) < 1 {
 		return fmt.Errorf("needs at least one argument")
 	}
 
+	args = useProtobuf(args...)
+
 	switch args[0] {
 	case "describe":
 		return i.jsonDescribe(os.Stdout)

rpc/common.go

@@ -7,12 +7,21 @@ import (
 	"bytes"
 	"encoding/gob"
 	"fmt"
+	"log"
 	"net/rpc"
+	"reflect"
 
 	"github.com/hashicorp/hcl/v2/hcldec"
 	"github.com/zclconf/go-cty/cty"
+	"google.golang.org/protobuf/proto"
 )
 
+// UseProto lets us determine whether or not we should use protobuf for serialising
+// data over RPC instead of gob.
+//
+// This is controlled by Packer using the `--use-proto` flag on plugin commands.
+var UseProto bool = false
+
 // commonClient allows to rpc call funcs that can be defined on the different
 // build blocks of Packer.
 type commonClient struct {
@@ -48,21 +57,91 @@ func (p *commonClient) ConfigSpec() hcldec.ObjectSpec {
 		panic(err.Error())
 	}
 
-	res := hcldec.ObjectSpec{}
-	err := gob.NewDecoder(bytes.NewReader(resp.ConfigSpec)).Decode(&res)
+	// Legacy: this will need to be removed when we discontinue gob-encoding
+	//
+	// This is required for backwards compatibility for now, but using
+	// gob to encode the spec objects will fail against the upstream cty
+	// library, since they removed support for it.
+	//
+	// This will be a breaking change, as older plugins won't be able to
+	// communicate with Packer any longer.
+	if !UseProto {
+		log.Printf("[DEBUG] - common: receiving ConfigSpec as gob")
+		res := hcldec.ObjectSpec{}
+		err := gob.NewDecoder(bytes.NewReader(resp.ConfigSpec)).Decode(&res)
+		if err != nil {
+			panic(fmt.Errorf("failed to decode HCL spec from gob: %s", err))
+		}
+		return res
+	}
+
+	log.Printf("[DEBUG] - common: receiving ConfigSpec as protobuf")
+	spec, err := protobufToHCL2Spec(resp.ConfigSpec)
 	if err != nil {
-		panic("ici:" + err.Error())
+		panic(err)
 	}
-	return res
+
+	return spec
 }
 
 func (s *commonServer) ConfigSpec(_ interface{}, reply *ConfigSpecResponse) error {
 	spec := s.selfConfigurable.ConfigSpec()
-	b := bytes.NewBuffer(nil)
-	err := gob.NewEncoder(b).Encode(spec)
-	reply.ConfigSpec = b.Bytes()
 
-	return err
+	if !UseProto {
+		log.Printf("[DEBUG] - common: sending ConfigSpec as gob")
+		b := &bytes.Buffer{}
+		err := gob.NewEncoder(b).Encode(spec)
+		if err != nil {
+			return fmt.Errorf("failed to encode spec from gob: %s", err)
+		}
+		reply.ConfigSpec = b.Bytes()
+
+		return nil
+	}
+
+	log.Printf("[DEBUG] - common: sending ConfigSpec as protobuf")
+	rawBytes, err := hcl2SpecToProtobuf(spec)
+	if err != nil {
+		return fmt.Errorf("failed to encode HCL spec from protobuf: %s", err)
+	}
+	reply.ConfigSpec = rawBytes
+
+	return nil
+}
+
+// hcl2SpecToProtobuf converts a hcldec.ObjectSpec to a protobuf-serialised
+// byte array so it can then be used to send to a Plugin/Packer.
+func hcl2SpecToProtobuf(spec hcldec.ObjectSpec) ([]byte, error) {
+	ret, err := ToProto(spec)
+	if err != nil {
+		return nil, fmt.Errorf("failed to convert hcldec.Spec to hclspec.Spec: %s", err)
+	}
+	rawBytes, err := proto.Marshal(ret)
+	if err != nil {
+		return nil, fmt.Errorf("failed to serialise hclspec.Spec to protobuf: %s", err)
+	}
+
+	return rawBytes, nil
+}
+
+// protobufToHCL2Spec converts a protobuf-encoded spec to a usable hcldec.Spec.
+func protobufToHCL2Spec(serData []byte) (hcldec.ObjectSpec, error) {
+	confSpec := &Spec{}
+	err := proto.Unmarshal(serData, confSpec)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal hclspec.Spec from raw protobuf: %q", err)
+	}
+	spec, err := confSpec.FromProto()
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode HCL spec: %q", err)
+	}
+
+	obj, ok := spec.(*hcldec.ObjectSpec)
+	if !ok {
+		return nil, fmt.Errorf("decoded HCL spec is not an object spec: %s", reflect.TypeOf(spec).String())
+	}
+
+	return *obj, nil
 }
 
 func init() {

rpc/datasource.go

@@ -7,10 +7,12 @@ import (
 	"bytes"
 	"encoding/gob"
 	"fmt"
+	"log"
 
 	"github.com/hashicorp/hcl/v2/hcldec"
 	"github.com/hashicorp/packer-plugin-sdk/packer"
 	"github.com/zclconf/go-cty/cty"
+	"github.com/zclconf/go-cty/cty/msgpack"
 )
 
 // An implementation of packer.Datasource where the data source is actually
@@ -52,10 +54,21 @@ func (d *datasource) OutputSpec() hcldec.ObjectSpec {
 		err := fmt.Errorf("Datasource.OutputSpec failed: %v", err)
 		panic(err.Error())
 	}
-	res := hcldec.ObjectSpec{}
-	err := gob.NewDecoder(bytes.NewReader(resp.OutputSpec)).Decode(&res)
+
+	if !UseProto {
+		log.Printf("[DEBUG] - datasource: receiving OutputSpec as gob")
+		res := hcldec.ObjectSpec{}
+		err := gob.NewDecoder(bytes.NewReader(resp.OutputSpec)).Decode(&res)
+		if err != nil {
+			panic(fmt.Sprintf("datasource: failed to deserialise HCL spec from gob: %s", err))
+		}
+		return res
+	}
+
+	log.Printf("[DEBUG] - datasource: receiving OutputSpec as gob")
+	res, err := protobufToHCL2Spec(resp.OutputSpec)
 	if err != nil {
-		panic("ici:" + err.Error())
+		panic(fmt.Sprintf("datasource: failed to deserialise HCL spec from protobuf: %s", err))
 	}
 	return res
 }
@@ -66,20 +79,35 @@ type ExecuteResponse struct {
 }
 
 func (d *datasource) Execute() (cty.Value, error) {
-	res := new(cty.Value)
 	resp := new(ExecuteResponse)
 	if err := d.client.Call(d.endpoint+".Execute", new(interface{}), resp); err != nil {
 		err := fmt.Errorf("Datasource.Execute failed: %v", err)
-		return *res, err
+		return cty.NilVal, err
+	}
+
+	if !UseProto {
+		log.Printf("[DEBUG] - datasource: receiving Execute as gob")
+		res := cty.Value{}
+		err := gob.NewDecoder(bytes.NewReader(resp.Value)).Decode(&res)
+		if err != nil {
+			return res, fmt.Errorf("failed to unmarshal cty.Value from gob blob: %s", err)
+		}
+		if resp.Error != nil {
+			err = resp.Error
+		}
+		return res, err
 	}
-	err := gob.NewDecoder(bytes.NewReader(resp.Value)).Decode(&res)
+
+	log.Printf("[DEBUG] - datasource: receiving Execute as msgpack")
+	res, err := msgpack.Unmarshal(resp.Value, cty.DynamicPseudoType)
 	if err != nil {
-		return *res, err
+		return cty.NilVal, fmt.Errorf("failed to unmarshal cty.Value from msgpack blob: %s", err)
 	}
+
 	if resp.Error != nil {
 		err = resp.Error
 	}
-	return *res, err
+	return res, err
 }
 
 // DatasourceServer wraps a packer.Datasource implementation and makes it
@@ -103,18 +131,43 @@ func (d *DatasourceServer) Configure(args *DatasourceConfigureArgs, reply *Datas
 
 func (d *DatasourceServer) OutputSpec(args *DatasourceConfigureArgs, reply *OutputSpecResponse) error {
 	spec := d.d.OutputSpec()
-	b := bytes.NewBuffer(nil)
-	err := gob.NewEncoder(b).Encode(spec)
-	reply.OutputSpec = b.Bytes()
+
+	if !UseProto {
+		log.Printf("[DEBUG] - datasource: sending OutputSpec as gob")
+		b := &bytes.Buffer{}
+		err := gob.NewEncoder(b).Encode(spec)
+		reply.OutputSpec = b.Bytes()
+		return err
+	}
+
+	log.Printf("[DEBUG] - datasource: sending OutputSpec as protobuf")
+	ret, err := hcl2SpecToProtobuf(spec)
+	if err != nil {
+		return err
+	}
+	reply.OutputSpec = ret
+
 	return err
 }
 
 func (d *DatasourceServer) Execute(args *interface{}, reply *ExecuteResponse) error {
 	spec, err := d.d.Execute()
 	reply.Error = NewBasicError(err)
-	b := bytes.NewBuffer(nil)
-	err = gob.NewEncoder(b).Encode(spec)
-	reply.Value = b.Bytes()
+
+	if !UseProto {
+		log.Printf("[DEBUG] - datasource: sending Execute as gob")
+		b := &bytes.Buffer{}
+		err = gob.NewEncoder(b).Encode(spec)
+		reply.Value = b.Bytes()
+		if reply.Error != nil {
+			err = reply.Error
+		}
+		return err
+	}
+
+	log.Printf("[DEBUG] - datasource: sending Execute as msgpack")
+	raw, err := msgpack.Marshal(spec, cty.DynamicPseudoType)
+	reply.Value = raw
 	if reply.Error != nil {
 		err = reply.Error
 	}