diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml index 08188e34d..541941d66 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/go-test.yml @@ -41,6 +41,9 @@ jobs: run: | mkdir -p ${{ env.TEST_RESULTS_PATH }}/packer-plugin-sdk + - name: Install buf + uses: bufbuild/buf-setup-action@v1.33.0 + - name: Run gofmt run: | make fmt-check diff --git a/Makefile b/Makefile index 47ca7ee29..030088a43 100644 --- a/Makefile +++ b/Makefile @@ -59,7 +59,7 @@ generate: install-gen-deps ## Generate dynamically generated code @find ./ -type f | xargs grep -l '^// Code generated' | xargs rm -f PROJECT_ROOT="$(CURDIR)" go generate ./... go fmt bootcommand/boot_command.go -# go run ./cmd/generate-fixer-deprecations + buf generate generate-check: generate ## Check go code generation is on par @echo "==> Checking that auto-generated code is not changed..." diff --git a/buf.gen.yaml b/buf.gen.yaml new file mode 100644 index 000000000..13dca9e9a --- /dev/null +++ b/buf.gen.yaml @@ -0,0 +1,7 @@ +version: v2 +plugins: +- remote: buf.build/protocolbuffers/go:v1.28.1 + out: rpc + opt: paths=source_relative +inputs: +- proto_file: rpc/hcl_spec.proto diff --git a/plugin/set.go b/plugin/set.go index 9211e7ea5..d9f10f1cd 100644 --- a/plugin/set.go +++ b/plugin/set.go @@ -33,21 +33,31 @@ type Set struct { version string sdkVersion string apiVersion string + useProto bool Builders map[string]packersdk.Builder PostProcessors map[string]packersdk.PostProcessor Provisioners map[string]packersdk.Provisioner Datasources map[string]packersdk.Datasource } +// ProtocolVersion2 serves as a compatibility argument to the SetDescription +// so plugins can report whether or not they support protobuf/msgpack for +// serialising some of their entities (typically ObjectSpec) to protobuf. +// +// If absent from the SetDescription, it means only gob is supported, and both +// Packer and the plugins should use that for communication. +const ProtocolVersion2 = "v2" + // SetDescription describes a Set. type SetDescription struct { - Version string `json:"version"` - SDKVersion string `json:"sdk_version"` - APIVersion string `json:"api_version"` - Builders []string `json:"builders"` - PostProcessors []string `json:"post_processors"` - Provisioners []string `json:"provisioners"` - Datasources []string `json:"datasources"` + Version string `json:"version"` + SDKVersion string `json:"sdk_version"` + APIVersion string `json:"api_version"` + Builders []string `json:"builders"` + PostProcessors []string `json:"post_processors"` + Provisioners []string `json:"provisioners"` + Datasources []string `json:"datasources"` + ProtocolVersion string `json:"protocol_version"` } //// @@ -106,11 +116,28 @@ func (i *Set) Run() error { return i.RunCommand(args...) } +// parseProtobufFlag walks over the args to find if `--protobuf` is set. +// +// It then returns the args without it for the commands to process them. +func (i *Set) parseProtobufFlag(args ...string) []string { + parsedArgs := make([]string, 0, len(args)) + for _, arg := range args { + if arg == "--protobuf" { + i.useProto = true + continue + } + parsedArgs = append(parsedArgs, arg) + } + return parsedArgs +} + func (i *Set) RunCommand(args ...string) error { if len(args) < 1 { return fmt.Errorf("needs at least one argument") } + args = i.parseProtobufFlag(args...) + switch args[0] { case "describe": return i.jsonDescribe(os.Stdout) @@ -130,6 +157,7 @@ func (i *Set) start(kind, name string) error { if err != nil { return err } + server.UseProto = i.useProto log.Printf("[TRACE] starting %s %s", kind, name) @@ -158,13 +186,14 @@ func (i *Set) start(kind, name string) error { func (i *Set) description() SetDescription { return SetDescription{ - Version: i.version, - SDKVersion: i.sdkVersion, - APIVersion: i.apiVersion, - Builders: i.buildersDescription(), - PostProcessors: i.postProcessorsDescription(), - Provisioners: i.provisionersDescription(), - Datasources: i.datasourceDescription(), + Version: i.version, + SDKVersion: i.sdkVersion, + APIVersion: i.apiVersion, + Builders: i.buildersDescription(), + PostProcessors: i.postProcessorsDescription(), + Provisioners: i.provisionersDescription(), + Datasources: i.datasourceDescription(), + ProtocolVersion: ProtocolVersion2, } } diff --git a/plugin/set_test.go b/plugin/set_test.go index f29ecc8f4..3490dc211 100644 --- a/plugin/set_test.go +++ b/plugin/set_test.go @@ -52,13 +52,14 @@ func TestSet(t *testing.T) { sdkVersion := pluginVersion.NewPluginVersion(pluginVersion.Version, pluginVersion.VersionPrerelease, "") if diff := cmp.Diff(SetDescription{ - Version: "1.1.1", - SDKVersion: sdkVersion.String(), - APIVersion: "x" + APIVersionMajor + "." + APIVersionMinor, - Builders: []string{"example", "example-2"}, - PostProcessors: []string{"example", "example-2"}, - Provisioners: []string{"example", "example-2"}, - Datasources: []string{"example", "example-2"}, + Version: "1.1.1", + SDKVersion: sdkVersion.String(), + APIVersion: "x" + APIVersionMajor + "." + APIVersionMinor, + Builders: []string{"example", "example-2"}, + PostProcessors: []string{"example", "example-2"}, + Provisioners: []string{"example", "example-2"}, + Datasources: []string{"example", "example-2"}, + ProtocolVersion: ProtocolVersion2, }, outputDesc); diff != "" { t.Fatalf("Unexpected description: %s", diff) } @@ -68,3 +69,58 @@ func TestSet(t *testing.T) { t.Fatalf("Unexpected error: %s", diff) } } + +func TestSetProtobufArgParsing(t *testing.T) { + testCases := []struct { + name string + useProto bool + in, out []string + }{ + { + name: "no --protobuf argument provided", + in: []string{"start", "builder", "example"}, + out: []string{"start", "builder", "example"}, + useProto: false, + }, + { + name: "providing --protobuf as first argument", + in: []string{"--protobuf", "start", "builder", "example"}, + out: []string{"start", "builder", "example"}, + useProto: true, + }, + { + name: "providing --protobuf as last argument", + in: []string{"start", "builder", "example", "--protobuf"}, + out: []string{"start", "builder", "example"}, + useProto: true, + }, + { + name: "providing --protobuf as middle argument", + in: []string{"start", "builder", "--protobuf", "example"}, + out: []string{"start", "builder", "example"}, + useProto: true, + }, + { + name: "providing --protobuf multiple times", + in: []string{"--protobuf", "start", "builder", "--protobuf", "example", "--protobuf"}, + out: []string{"start", "builder", "example"}, + useProto: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + set := NewSet() + got := set.parseProtobufFlag(tc.in...) + + if diff := cmp.Diff(got, tc.out); diff != "" { + t.Errorf("Unexpected args: %s", diff) + } + + if set.useProto != tc.useProto { + t.Errorf("expected useProto to be %t when %s but got %t", tc.useProto, tc.name, set.useProto) + } + }) + + } +} diff --git a/rpc/client.go b/rpc/client.go index 5c660c638..f4bba8cd4 100644 --- a/rpc/client.go +++ b/rpc/client.go @@ -21,6 +21,9 @@ type Client struct { mux *muxBroker client *rpc.Client closeMux bool + // UseProto makes it so that clients started from this will use + // protobuf/msgpack for serialisation instead of gob + UseProto bool } func NewClient(rwc io.ReadWriteCloser) (*Client, error) { @@ -76,6 +79,13 @@ func (c *Client) Artifact() packer.Artifact { commonClient: commonClient{ endpoint: DefaultArtifactEndpoint, client: c.client, + // Setting useProto to false is essentially a noop for + // this type of client since they don't exchange cty + // values, and there's no HCLSpec object tied to this. + // + // For documentation purposes though, we keep it visible + // in order to change this later if it becomes relevant. + useProto: false, }, } } @@ -86,6 +96,13 @@ func (c *Client) Build() packer.Build { endpoint: DefaultBuildEndpoint, client: c.client, mux: c.mux, + // Setting useProto to false is essentially a noop for + // this type of client since they don't exchange cty + // values, and there's no HCLSpec object tied to this. + // + // For documentation purposes though, we keep it visible + // in order to change this later if it becomes relevant. + useProto: false, }, } } @@ -96,6 +113,7 @@ func (c *Client) Builder() packer.Builder { endpoint: DefaultBuilderEndpoint, client: c.client, mux: c.mux, + useProto: c.UseProto, }, } } @@ -106,6 +124,13 @@ func (c *Client) Communicator() packer.Communicator { endpoint: DefaultCommunicatorEndpoint, client: c.client, mux: c.mux, + // Setting useProto to false is essentially a noop for + // this type of client since they don't exchange cty + // values, and there's no HCLSpec object tied to this. + // + // For documentation purposes though, we keep it visible + // in order to change this later if it becomes relevant. + useProto: false, }, } } @@ -116,6 +141,13 @@ func (c *Client) Hook() packer.Hook { endpoint: DefaultHookEndpoint, client: c.client, mux: c.mux, + // Setting useProto to false is essentially a noop for + // this type of client since they don't exchange cty + // values, and there's no HCLSpec object tied to this. + // + // For documentation purposes though, we keep it visible + // in order to change this later if it becomes relevant. + useProto: false, }, } } @@ -126,6 +158,7 @@ func (c *Client) PostProcessor() packer.PostProcessor { endpoint: DefaultPostProcessorEndpoint, client: c.client, mux: c.mux, + useProto: c.UseProto, }, } } @@ -136,6 +169,7 @@ func (c *Client) Provisioner() packer.Provisioner { endpoint: DefaultProvisionerEndpoint, client: c.client, mux: c.mux, + useProto: c.UseProto, }, } } @@ -146,6 +180,7 @@ func (c *Client) Datasource() packer.Datasource { endpoint: DefaultDatasourceEndpoint, client: c.client, mux: c.mux, + useProto: c.UseProto, }, } } @@ -155,6 +190,13 @@ func (c *Client) Ui() packer.Ui { commonClient: commonClient{ endpoint: DefaultUiEndpoint, client: c.client, + // Setting useProto to false is essentially a noop for + // this type of client since they don't exchange cty + // values, and there's no HCLSpec object tied to this. + // + // For documentation purposes though, we keep it visible + // in order to change this later if it becomes relevant. + useProto: false, }, endpoint: DefaultUiEndpoint, } diff --git a/rpc/common.go b/rpc/common.go index 2a0dadc34..a97193b50 100644 --- a/rpc/common.go +++ b/rpc/common.go @@ -7,10 +7,13 @@ import ( "bytes" "encoding/gob" "fmt" + "log" "net/rpc" + "reflect" "github.com/hashicorp/hcl/v2/hcldec" "github.com/zclconf/go-cty/cty" + "google.golang.org/protobuf/proto" ) // commonClient allows to rpc call funcs that can be defined on the different @@ -22,6 +25,12 @@ type commonClient struct { endpoint string client *rpc.Client mux *muxBroker + + // useProto lets us determine whether or not we should use protobuf for serialising + // data over RPC instead of gob. + // + // This is controlled by Packer using the `--use-proto` flag on plugin commands. + useProto bool } type commonServer struct { @@ -30,6 +39,12 @@ type commonServer struct { selfConfigurable interface { ConfigSpec() hcldec.ObjectSpec } + + // useProto lets us determine whether or not we should use protobuf for serialising + // data over RPC instead of gob. + // + // This is controlled by Packer using the `--use-proto` flag on plugin commands. + useProto bool } type ConfigSpecResponse struct { @@ -48,21 +63,91 @@ func (p *commonClient) ConfigSpec() hcldec.ObjectSpec { panic(err.Error()) } - res := hcldec.ObjectSpec{} - err := gob.NewDecoder(bytes.NewReader(resp.ConfigSpec)).Decode(&res) + // Legacy: this will need to be removed when we discontinue gob-encoding + // + // This is required for backwards compatibility for now, but using + // gob to encode the spec objects will fail against the upstream cty + // library, since they removed support for it. + // + // This will be a breaking change, as older plugins won't be able to + // communicate with Packer any longer. + if !p.useProto { + log.Printf("[DEBUG] - common: receiving ConfigSpec as gob") + res := hcldec.ObjectSpec{} + err := gob.NewDecoder(bytes.NewReader(resp.ConfigSpec)).Decode(&res) + if err != nil { + panic(fmt.Errorf("failed to decode HCL spec from gob: %s", err)) + } + return res + } + + log.Printf("[DEBUG] - common: receiving ConfigSpec as protobuf") + spec, err := protobufToHCL2Spec(resp.ConfigSpec) if err != nil { - panic("ici:" + err.Error()) + panic(err) } - return res + + return spec } func (s *commonServer) ConfigSpec(_ interface{}, reply *ConfigSpecResponse) error { spec := s.selfConfigurable.ConfigSpec() - b := bytes.NewBuffer(nil) - err := gob.NewEncoder(b).Encode(spec) - reply.ConfigSpec = b.Bytes() - return err + if !s.useProto { + log.Printf("[DEBUG] - common: sending ConfigSpec as gob") + b := &bytes.Buffer{} + err := gob.NewEncoder(b).Encode(spec) + if err != nil { + return fmt.Errorf("failed to encode spec from gob: %s", err) + } + reply.ConfigSpec = b.Bytes() + + return nil + } + + log.Printf("[DEBUG] - common: sending ConfigSpec as protobuf") + rawBytes, err := hcl2SpecToProtobuf(spec) + if err != nil { + return fmt.Errorf("failed to encode HCL spec from protobuf: %s", err) + } + reply.ConfigSpec = rawBytes + + return nil +} + +// hcl2SpecToProtobuf converts a hcldec.ObjectSpec to a protobuf-serialised +// byte array so it can then be used to send to a Plugin/Packer. +func hcl2SpecToProtobuf(spec hcldec.ObjectSpec) ([]byte, error) { + ret, err := ToProto(spec) + if err != nil { + return nil, fmt.Errorf("failed to convert hcldec.Spec to hclspec.Spec: %s", err) + } + rawBytes, err := proto.Marshal(ret) + if err != nil { + return nil, fmt.Errorf("failed to serialise hclspec.Spec to protobuf: %s", err) + } + + return rawBytes, nil +} + +// protobufToHCL2Spec converts a protobuf-encoded spec to a usable hcldec.Spec. +func protobufToHCL2Spec(serData []byte) (hcldec.ObjectSpec, error) { + confSpec := &Spec{} + err := proto.Unmarshal(serData, confSpec) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal hclspec.Spec from raw protobuf: %q", err) + } + spec, err := confSpec.FromProto() + if err != nil { + return nil, fmt.Errorf("failed to decode HCL spec: %q", err) + } + + obj, ok := spec.(*hcldec.ObjectSpec) + if !ok { + return nil, fmt.Errorf("decoded HCL spec is not an object spec: %s", reflect.TypeOf(spec).String()) + } + + return *obj, nil } func init() { diff --git a/rpc/convert.go b/rpc/convert.go new file mode 100644 index 000000000..12f397499 --- /dev/null +++ b/rpc/convert.go @@ -0,0 +1,263 @@ +package rpc + +import ( + "fmt" + "reflect" + + "github.com/hashicorp/hcl/v2/hcldec" + "github.com/zclconf/go-cty/cty" +) + +// ToProto converts a hcldec.Spec to a protobuf-serialisable equivalent. +// +// This can then be used for gRPC communication over-the-wire for Packer plugins. +func ToProto(spec hcldec.Spec) (*Spec, error) { + switch concreteSpec := spec.(type) { + case *hcldec.AttrSpec: + attr, err := attrSpecToProto(concreteSpec) + if err != nil { + return nil, fmt.Errorf("failed to decode attribute spec: %s", err) + } + return &Spec{ + Block: &Spec_Attr{ + Attr: attr, + }, + }, nil + case *hcldec.BlockListSpec: + blspec, err := blockListSpecToProto(concreteSpec) + if err != nil { + return nil, fmt.Errorf("failed to decode block list spec: %s", err) + } + return &Spec{ + Block: &Spec_BlockList{ + BlockList: blspec, + }, + }, nil + case hcldec.ObjectSpec: + objSpec, err := objectSpecToProto(&concreteSpec) + if err != nil { + return nil, fmt.Errorf("failed to decode object spec: %s", err) + } + return &Spec{ + Block: &Spec_Object{ + Object: objSpec, + }, + }, nil + case *hcldec.BlockSpec: + blockSpec, err := blockSpecToProto(concreteSpec) + if err != nil { + return nil, fmt.Errorf("failed to decode object spec: %s", err) + } + return &Spec{ + Block: &Spec_BlockValue{ + BlockValue: blockSpec, + }, + }, nil + } + + return nil, fmt.Errorf("unsupported hcldec.Spec type: %#v", reflect.TypeOf(spec).String()) +} + +func blockSpecToProto(bsp *hcldec.BlockSpec) (*Block, error) { + nst, err := ToProto(bsp.Nested) + if err != nil { + return nil, fmt.Errorf("failed to decode block nested spec: %s", err) + } + + return &Block{ + Name: bsp.TypeName, + Required: bsp.Required, + Nested: nst, + }, nil +} + +func objectSpecToProto(hsp *hcldec.ObjectSpec) (*Object, error) { + outSpec := &Object{ + Attributes: map[string]*Spec{}, + } + + for name, spec := range *hsp { + pSpec, err := ToProto(spec) + if err != nil { + return nil, fmt.Errorf("failed to convert object attribute %q to hcldec.Spec: %s", name, err) + } + outSpec.Attributes[name] = pSpec + } + + return outSpec, nil +} + +func blockListSpecToProto(blspec *hcldec.BlockListSpec) (*BlockList, error) { + nested, err := ToProto(blspec.Nested) + if err != nil { + return nil, fmt.Errorf("failed to decode block list nested type: %s", err) + } + + return &BlockList{ + Name: blspec.TypeName, + Nested: nested, + }, nil +} + +func attrSpecToProto(attrSpec *hcldec.AttrSpec) (*Attr, error) { + convertedType, err := ctyTypeToProto(attrSpec.Type) + if err != nil { + return nil, fmt.Errorf("failed to convert ctyType for attribute %q: %s", attrSpec.Name, err) + } + + return &Attr{ + Name: attrSpec.Name, + Required: attrSpec.Required, + Type: convertedType, + }, nil +} + +func ctyTypeToProto(cType cty.Type) (*CtyType, error) { + if cType.IsPrimitiveType() { + switch cType { + case cty.Bool, + cty.String, + cty.Number: + return &CtyType{ + TypeDef: &CtyType_Primitive{ + Primitive: &CtyPrimitive{ + TypeString: cType.GoString(), + }, + }, + }, nil + default: + return nil, fmt.Errorf("Unknown primitive type: %s", cType.FriendlyName()) + } + } + + if cType.IsListType() { + el := cType.ElementType() + elType, err := ctyTypeToProto(el) + if err != nil { + return nil, fmt.Errorf("failed to extract valid cty.Type from list element: %s", err) + } + return &CtyType{ + TypeDef: &CtyType_List{ + List: &CtyList{ + ElementType: elType, + }, + }, + }, nil + } + + // As per the specification, cty.Map are always a map from string to a cty type + // + // Therefore, we don't need to worry about other types than the element's + if cType.IsMapType() { + el := cType.MapElementType() + elType, err := ctyTypeToProto(*el) + if err != nil { + return nil, fmt.Errorf("failed to extract valid cty.Type from map: %s", err) + } + return &CtyType{ + TypeDef: &CtyType_Map{ + Map: &CtyMap{ + ElementType: elType, + }, + }, + }, nil + } + + return nil, fmt.Errorf("unsupported cty.Type conversion to protobuf-compatible structure: %+v", cType) +} + +func (spec *Spec) FromProto() (hcldec.Spec, error) { + switch realSpec := spec.Block.(type) { + case *Spec_Attr: + return protoArgToHCLDecSpec(realSpec.Attr) + case *Spec_BlockList: + return protoBlockListToHCLDecSpec(realSpec.BlockList) + case *Spec_BlockValue: + return protoBlockToHCLDecSpec(realSpec.BlockValue) + case *Spec_Object: + return protoObjectSpecToHCLDecSpec(realSpec.Object) + } + + return nil, fmt.Errorf("unsupported spec type: %s", spec.String()) +} + +func protoObjectSpecToHCLDecSpec(protoSpec *Object) (*hcldec.ObjectSpec, error) { + outSpec := hcldec.ObjectSpec{} + + for name, spec := range protoSpec.Attributes { + attrSpec, err := spec.FromProto() + if err != nil { + return nil, fmt.Errorf("failed to decode object attribute %q: %s", name, err) + } + outSpec[name] = attrSpec + } + + return &outSpec, nil +} + +func protoBlockToHCLDecSpec(bl *Block) (*hcldec.BlockSpec, error) { + nested, err := bl.Nested.FromProto() + if err != nil { + return nil, fmt.Errorf("failed to decode block nested type from proto: %s", err) + } + + return &hcldec.BlockSpec{ + TypeName: bl.Name, + Required: bl.Required, + Nested: nested, + }, nil +} + +func protoBlockListToHCLDecSpec(bll *BlockList) (*hcldec.BlockListSpec, error) { + blSpec := bll.Nested + nested, err := blSpec.FromProto() + if err != nil { + return nil, fmt.Errorf("failed to decode block list nested type from proto: %s", err) + } + + return &hcldec.BlockListSpec{ + TypeName: bll.Name, + Nested: nested, + }, nil +} + +func protoArgToHCLDecSpec(attr *Attr) (*hcldec.AttrSpec, error) { + relType, err := protoTypeToCtyType(attr.Type) + if err != nil { + return nil, fmt.Errorf("failed to convert type of attribute %q: %s", attr.Name, err) + } + + return &hcldec.AttrSpec{ + Name: attr.Name, + Required: attr.Required, + Type: relType, + }, nil +} + +func protoTypeToCtyType(protoType *CtyType) (cty.Type, error) { + switch concrete := protoType.TypeDef.(type) { + case *CtyType_Primitive: + switch concrete.Primitive.TypeString { + case "cty.String": + return cty.String, nil + case "cty.Bool": + return cty.Bool, nil + case "cty.Number": + return cty.Number, nil + } + case *CtyType_List: + elType, err := protoTypeToCtyType(concrete.List.ElementType) + if err != nil { + return cty.NilType, fmt.Errorf("failed to convert list element type: %s", err) + } + return cty.List(elType), nil + case *CtyType_Map: + elType, err := protoTypeToCtyType(concrete.Map.ElementType) + if err != nil { + return cty.NilType, fmt.Errorf("failed to convert map element type: %s", err) + } + return cty.Map(elType), nil + } + + return cty.NilType, fmt.Errorf("unsupported cty.Type: %+v", protoType) +} diff --git a/rpc/datasource.go b/rpc/datasource.go index 443eeee85..74b35ee0b 100644 --- a/rpc/datasource.go +++ b/rpc/datasource.go @@ -7,10 +7,12 @@ import ( "bytes" "encoding/gob" "fmt" + "log" "github.com/hashicorp/hcl/v2/hcldec" "github.com/hashicorp/packer-plugin-sdk/packer" "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/msgpack" ) // An implementation of packer.Datasource where the data source is actually @@ -52,10 +54,21 @@ func (d *datasource) OutputSpec() hcldec.ObjectSpec { err := fmt.Errorf("Datasource.OutputSpec failed: %v", err) panic(err.Error()) } - res := hcldec.ObjectSpec{} - err := gob.NewDecoder(bytes.NewReader(resp.OutputSpec)).Decode(&res) + + if !d.useProto { + log.Printf("[DEBUG] - datasource: receiving OutputSpec as gob") + res := hcldec.ObjectSpec{} + err := gob.NewDecoder(bytes.NewReader(resp.OutputSpec)).Decode(&res) + if err != nil { + panic(fmt.Sprintf("datasource: failed to deserialise HCL spec from gob: %s", err)) + } + return res + } + + log.Printf("[DEBUG] - datasource: receiving OutputSpec as gob") + res, err := protobufToHCL2Spec(resp.OutputSpec) if err != nil { - panic("ici:" + err.Error()) + panic(fmt.Sprintf("datasource: failed to deserialise HCL spec from protobuf: %s", err)) } return res } @@ -66,20 +79,35 @@ type ExecuteResponse struct { } func (d *datasource) Execute() (cty.Value, error) { - res := new(cty.Value) resp := new(ExecuteResponse) if err := d.client.Call(d.endpoint+".Execute", new(interface{}), resp); err != nil { err := fmt.Errorf("Datasource.Execute failed: %v", err) - return *res, err + return cty.NilVal, err + } + + if !d.useProto { + log.Printf("[DEBUG] - datasource: receiving Execute as gob") + res := cty.Value{} + err := gob.NewDecoder(bytes.NewReader(resp.Value)).Decode(&res) + if err != nil { + return res, fmt.Errorf("failed to unmarshal cty.Value from gob blob: %s", err) + } + if resp.Error != nil { + err = resp.Error + } + return res, err } - err := gob.NewDecoder(bytes.NewReader(resp.Value)).Decode(&res) + + log.Printf("[DEBUG] - datasource: receiving Execute as msgpack") + res, err := msgpack.Unmarshal(resp.Value, cty.DynamicPseudoType) if err != nil { - return *res, err + return cty.NilVal, fmt.Errorf("failed to unmarshal cty.Value from msgpack blob: %s", err) } + if resp.Error != nil { err = resp.Error } - return *res, err + return res, err } // DatasourceServer wraps a packer.Datasource implementation and makes it @@ -103,18 +131,43 @@ func (d *DatasourceServer) Configure(args *DatasourceConfigureArgs, reply *Datas func (d *DatasourceServer) OutputSpec(args *DatasourceConfigureArgs, reply *OutputSpecResponse) error { spec := d.d.OutputSpec() - b := bytes.NewBuffer(nil) - err := gob.NewEncoder(b).Encode(spec) - reply.OutputSpec = b.Bytes() + + if !d.useProto { + log.Printf("[DEBUG] - datasource: sending OutputSpec as gob") + b := &bytes.Buffer{} + err := gob.NewEncoder(b).Encode(spec) + reply.OutputSpec = b.Bytes() + return err + } + + log.Printf("[DEBUG] - datasource: sending OutputSpec as protobuf") + ret, err := hcl2SpecToProtobuf(spec) + if err != nil { + return err + } + reply.OutputSpec = ret + return err } func (d *DatasourceServer) Execute(args *interface{}, reply *ExecuteResponse) error { spec, err := d.d.Execute() reply.Error = NewBasicError(err) - b := bytes.NewBuffer(nil) - err = gob.NewEncoder(b).Encode(spec) - reply.Value = b.Bytes() + + if !d.useProto { + log.Printf("[DEBUG] - datasource: sending Execute as gob") + b := &bytes.Buffer{} + err = gob.NewEncoder(b).Encode(spec) + reply.Value = b.Bytes() + if reply.Error != nil { + err = reply.Error + } + return err + } + + log.Printf("[DEBUG] - datasource: sending Execute as msgpack") + raw, err := msgpack.Marshal(spec, cty.DynamicPseudoType) + reply.Value = raw if reply.Error != nil { err = reply.Error } diff --git a/rpc/hcl_spec.pb.go b/rpc/hcl_spec.pb.go new file mode 100644 index 000000000..93de45917 --- /dev/null +++ b/rpc/hcl_spec.pb.go @@ -0,0 +1,948 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.28.1 +// protoc (unknown) +// source: hcl_spec.proto + +package rpc + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// CtyType is any of the cty types that can be used for a Attribute. +// +// Bodies aren't an issue since they're encompassing a bunch of different +// attributes, which end-up referencing a type from this structure. +type CtyType struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Types that are assignable to TypeDef: + // + // *CtyType_Primitive + // *CtyType_List + // *CtyType_Map + TypeDef isCtyType_TypeDef `protobuf_oneof:"typeDef"` +} + +func (x *CtyType) Reset() { + *x = CtyType{} + if protoimpl.UnsafeEnabled { + mi := &file_hcl_spec_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *CtyType) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CtyType) ProtoMessage() {} + +func (x *CtyType) ProtoReflect() protoreflect.Message { + mi := &file_hcl_spec_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CtyType.ProtoReflect.Descriptor instead. +func (*CtyType) Descriptor() ([]byte, []int) { + return file_hcl_spec_proto_rawDescGZIP(), []int{0} +} + +func (m *CtyType) GetTypeDef() isCtyType_TypeDef { + if m != nil { + return m.TypeDef + } + return nil +} + +func (x *CtyType) GetPrimitive() *CtyPrimitive { + if x, ok := x.GetTypeDef().(*CtyType_Primitive); ok { + return x.Primitive + } + return nil +} + +func (x *CtyType) GetList() *CtyList { + if x, ok := x.GetTypeDef().(*CtyType_List); ok { + return x.List + } + return nil +} + +func (x *CtyType) GetMap() *CtyMap { + if x, ok := x.GetTypeDef().(*CtyType_Map); ok { + return x.Map + } + return nil +} + +type isCtyType_TypeDef interface { + isCtyType_TypeDef() +} + +type CtyType_Primitive struct { + Primitive *CtyPrimitive `protobuf:"bytes,1,opt,name=primitive,proto3,oneof"` +} + +type CtyType_List struct { + List *CtyList `protobuf:"bytes,2,opt,name=list,proto3,oneof"` +} + +type CtyType_Map struct { + Map *CtyMap `protobuf:"bytes,3,opt,name=map,proto3,oneof"` +} + +func (*CtyType_Primitive) isCtyType_TypeDef() {} + +func (*CtyType_List) isCtyType_TypeDef() {} + +func (*CtyType_Map) isCtyType_TypeDef() {} + +// CtyPrimitive is any of the cty.Type that match the `IsPrimitiveType` function +// i.e. either Number, Bool or String. +type CtyPrimitive struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + TypeString string `protobuf:"bytes,1,opt,name=typeString,proto3" json:"typeString,omitempty"` +} + +func (x *CtyPrimitive) Reset() { + *x = CtyPrimitive{} + if protoimpl.UnsafeEnabled { + mi := &file_hcl_spec_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *CtyPrimitive) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CtyPrimitive) ProtoMessage() {} + +func (x *CtyPrimitive) ProtoReflect() protoreflect.Message { + mi := &file_hcl_spec_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CtyPrimitive.ProtoReflect.Descriptor instead. +func (*CtyPrimitive) Descriptor() ([]byte, []int) { + return file_hcl_spec_proto_rawDescGZIP(), []int{1} +} + +func (x *CtyPrimitive) GetTypeString() string { + if x != nil { + return x.TypeString + } + return "" +} + +// CtyList is a list of a cty.Type +type CtyList struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ElementType *CtyType `protobuf:"bytes,1,opt,name=elementType,proto3" json:"elementType,omitempty"` +} + +func (x *CtyList) Reset() { + *x = CtyList{} + if protoimpl.UnsafeEnabled { + mi := &file_hcl_spec_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *CtyList) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CtyList) ProtoMessage() {} + +func (x *CtyList) ProtoReflect() protoreflect.Message { + mi := &file_hcl_spec_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CtyList.ProtoReflect.Descriptor instead. +func (*CtyList) Descriptor() ([]byte, []int) { + return file_hcl_spec_proto_rawDescGZIP(), []int{2} +} + +func (x *CtyList) GetElementType() *CtyType { + if x != nil { + return x.ElementType + } + return nil +} + +// CtyMap is a map from one type to another +type CtyMap struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ElementType *CtyType `protobuf:"bytes,1,opt,name=elementType,proto3" json:"elementType,omitempty"` +} + +func (x *CtyMap) Reset() { + *x = CtyMap{} + if protoimpl.UnsafeEnabled { + mi := &file_hcl_spec_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *CtyMap) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CtyMap) ProtoMessage() {} + +func (x *CtyMap) ProtoReflect() protoreflect.Message { + mi := &file_hcl_spec_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CtyMap.ProtoReflect.Descriptor instead. +func (*CtyMap) Descriptor() ([]byte, []int) { + return file_hcl_spec_proto_rawDescGZIP(), []int{3} +} + +func (x *CtyMap) GetElementType() *CtyType { + if x != nil { + return x.ElementType + } + return nil +} + +// HCL2Spec matches what Packer already consumes from plugins in order to describe +// their contents' schema, and lets Packer decode the configuration provided by +// the user to cty values, and detect problems with the contents before executing them. +// +// These are sent over-the-wire over gRPC, much like the old system did using gob +// encoding and standard go RPC servers. +type HCL2Spec struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + TypeSpec map[string]*Spec `protobuf:"bytes,1,rep,name=TypeSpec,proto3" json:"TypeSpec,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` +} + +func (x *HCL2Spec) Reset() { + *x = HCL2Spec{} + if protoimpl.UnsafeEnabled { + mi := &file_hcl_spec_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *HCL2Spec) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HCL2Spec) ProtoMessage() {} + +func (x *HCL2Spec) ProtoReflect() protoreflect.Message { + mi := &file_hcl_spec_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HCL2Spec.ProtoReflect.Descriptor instead. +func (*HCL2Spec) Descriptor() ([]byte, []int) { + return file_hcl_spec_proto_rawDescGZIP(), []int{4} +} + +func (x *HCL2Spec) GetTypeSpec() map[string]*Spec { + if x != nil { + return x.TypeSpec + } + return nil +} + +// A Spec is any kind of object that can convert losslessly to any of the hcldec.Spec types. +type Spec struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Types that are assignable to Block: + // + // *Spec_Object + // *Spec_Attr + // *Spec_BlockValue + // *Spec_BlockList + Block isSpec_Block `protobuf_oneof:"block"` +} + +func (x *Spec) Reset() { + *x = Spec{} + if protoimpl.UnsafeEnabled { + mi := &file_hcl_spec_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Spec) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Spec) ProtoMessage() {} + +func (x *Spec) ProtoReflect() protoreflect.Message { + mi := &file_hcl_spec_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Spec.ProtoReflect.Descriptor instead. +func (*Spec) Descriptor() ([]byte, []int) { + return file_hcl_spec_proto_rawDescGZIP(), []int{5} +} + +func (m *Spec) GetBlock() isSpec_Block { + if m != nil { + return m.Block + } + return nil +} + +func (x *Spec) GetObject() *Object { + if x, ok := x.GetBlock().(*Spec_Object); ok { + return x.Object + } + return nil +} + +func (x *Spec) GetAttr() *Attr { + if x, ok := x.GetBlock().(*Spec_Attr); ok { + return x.Attr + } + return nil +} + +func (x *Spec) GetBlockValue() *Block { + if x, ok := x.GetBlock().(*Spec_BlockValue); ok { + return x.BlockValue + } + return nil +} + +func (x *Spec) GetBlockList() *BlockList { + if x, ok := x.GetBlock().(*Spec_BlockList); ok { + return x.BlockList + } + return nil +} + +type isSpec_Block interface { + isSpec_Block() +} + +type Spec_Object struct { + Object *Object `protobuf:"bytes,1,opt,name=object,proto3,oneof"` +} + +type Spec_Attr struct { + Attr *Attr `protobuf:"bytes,2,opt,name=attr,proto3,oneof"` +} + +type Spec_BlockValue struct { + BlockValue *Block `protobuf:"bytes,3,opt,name=block_value,json=blockValue,proto3,oneof"` +} + +type Spec_BlockList struct { + BlockList *BlockList `protobuf:"bytes,4,opt,name=block_list,json=blockList,proto3,oneof"` +} + +func (*Spec_Object) isSpec_Block() {} + +func (*Spec_Attr) isSpec_Block() {} + +func (*Spec_BlockValue) isSpec_Block() {} + +func (*Spec_BlockList) isSpec_Block() {} + +// Attr spec type reads the value of an attribute in the current body +// and returns that value as its result. It also creates validation constraints +// for the given attribute name and its value. +type Attr struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Type *CtyType `protobuf:"bytes,2,opt,name=type,proto3" json:"type,omitempty"` + Required bool `protobuf:"varint,3,opt,name=required,proto3" json:"required,omitempty"` +} + +func (x *Attr) Reset() { + *x = Attr{} + if protoimpl.UnsafeEnabled { + mi := &file_hcl_spec_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Attr) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Attr) ProtoMessage() {} + +func (x *Attr) ProtoReflect() protoreflect.Message { + mi := &file_hcl_spec_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Attr.ProtoReflect.Descriptor instead. +func (*Attr) Descriptor() ([]byte, []int) { + return file_hcl_spec_proto_rawDescGZIP(), []int{6} +} + +func (x *Attr) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Attr) GetType() *CtyType { + if x != nil { + return x.Type + } + return nil +} + +func (x *Attr) GetRequired() bool { + if x != nil { + return x.Required + } + return false +} + +// Block spec type applies one nested spec block to the contents of a +// block within the current body and returns the result of that spec. It also +// creates validation constraints for the given block type name. +type Block struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Required bool `protobuf:"varint,2,opt,name=required,proto3" json:"required,omitempty"` + Nested *Spec `protobuf:"bytes,3,opt,name=nested,proto3" json:"nested,omitempty"` +} + +func (x *Block) Reset() { + *x = Block{} + if protoimpl.UnsafeEnabled { + mi := &file_hcl_spec_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Block) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Block) ProtoMessage() {} + +func (x *Block) ProtoReflect() protoreflect.Message { + mi := &file_hcl_spec_proto_msgTypes[7] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Block.ProtoReflect.Descriptor instead. +func (*Block) Descriptor() ([]byte, []int) { + return file_hcl_spec_proto_rawDescGZIP(), []int{7} +} + +func (x *Block) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Block) GetRequired() bool { + if x != nil { + return x.Required + } + return false +} + +func (x *Block) GetNested() *Spec { + if x != nil { + return x.Nested + } + return nil +} + +// BlockList spec type is similar to `Block`, but it accepts zero or +// more blocks of a specified type rather than requiring zero or one. The +// result is a JSON array with one entry per block of the given type. +type BlockList struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Nested *Spec `protobuf:"bytes,2,opt,name=nested,proto3" json:"nested,omitempty"` +} + +func (x *BlockList) Reset() { + *x = BlockList{} + if protoimpl.UnsafeEnabled { + mi := &file_hcl_spec_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *BlockList) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BlockList) ProtoMessage() {} + +func (x *BlockList) ProtoReflect() protoreflect.Message { + mi := &file_hcl_spec_proto_msgTypes[8] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BlockList.ProtoReflect.Descriptor instead. +func (*BlockList) Descriptor() ([]byte, []int) { + return file_hcl_spec_proto_rawDescGZIP(), []int{8} +} + +func (x *BlockList) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *BlockList) GetNested() *Spec { + if x != nil { + return x.Nested + } + return nil +} + +// Object spec type is the most commonly used at the root of a spec file. +// Its result is a JSON object whose properties are set based on any nested +// spec blocks: +type Object struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Attributes map[string]*Spec `protobuf:"bytes,1,rep,name=attributes,proto3" json:"attributes,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` +} + +func (x *Object) Reset() { + *x = Object{} + if protoimpl.UnsafeEnabled { + mi := &file_hcl_spec_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Object) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Object) ProtoMessage() {} + +func (x *Object) ProtoReflect() protoreflect.Message { + mi := &file_hcl_spec_proto_msgTypes[9] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Object.ProtoReflect.Descriptor instead. +func (*Object) Descriptor() ([]byte, []int) { + return file_hcl_spec_proto_rawDescGZIP(), []int{9} +} + +func (x *Object) GetAttributes() map[string]*Spec { + if x != nil { + return x.Attributes + } + return nil +} + +var File_hcl_spec_proto protoreflect.FileDescriptor + +var file_hcl_spec_proto_rawDesc = []byte{ + 0x0a, 0x0e, 0x68, 0x63, 0x6c, 0x5f, 0x73, 0x70, 0x65, 0x63, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x22, 0x80, 0x01, 0x0a, 0x07, 0x43, 0x74, 0x79, 0x54, 0x79, 0x70, 0x65, 0x12, 0x2d, 0x0a, 0x09, + 0x70, 0x72, 0x69, 0x6d, 0x69, 0x74, 0x69, 0x76, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x0d, 0x2e, 0x43, 0x74, 0x79, 0x50, 0x72, 0x69, 0x6d, 0x69, 0x74, 0x69, 0x76, 0x65, 0x48, 0x00, + 0x52, 0x09, 0x70, 0x72, 0x69, 0x6d, 0x69, 0x74, 0x69, 0x76, 0x65, 0x12, 0x1e, 0x0a, 0x04, 0x6c, + 0x69, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x08, 0x2e, 0x43, 0x74, 0x79, 0x4c, + 0x69, 0x73, 0x74, 0x48, 0x00, 0x52, 0x04, 0x6c, 0x69, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x03, 0x6d, + 0x61, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x07, 0x2e, 0x43, 0x74, 0x79, 0x4d, 0x61, + 0x70, 0x48, 0x00, 0x52, 0x03, 0x6d, 0x61, 0x70, 0x42, 0x09, 0x0a, 0x07, 0x74, 0x79, 0x70, 0x65, + 0x44, 0x65, 0x66, 0x22, 0x2e, 0x0a, 0x0c, 0x43, 0x74, 0x79, 0x50, 0x72, 0x69, 0x6d, 0x69, 0x74, + 0x69, 0x76, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x74, 0x79, 0x70, 0x65, 0x53, 0x74, 0x72, 0x69, 0x6e, + 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x74, 0x79, 0x70, 0x65, 0x53, 0x74, 0x72, + 0x69, 0x6e, 0x67, 0x22, 0x35, 0x0a, 0x07, 0x43, 0x74, 0x79, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x2a, + 0x0a, 0x0b, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x08, 0x2e, 0x43, 0x74, 0x79, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0b, 0x65, + 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x22, 0x34, 0x0a, 0x06, 0x43, 0x74, + 0x79, 0x4d, 0x61, 0x70, 0x12, 0x2a, 0x0a, 0x0b, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x54, + 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x08, 0x2e, 0x43, 0x74, 0x79, 0x54, + 0x79, 0x70, 0x65, 0x52, 0x0b, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, + 0x22, 0x83, 0x01, 0x0a, 0x08, 0x48, 0x43, 0x4c, 0x32, 0x53, 0x70, 0x65, 0x63, 0x12, 0x33, 0x0a, + 0x08, 0x54, 0x79, 0x70, 0x65, 0x53, 0x70, 0x65, 0x63, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x17, 0x2e, 0x48, 0x43, 0x4c, 0x32, 0x53, 0x70, 0x65, 0x63, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x53, + 0x70, 0x65, 0x63, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x54, 0x79, 0x70, 0x65, 0x53, 0x70, + 0x65, 0x63, 0x1a, 0x42, 0x0a, 0x0d, 0x54, 0x79, 0x70, 0x65, 0x53, 0x70, 0x65, 0x63, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x1b, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x05, 0x2e, 0x53, 0x70, 0x65, 0x63, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xa7, 0x01, 0x0a, 0x04, 0x53, 0x70, 0x65, 0x63, 0x12, + 0x21, 0x0a, 0x06, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x07, 0x2e, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x48, 0x00, 0x52, 0x06, 0x6f, 0x62, 0x6a, 0x65, + 0x63, 0x74, 0x12, 0x1b, 0x0a, 0x04, 0x61, 0x74, 0x74, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x05, 0x2e, 0x41, 0x74, 0x74, 0x72, 0x48, 0x00, 0x52, 0x04, 0x61, 0x74, 0x74, 0x72, 0x12, + 0x29, 0x0a, 0x0b, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x06, 0x2e, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x48, 0x00, 0x52, 0x0a, + 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x2b, 0x0a, 0x0a, 0x62, 0x6c, + 0x6f, 0x63, 0x6b, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0a, + 0x2e, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x4c, 0x69, 0x73, 0x74, 0x48, 0x00, 0x52, 0x09, 0x62, 0x6c, + 0x6f, 0x63, 0x6b, 0x4c, 0x69, 0x73, 0x74, 0x42, 0x07, 0x0a, 0x05, 0x62, 0x6c, 0x6f, 0x63, 0x6b, + 0x22, 0x54, 0x0a, 0x04, 0x41, 0x74, 0x74, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x04, + 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x08, 0x2e, 0x43, 0x74, 0x79, + 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, + 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x72, 0x65, + 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x22, 0x56, 0x0a, 0x05, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x12, + 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, + 0x61, 0x6d, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x12, + 0x1d, 0x0a, 0x06, 0x6e, 0x65, 0x73, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x05, 0x2e, 0x53, 0x70, 0x65, 0x63, 0x52, 0x06, 0x6e, 0x65, 0x73, 0x74, 0x65, 0x64, 0x22, 0x3e, + 0x0a, 0x09, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, + 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, + 0x1d, 0x0a, 0x06, 0x6e, 0x65, 0x73, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x05, 0x2e, 0x53, 0x70, 0x65, 0x63, 0x52, 0x06, 0x6e, 0x65, 0x73, 0x74, 0x65, 0x64, 0x22, 0x87, + 0x01, 0x0a, 0x06, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x37, 0x0a, 0x0a, 0x61, 0x74, 0x74, + 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, + 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x2e, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, + 0x65, 0x73, 0x1a, 0x44, 0x0a, 0x0f, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x1b, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x05, 0x2e, 0x53, 0x70, 0x65, 0x63, 0x52, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x06, 0x5a, 0x04, 0x72, 0x70, 0x63, 0x2f, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_hcl_spec_proto_rawDescOnce sync.Once + file_hcl_spec_proto_rawDescData = file_hcl_spec_proto_rawDesc +) + +func file_hcl_spec_proto_rawDescGZIP() []byte { + file_hcl_spec_proto_rawDescOnce.Do(func() { + file_hcl_spec_proto_rawDescData = protoimpl.X.CompressGZIP(file_hcl_spec_proto_rawDescData) + }) + return file_hcl_spec_proto_rawDescData +} + +var file_hcl_spec_proto_msgTypes = make([]protoimpl.MessageInfo, 12) +var file_hcl_spec_proto_goTypes = []interface{}{ + (*CtyType)(nil), // 0: CtyType + (*CtyPrimitive)(nil), // 1: CtyPrimitive + (*CtyList)(nil), // 2: CtyList + (*CtyMap)(nil), // 3: CtyMap + (*HCL2Spec)(nil), // 4: HCL2Spec + (*Spec)(nil), // 5: Spec + (*Attr)(nil), // 6: Attr + (*Block)(nil), // 7: Block + (*BlockList)(nil), // 8: BlockList + (*Object)(nil), // 9: Object + nil, // 10: HCL2Spec.TypeSpecEntry + nil, // 11: Object.AttributesEntry +} +var file_hcl_spec_proto_depIdxs = []int32{ + 1, // 0: CtyType.primitive:type_name -> CtyPrimitive + 2, // 1: CtyType.list:type_name -> CtyList + 3, // 2: CtyType.map:type_name -> CtyMap + 0, // 3: CtyList.elementType:type_name -> CtyType + 0, // 4: CtyMap.elementType:type_name -> CtyType + 10, // 5: HCL2Spec.TypeSpec:type_name -> HCL2Spec.TypeSpecEntry + 9, // 6: Spec.object:type_name -> Object + 6, // 7: Spec.attr:type_name -> Attr + 7, // 8: Spec.block_value:type_name -> Block + 8, // 9: Spec.block_list:type_name -> BlockList + 0, // 10: Attr.type:type_name -> CtyType + 5, // 11: Block.nested:type_name -> Spec + 5, // 12: BlockList.nested:type_name -> Spec + 11, // 13: Object.attributes:type_name -> Object.AttributesEntry + 5, // 14: HCL2Spec.TypeSpecEntry.value:type_name -> Spec + 5, // 15: Object.AttributesEntry.value:type_name -> Spec + 16, // [16:16] is the sub-list for method output_type + 16, // [16:16] is the sub-list for method input_type + 16, // [16:16] is the sub-list for extension type_name + 16, // [16:16] is the sub-list for extension extendee + 0, // [0:16] is the sub-list for field type_name +} + +func init() { file_hcl_spec_proto_init() } +func file_hcl_spec_proto_init() { + if File_hcl_spec_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_hcl_spec_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*CtyType); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_hcl_spec_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*CtyPrimitive); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_hcl_spec_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*CtyList); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_hcl_spec_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*CtyMap); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_hcl_spec_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*HCL2Spec); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_hcl_spec_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Spec); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_hcl_spec_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Attr); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_hcl_spec_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Block); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_hcl_spec_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*BlockList); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_hcl_spec_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Object); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_hcl_spec_proto_msgTypes[0].OneofWrappers = []interface{}{ + (*CtyType_Primitive)(nil), + (*CtyType_List)(nil), + (*CtyType_Map)(nil), + } + file_hcl_spec_proto_msgTypes[5].OneofWrappers = []interface{}{ + (*Spec_Object)(nil), + (*Spec_Attr)(nil), + (*Spec_BlockValue)(nil), + (*Spec_BlockList)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_hcl_spec_proto_rawDesc, + NumEnums: 0, + NumMessages: 12, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_hcl_spec_proto_goTypes, + DependencyIndexes: file_hcl_spec_proto_depIdxs, + MessageInfos: file_hcl_spec_proto_msgTypes, + }.Build() + File_hcl_spec_proto = out.File + file_hcl_spec_proto_rawDesc = nil + file_hcl_spec_proto_goTypes = nil + file_hcl_spec_proto_depIdxs = nil +} diff --git a/rpc/hcl_spec.proto b/rpc/hcl_spec.proto new file mode 100644 index 000000000..3e188731e --- /dev/null +++ b/rpc/hcl_spec.proto @@ -0,0 +1,93 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +syntax = "proto3"; + +option go_package = "rpc/"; + +// CtyType is any of the cty types that can be used for a Attribute. +// +// Bodies aren't an issue since they're encompassing a bunch of different +// attributes, which end-up referencing a type from this structure. +message CtyType { + oneof typeDef { + CtyPrimitive primitive = 1; + CtyList list = 2; + CtyMap map = 3; + } +} + +// CtyPrimitive is any of the cty.Type that match the `IsPrimitiveType` function +// i.e. either Number, Bool or String. +message CtyPrimitive { + string typeString = 1; +} + +// CtyList is a list of a cty.Type +message CtyList { + CtyType elementType = 1; +} + +// CtyMap is a map from one type to another +message CtyMap { + CtyType elementType = 1; +} + +/* +HCL2Spec matches what Packer already consumes from plugins in order to describe +their contents' schema, and lets Packer decode the configuration provided by +the user to cty values, and detect problems with the contents before executing them. + +These are sent over-the-wire over gRPC, much like the old system did using gob +encoding and standard go RPC servers. +*/ +message HCL2Spec { + map TypeSpec = 1; +} + +// A Spec is any kind of object that can convert losslessly to any of the hcldec.Spec types. +message Spec { + oneof block { + Object object = 1; + Attr attr = 2; + Block block_value = 3; + BlockList block_list = 4; + } +} + +/* Attr spec type reads the value of an attribute in the current body +and returns that value as its result. It also creates validation constraints +for the given attribute name and its value. +*/ +message Attr { + string name = 1; + CtyType type = 2; + bool required = 3; +} + +/* Block spec type applies one nested spec block to the contents of a +block within the current body and returns the result of that spec. It also +creates validation constraints for the given block type name. +*/ +message Block { + string name = 1; + bool required = 2; + Spec nested = 3; +} + +/* BlockList spec type is similar to `Block`, but it accepts zero or +more blocks of a specified type rather than requiring zero or one. The +result is a JSON array with one entry per block of the given type. +*/ +message BlockList { + string name = 1; + Spec nested = 2; +} + +/* Object spec type is the most commonly used at the root of a spec file. +Its result is a JSON object whose properties are set based on any nested +spec blocks: +*/ +message Object { + map attributes = 1; +} diff --git a/rpc/server.go b/rpc/server.go index 759f0594f..d3bd11ed2 100644 --- a/rpc/server.go +++ b/rpc/server.go @@ -35,6 +35,13 @@ type PluginServer struct { streamId uint32 server *rpc.Server closeMux bool + // UseProto forces the server to use protobuf/msgpack for serialization + // instead of gob. + // Setting UseProto on a server endpoint that only supports gob is + // treated as a noop. + // This field is set by the plugin `Set` type for plugins who support + // protocol version v2. + UseProto bool } // NewServer returns a new Packer RPC server. @@ -85,6 +92,7 @@ func (s *PluginServer) RegisterBuilder(b packer.Builder) error { commonServer: commonServer{ selfConfigurable: b, mux: s.mux, + useProto: s.UseProto, }, builder: b, }) @@ -95,6 +103,13 @@ func (s *PluginServer) RegisterCommunicator(c packer.Communicator) error { c: c, commonServer: commonServer{ mux: s.mux, + // Setting useProto to false is essentially a noop for + // this type of server since they don't exchange cty + // values, and there's no HCLSpec object tied to this. + // + // For documentation purposes though, we keep it visible + // in order to change this later if it becomes relevant. + useProto: false, }, }) } @@ -111,6 +126,7 @@ func (s *PluginServer) RegisterPostProcessor(p packer.PostProcessor) error { commonServer: commonServer{ selfConfigurable: p, mux: s.mux, + useProto: s.UseProto, }, p: p, }) @@ -121,6 +137,7 @@ func (s *PluginServer) RegisterProvisioner(p packer.Provisioner) error { commonServer: commonServer{ selfConfigurable: p, mux: s.mux, + useProto: s.UseProto, }, p: p, }) @@ -131,6 +148,7 @@ func (s *PluginServer) RegisterDatasource(d packer.Datasource) error { commonServer: commonServer{ selfConfigurable: d, mux: s.mux, + useProto: s.UseProto, }, d: d, })