Merge pull request #4541 from cnmcavoy/awsmachine-vpc-selection

✨ Add AWSMachine fields to control vpc placement for the instance

api/v1beta1/awsmachine_conversion.go

@@ -39,6 +39,7 @@ func (src *AWSMachine) ConvertTo(dstRaw conversion.Hub) error {
 	dst.Spec.InstanceMetadataOptions = restored.Spec.InstanceMetadataOptions
 	dst.Spec.PlacementGroupName = restored.Spec.PlacementGroupName
 	dst.Spec.PrivateDNSName = restored.Spec.PrivateDNSName
+	dst.Spec.SecurityGroupOverrides = restored.Spec.SecurityGroupOverrides
 
 	return nil
 }
@@ -87,6 +88,7 @@ func (r *AWSMachineTemplate) ConvertTo(dstRaw conversion.Hub) error {
 	dst.Spec.Template.Spec.InstanceMetadataOptions = restored.Spec.Template.Spec.InstanceMetadataOptions
 	dst.Spec.Template.Spec.PlacementGroupName = restored.Spec.Template.Spec.PlacementGroupName
 	dst.Spec.Template.Spec.PrivateDNSName = restored.Spec.Template.Spec.PrivateDNSName
+	dst.Spec.Template.Spec.SecurityGroupOverrides = restored.Spec.Template.Spec.SecurityGroupOverrides
 
 	return nil
 }

api/v1beta2/awsmachine_types.go

@@ -114,6 +114,11 @@ type AWSMachineSpec struct {
 	// +optional
 	Subnet *AWSResourceReference `json:"subnet,omitempty"`
 
+	// SecurityGroupOverrides is an optional set of security groups to use for the node.
+	// This is optional - if not provided security groups from the cluster will be used.
+	// +optional
+	SecurityGroupOverrides map[SecurityGroupRole]string `json:"securityGroupOverrides,omitempty"`
+
 	// SSHKeyName is the name of the ssh key to attach to the instance. Valid values are empty string (do not use SSH keys), a valid SSH key name, or omitted (use the default SSH key name)
 	// +optional
 	SSHKeyName *string `json:"sshKeyName,omitempty"`

config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachines.yaml

@@ -848,6 +848,13 @@ spec:
                 required:
                 - size
                 type: object
+              securityGroupOverrides:
+                additionalProperties:
+                  type: string
+                description: SecurityGroupOverrides is an optional set of security
+                  groups to use for the node. This is optional - if not provided security
+                  groups from the cluster will be used.
+                type: object
               spotMarketOptions:
                 description: SpotMarketOptions allows users to configure instances
                   to be run using AWS Spot instances.

config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinetemplates.yaml

@@ -807,6 +807,14 @@ spec:
                         required:
                         - size
                         type: object
+                      securityGroupOverrides:
+                        additionalProperties:
+                          type: string
+                        description: SecurityGroupOverrides is an optional set of
+                          security groups to use for the node. This is optional -
+                          if not provided security groups from the cluster will be
+                          used.
+                        type: object
                       spotMarketOptions:
                         description: SpotMarketOptions allows users to configure instances
                           to be run using AWS Spot instances.

controllers/awsmachine_controller_test.go

@@ -530,10 +530,6 @@ func mockedCreateSecretCall(s *mock_services.MockSecretInterfaceMockRecorder) {
 func mockedCreateInstanceCalls(m *mocks.MockEC2APIMockRecorder) {
 	m.DescribeInstancesWithContext(context.TODO(), gomock.Eq(&ec2.DescribeInstancesInput{
 		Filters: []*ec2.Filter{
-			{
-				Name:   aws.String("vpc-id"),
-				Values: aws.StringSlice([]string{""}),
-			},
 			{
 				Name:   aws.String("tag:sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"),
 				Values: aws.StringSlice([]string{"owned"}),
@@ -642,10 +638,6 @@ func mockedCreateInstanceCalls(m *mocks.MockEC2APIMockRecorder) {
 			Name:   aws.String("state"),
 			Values: aws.StringSlice([]string{"pending", "available"}),
 		},
-		{
-			Name:   aws.String("vpc-id"),
-			Values: aws.StringSlice([]string{""}),
-		},
 		{
 			Name:   aws.String("subnet-id"),
 			Values: aws.StringSlice([]string{"subnet-1"}),

docs/book/src/topics/bring-your-own-aws-infrastructure.md

@@ -120,6 +120,30 @@ spec:
 
 Users may either specify `failureDomain` on the Machine or MachineDeployment objects, _or_ users may explicitly specify subnet IDs on the AWSMachine or AWSMachineTemplate objects. If both are specified, the subnet ID is used and the `failureDomain` is ignored.
 
+### Placing EC2 Instances in Specific External VPCs
+
+CAPA clusters are deployed within a single VPC, but it's possible to place machines that live in external VPCs. For this kind of configuration, we assume that all the VPCs have the ability to communicate, either through external peering, a transit gateway, or some other mechanism already established outside of CAPA. CAPA will not create a tunnel or manage the network configuration for any secondary VPCs.
+
+The AWSMachineTemplate `subnet` field allows specifying filters or specific subnet ids for worker machine placement. If the filters or subnet id is specified in a secondary VPC, CAPA will place the machine in that VPC and subnet.
+
+```yaml
+spec:
+  template:
+    spec:
+      subnet:
+        filters:
+          name: "vpc-id"
+          values:
+            - "secondary-vpc-id"
+      securityGroupOverrides:
+        node: sg-04e870a3507a5ad2c5c8c2
+        node-eks-additional: sg-04e870a3507a5ad2c5c8c1
+```
+
+#### Caveats/Notes
+
+CAPA helpfully creates security groups for various roles in the cluster and automatically attaches them to workers. However, security groups are tied to a specific VPC, so workers placed in a VPC outside of the cluster will need to have these security groups created by some external process first and set in the `securityGroupOverrides` field, otherwise the ec2 creation will fail.
+
 ### Security Groups
 
 To use existing security groups for instances for a cluster, add this to the AWSCluster specification:
@@ -147,6 +171,15 @@ spec:
     - ...
 ```
 
+It's also possible to override the cluster security groups for an individual AWSMachine or AWSMachineTemplate:
+
+```yaml
+spec:
+  SecurityGroupOverrides:
+    node: sg-04e870a3507a5ad2c5c8c2
+    node-eks-additional: sg-04e870a3507a5ad2c5c8c1
+```
+
 ### Control Plane Load Balancer
 
 The cluster control plane is accessed through a Classic ELB. By default, Cluster API creates the Classic ELB. To use an existing Classic ELB, add its name to the AWSCluster specification:

pkg/cloud/services/ec2/instances.go

@@ -45,7 +45,6 @@ func (s *Service) GetRunningInstanceByTags(scope *scope.MachineScope) (*infrav1.
 
 	input := &ec2.DescribeInstancesInput{
 		Filters: []*ec2.Filter{
-			filter.EC2.VPC(s.scope.VPC().ID),
 			filter.EC2.ClusterOwned(s.scope.Name()),
 			filter.EC2.Name(scope.Name()),
 			filter.EC2.InstanceStates(ec2.InstanceStateNamePending, ec2.InstanceStateNameRunning),
@@ -308,9 +307,6 @@ func (s *Service) findSubnet(scope *scope.MachineScope) (string, error) {
 		criteria := []*ec2.Filter{
 			filter.EC2.SubnetStates(ec2.SubnetStatePending, ec2.SubnetStateAvailable),
 		}
-		if !scope.IsExternallyManaged() {
-			criteria = append(criteria, filter.EC2.VPC(s.scope.VPC().ID))
-		}
 		if scope.AWSMachine.Spec.Subnet.ID != nil {
 			criteria = append(criteria, &ec2.Filter{Name: aws.String("subnet-id"), Values: aws.StringSlice([]string{*scope.AWSMachine.Spec.Subnet.ID})})
 		}
@@ -345,6 +341,11 @@ func (s *Service) findSubnet(scope *scope.MachineScope) (string, error) {
 			}
 			filtered = append(filtered, subnet)
 		}
+		// prefer a subnet in the cluster VPC if multiple match
+		clusterVPC := s.scope.VPC().ID
+		sort.SliceStable(filtered, func(i, j int) bool {
+			return strings.Compare(*filtered[i].VpcId, clusterVPC) > strings.Compare(*filtered[j].VpcId, clusterVPC)
+		})
 		if len(filtered) == 0 {
 			errMessage = fmt.Sprintf("failed to run machine %q, found %d subnets matching criteria but post-filtering failed.",
 				scope.Name(), len(subnets)) + errMessage
@@ -440,10 +441,15 @@ func (s *Service) GetCoreSecurityGroups(scope *scope.MachineScope) ([]string, er
 	}
 	ids := make([]string, 0, len(sgRoles))
 	for _, sg := range sgRoles {
-		if _, ok := s.scope.SecurityGroups()[sg]; !ok {
-			return nil, awserrors.NewFailedDependency(fmt.Sprintf("%s security group not available", sg))
+		if _, ok := scope.AWSMachine.Spec.SecurityGroupOverrides[sg]; ok {
+			ids = append(ids, scope.AWSMachine.Spec.SecurityGroupOverrides[sg])
+			continue
 		}
-		ids = append(ids, s.scope.SecurityGroups()[sg].ID)
+		if _, ok := s.scope.SecurityGroups()[sg]; ok {
+			ids = append(ids, s.scope.SecurityGroups()[sg].ID)
+			continue
+		}
+		return nil, awserrors.NewFailedDependency(fmt.Sprintf("%s security group not available", sg))
 	}
 	return ids, nil
 }