amazon-ssm-agent service fails connecting to SSM due to eventual consistency

builder/common/helper_funcs.go

@@ -76,3 +76,10 @@ func DestroyAMIs(imageids []*string, ec2conn *ec2.EC2) error {
 	}
 	return nil
 }
+
+func AwsPartition(isRestricted bool) string {
+	if isRestricted {
+		return "aws-cn"
+	}
+	return "aws"
+}

builder/common/run_config.go

@@ -8,6 +8,7 @@ package common
 
 import (
 	"fmt"
+	"log"
 	"net"
 	"os"
 	"regexp"
@@ -783,10 +784,8 @@ func (c *RunConfig) Prepare(ctx *interpolate.Context) []error {
 			msg := fmt.Errorf(`session_manager connectivity is not supported with the "winrm" communicator; please use "ssh"`)
 			errs = append(errs, msg)
 		}
-
-		if c.IamInstanceProfile == "" && c.TemporaryIamInstanceProfilePolicyDocument == nil {
-			msg := fmt.Errorf(`no iam_instance_profile defined; session_manager connectivity requires a valid instance profile with AmazonSSMManagedInstanceCore permissions. Alternatively a temporary_iam_instance_profile_policy_document can be used.`)
-			errs = append(errs, msg)
+		if c.IamInstanceProfile != "" {
+			log.Printf("[WARNING] (aws): session_manager connectivity requires a valid instance profile with AmazonSSMManagedInstanceCore permissions. Please make sure iam_instance_profile has proper permissions.")
 		}
 	}
 
@@ -940,8 +939,7 @@ func (c *RunConfig) IsSpotInstance() bool {
 }
 
 func (c *RunConfig) SSMAgentEnabled() bool {
-	hasIamInstanceProfile := c.IamInstanceProfile != "" || c.TemporaryIamInstanceProfilePolicyDocument != nil
-	return c.SSHInterface == "session_manager" && hasIamInstanceProfile
+	return c.SSHInterface == "session_manager"
 }
 
 // IsBurstableInstanceType checks if the InstanceType for the config is one

builder/common/step_iam_instance_profile.go

@@ -17,11 +17,17 @@ import (
 	"github.com/hashicorp/packer-plugin-sdk/uuid"
 )
 
+const (
+	AmazonSSMManagedInstanceCorePolicyArnPart = "iam::aws:policy/AmazonSSMManagedInstanceCore"
+)
+
 type StepIamInstanceProfile struct {
 	PollingConfig                             *AWSPollingConfig
 	IamInstanceProfile                        string
 	SkipProfileValidation                     bool
 	TemporaryIamInstanceProfilePolicyDocument *PolicyDocument
+	SSMAgentEnabled                           bool
+	IsRestricted                              bool
 	createdInstanceProfileName                string
 	createdRoleName                           string
 	createdPolicyName                         string
@@ -30,6 +36,17 @@ type StepIamInstanceProfile struct {
 	Ctx                                       interpolate.Context
 }
 
+func handleError(state multistep.StateBag, err error, message ...string) multistep.StepAction {
+	log.Printf("[DEBUG] %s", err.Error())
+	state.Get("ui").(packersdk.Ui).Error(err.Error())
+	if len(message) > 0 {
+		state.Put("error", fmt.Errorf("%s: %s", message[0], err))
+	} else {
+		state.Put("error", err)
+	}
+	return multistep.ActionHalt
+}
+
 func (s *StepIamInstanceProfile) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
 	iamsvc := state.Get("iam").(*iam.IAM)
 	ui := state.Get("ui").(packersdk.Ui)
@@ -44,79 +61,55 @@ func (s *StepIamInstanceProfile) Run(ctx context.Context, state multistep.StateB
 				},
 			)
 			if err != nil {
-				err := fmt.Errorf("Couldn't find specified instance profile: %s", err)
-				log.Printf("[DEBUG] %s", err.Error())
-				state.Put("error", err)
-				return multistep.ActionHalt
+				return handleError(state, err, "Couldn't find specified instance profile: %s")
 			}
 		}
 		log.Printf("Using specified instance profile: %v", s.IamInstanceProfile)
 		state.Put("iamInstanceProfile", s.IamInstanceProfile)
 		return multistep.ActionContinue
 	}
 
-	if s.TemporaryIamInstanceProfilePolicyDocument != nil {
+	if s.SSMAgentEnabled || s.TemporaryIamInstanceProfilePolicyDocument != nil {
 		// Create the profile
 		profileName := fmt.Sprintf("packer-%s", uuid.TimeOrderedUUID())
 
-		policy, err := json.Marshal(s.TemporaryIamInstanceProfilePolicyDocument)
-		if err != nil {
-			ui.Error(err.Error())
-			state.Put("error", err)
-			return multistep.ActionHalt
-		}
-
-		ui.Say(fmt.Sprintf("Creating temporary instance profile for this instance: %s", profileName))
+		ui.Sayf("Creating temporary instance profile for this instance: %s", profileName)
 
 		region := state.Get("region").(*string)
 		iamProfileTags, err := TagMap(s.Tags).IamTags(s.Ctx, *region, state)
 		if err != nil {
-			err := fmt.Errorf("Error creating IAM tags: %s", err)
-			state.Put("error", err)
-			return multistep.ActionHalt
-		}
-		profileResp, err := iamsvc.CreateInstanceProfile(&iam.CreateInstanceProfileInput{
-			InstanceProfileName: aws.String(profileName),
-			Tags:                iamProfileTags,
-		})
-		if err != nil {
-			ui.Error(err.Error())
-			state.Put("error", err)
-			return multistep.ActionHalt
+			return handleError(state, err, "Error creating IAM tags")
 		}
-		s.createdInstanceProfileName = aws.StringValue(profileResp.InstanceProfile.InstanceProfileName)
 
-		log.Printf("[DEBUG] Waiting for temporary instance profile: %s", s.createdInstanceProfileName)
-		err = iamsvc.WaitUntilInstanceProfileExists(&iam.GetInstanceProfileInput{
-			InstanceProfileName: aws.String(s.createdInstanceProfileName),
-		})
-
-		if err == nil {
-			log.Printf("[DEBUG] Found instance profile %s", s.createdInstanceProfileName)
-		} else {
-			err := fmt.Errorf("Timed out waiting for instance profile %s: %s", s.createdInstanceProfileName, err)
-			log.Printf("[DEBUG] %s", err.Error())
-			state.Put("error", err)
-			return multistep.ActionHalt
+		ui.Sayf("Creating temporary role for this instance: %s", profileName)
+		service := "ec2.amazonaws.com"
+		if s.IsRestricted {
+			service = "ec2.amazonaws.com.cn"
 		}
-
-		ui.Say(fmt.Sprintf("Creating temporary role for this instance: %s", profileName))
-
+		trustPolicy := fmt.Sprintf(`{
+			"Version": "2012-10-17",
+			"Statement": [
+					{
+							"Effect": "Allow",
+							"Principal": {
+									"Service": "%s"
+							},
+							"Action": "sts:AssumeRole"
+					}
+			]
+	}`, service)
 		roleResp, err := iamsvc.CreateRole(&iam.CreateRoleInput{
 			RoleName:                 aws.String(profileName),
 			Description:              aws.String("Temporary role for Packer"),
-			AssumeRolePolicyDocument: aws.String("{\"Version\": \"2012-10-17\",\"Statement\": [{\"Effect\": \"Allow\",\"Principal\": {\"Service\": \"ec2.amazonaws.com\"},\"Action\": \"sts:AssumeRole\"}]}"),
+			AssumeRolePolicyDocument: aws.String(trustPolicy),
 			Tags:                     iamProfileTags,
 		})
 		if err != nil {
-			ui.Error(err.Error())
-			state.Put("error", err)
-			return multistep.ActionHalt
+			return handleError(state, err, "Error creating role")
 		}
+		s.createdRoleName = *roleResp.Role.RoleName
 
-		s.createdRoleName = aws.StringValue(roleResp.Role.RoleName)
-
-		log.Printf("[DEBUG] Waiting for temporary role: %s", s.createdInstanceProfileName)
+		log.Printf("[DEBUG] Waiting for temporary role: %s", s.createdRoleName)
 		err = iamsvc.WaitUntilRoleExistsWithContext(
 			aws.BackgroundContext(),
 			&iam.GetRoleInput{
@@ -127,39 +120,80 @@ func (s *StepIamInstanceProfile) Run(ctx context.Context, state multistep.StateB
 		if err == nil {
 			log.Printf("[DEBUG] Found temporary role %s", s.createdRoleName)
 		} else {
-			err := fmt.Errorf("Timed out waiting for temporary role %s: %s", s.createdRoleName, err)
-			log.Printf("[DEBUG] %s", err.Error())
-			state.Put("error", err)
-			return multistep.ActionHalt
+			return handleError(state, err, fmt.Sprintf("Timed out waiting for temporary role %s", s.createdRoleName))
 		}
 
-		ui.Say(fmt.Sprintf("Attaching policy to the temporary role: %s", profileName))
+		ui.Sayf("Attaching policy to the temporary role: %s", profileName)
+
+		if s.TemporaryIamInstanceProfilePolicyDocument != nil {
+			inlinePolicyJSON, err := json.Marshal(s.TemporaryIamInstanceProfilePolicyDocument)
+			if err != nil {
+				return handleError(state, err, "Error parsing policy document")
+			}
+			_, err = iamsvc.PutRolePolicy(&iam.PutRolePolicyInput{
+				RoleName:       aws.String(s.createdRoleName),
+				PolicyName:     aws.String(profileName),
+				PolicyDocument: aws.String(string(inlinePolicyJSON)),
+			})
+			if err != nil {
+				return handleError(state, err, "Error attaching policy to role")
+			}
+			s.createdPolicyName = profileName
+		}
+		if s.SSMAgentEnabled {
+			ssmPolicyArn := aws.String(fmt.Sprintf("arn:%s:%s", AwsPartition(s.IsRestricted), AmazonSSMManagedInstanceCorePolicyArnPart))
+			_, err = iamsvc.AttachRolePolicy(&iam.AttachRolePolicyInput{
+				PolicyArn: ssmPolicyArn,
+				RoleName:  aws.String(s.createdRoleName),
+			})
+			if err != nil {
+				return handleError(state, err, "Error attaching AmazonSSMManagedInstanceCore policy to role")
+			}
+			log.Printf("[DEBUG] Waiting for AmazonSSMManagedInstanceCore attached policy ready")
+			err = iamsvc.WaitUntilPolicyExistsWithContext(
+				aws.BackgroundContext(),
+				&iam.GetPolicyInput{
+					PolicyArn: ssmPolicyArn,
+				},
+				s.PollingConfig.getWaiterOptions()...,
+			)
+			if err == nil {
+				log.Printf("[DEBUG] Found AmazonSSMManagedInstanceCore attached policy in %s", s.createdRoleName)
+			} else {
+				return handleError(state, err, fmt.Sprintf("Timed out waiting for AmazonSSMManagedInstanceCore attached policy in %s", s.createdRoleName))
+			}
+		}
 
-		_, err = iamsvc.PutRolePolicy(&iam.PutRolePolicyInput{
-			RoleName:       roleResp.Role.RoleName,
-			PolicyName:     aws.String(profileName),
-			PolicyDocument: aws.String(string(policy)),
+		profileResp, err := iamsvc.CreateInstanceProfile(&iam.CreateInstanceProfileInput{
+			InstanceProfileName: aws.String(profileName),
+			Tags:                iamProfileTags,
 		})
 		if err != nil {
-			ui.Error(err.Error())
-			state.Put("error", err)
-			return multistep.ActionHalt
+			return handleError(state, err, "Error creating instance profile")
 		}
+		s.createdInstanceProfileName = *profileResp.InstanceProfile.InstanceProfileName
 
-		s.createdPolicyName = aws.StringValue(roleResp.Role.RoleName)
+		log.Printf("[DEBUG] Waiting for temporary instance profile: %s", s.createdInstanceProfileName)
+		err = iamsvc.WaitUntilInstanceProfileExists(&iam.GetInstanceProfileInput{
+			InstanceProfileName: aws.String(s.createdInstanceProfileName),
+		})
+
+		if err == nil {
+			log.Printf("[DEBUG] Found instance profile %s", s.createdInstanceProfileName)
+		} else {
+			return handleError(state, err, fmt.Sprintf("Timed out waiting for instance profile %s", s.createdInstanceProfileName))
+		}
 
 		_, err = iamsvc.AddRoleToInstanceProfile(&iam.AddRoleToInstanceProfileInput{
-			RoleName:            roleResp.Role.RoleName,
-			InstanceProfileName: profileResp.InstanceProfile.InstanceProfileName,
+			InstanceProfileName: aws.String(s.createdInstanceProfileName),
+			RoleName:            aws.String(s.createdRoleName),
 		})
 		if err != nil {
-			ui.Error(err.Error())
-			state.Put("error", err)
-			return multistep.ActionHalt
+			return handleError(state, err, "Error attaching role to instance profile")
 		}
 
 		s.roleIsAttached = true
-		state.Put("iamInstanceProfile", aws.StringValue(profileResp.InstanceProfile.InstanceProfileName))
+		state.Put("iamInstanceProfile", s.createdInstanceProfileName)
 	}
 
 	return multistep.ActionContinue
@@ -170,9 +204,15 @@ func (s *StepIamInstanceProfile) Cleanup(state multistep.StateBag) {
 	ui := state.Get("ui").(packersdk.Ui)
 	var err error
 
-	if s.roleIsAttached == true {
+	if s.roleIsAttached {
 		ui.Say("Detaching temporary role from instance profile...")
 
+		if s.SSMAgentEnabled {
+			_, _ = iamsvc.DetachRolePolicy(&iam.DetachRolePolicyInput{
+				PolicyArn: aws.String(fmt.Sprintf("arn:%s:%s", AwsPartition(s.IsRestricted), AmazonSSMManagedInstanceCorePolicyArnPart)),
+				RoleName:  aws.String(s.createdRoleName),
+			})
+		}
 		_, err := iamsvc.RemoveRoleFromInstanceProfile(&iam.RemoveRoleFromInstanceProfileInput{
 			InstanceProfileName: aws.String(s.createdInstanceProfileName),
 			RoleName:            aws.String(s.createdRoleName),

builder/ebs/builder.go

@@ -340,9 +340,11 @@ func (b *Builder) Run(ctx context.Context, ui packersdk.Ui, hook packersdk.Hook)
 			Ctx:                       b.config.ctx,
 		},
 		&awscommon.StepIamInstanceProfile{
-			PollingConfig:                             b.config.PollingConfig,
-			IamInstanceProfile:                        b.config.IamInstanceProfile,
-			SkipProfileValidation:                     b.config.SkipProfileValidation,
+			PollingConfig:         b.config.PollingConfig,
+			IamInstanceProfile:    b.config.IamInstanceProfile,
+			SkipProfileValidation: b.config.SkipProfileValidation,
+			SSMAgentEnabled:       b.config.SSMAgentEnabled(),
+			IsRestricted:          b.config.IsChinaCloud(),
 			TemporaryIamInstanceProfilePolicyDocument: b.config.TemporaryIamInstanceProfilePolicyDocument,
 			Tags: b.config.RunTags,
 			Ctx:  b.config.ctx,

builder/ebssurrogate/builder.go

@@ -424,9 +424,11 @@ func (b *Builder) Run(ctx context.Context, ui packersdk.Ui, hook packersdk.Hook)
 			Ctx:                       b.config.ctx,
 		},
 		&awscommon.StepIamInstanceProfile{
-			PollingConfig:                             b.config.PollingConfig,
-			IamInstanceProfile:                        b.config.IamInstanceProfile,
-			SkipProfileValidation:                     b.config.SkipProfileValidation,
+			PollingConfig:         b.config.PollingConfig,
+			IamInstanceProfile:    b.config.IamInstanceProfile,
+			SkipProfileValidation: b.config.SkipProfileValidation,
+			SSMAgentEnabled:       b.config.SSMAgentEnabled(),
+			IsRestricted:          b.config.IsChinaCloud(),
 			TemporaryIamInstanceProfilePolicyDocument: b.config.TemporaryIamInstanceProfilePolicyDocument,
 		},
 		&awscommon.StepCleanupVolumes{

builder/ebsvolume/builder.go

@@ -312,9 +312,11 @@ func (b *Builder) Run(ctx context.Context, ui packersdk.Ui, hook packersdk.Hook)
 			Ctx:                       b.config.ctx,
 		},
 		&awscommon.StepIamInstanceProfile{
-			PollingConfig:                             b.config.PollingConfig,
-			IamInstanceProfile:                        b.config.IamInstanceProfile,
-			SkipProfileValidation:                     b.config.SkipProfileValidation,
+			PollingConfig:         b.config.PollingConfig,
+			IamInstanceProfile:    b.config.IamInstanceProfile,
+			SkipProfileValidation: b.config.SkipProfileValidation,
+			SSMAgentEnabled:       b.config.SSMAgentEnabled(),
+			IsRestricted:          b.config.IsChinaCloud(),
 			TemporaryIamInstanceProfilePolicyDocument: b.config.TemporaryIamInstanceProfilePolicyDocument,
 		},
 		instanceStep,

builder/instance/builder.go

@@ -383,9 +383,11 @@ func (b *Builder) Run(ctx context.Context, ui packersdk.Ui, hook packersdk.Hook)
 			Ctx:                       b.config.ctx,
 		},
 		&awscommon.StepIamInstanceProfile{
-			PollingConfig:                             b.config.PollingConfig,
-			IamInstanceProfile:                        b.config.IamInstanceProfile,
-			SkipProfileValidation:                     b.config.SkipProfileValidation,
+			PollingConfig:         b.config.PollingConfig,
+			IamInstanceProfile:    b.config.IamInstanceProfile,
+			SkipProfileValidation: b.config.SkipProfileValidation,
+			SSMAgentEnabled:       b.config.SSMAgentEnabled(),
+			IsRestricted:          b.config.IsChinaCloud(),
 			TemporaryIamInstanceProfilePolicyDocument: b.config.TemporaryIamInstanceProfilePolicyDocument,
 		},
 		instanceStep,