Merge pull request #14 from wttech/instance-prereqs-init

Instance prereqs init

examples/aws_ssm/aws.tf

@@ -1,18 +1,18 @@
 resource "aws_instance" "aem_single" {
   ami                  = "ami-043e06a423cbdca17" // RHEL 8
   instance_type        = "m5.xlarge"
-  iam_instance_profile = aws_iam_instance_profile.ssm.name
+  iam_instance_profile = aws_iam_instance_profile.aem_ec2.name
   tags                 = local.tags
 }
 
-resource "aws_iam_instance_profile" "ssm" {
-  name = "${local.workspace}_ssm_ec2"
-  role = aws_iam_role.ssm.name
+resource "aws_iam_instance_profile" "aem_ec2" {
+  name = "${local.workspace}_aem_ec2"
+  role = aws_iam_role.aem_ec2.name
   tags = local.tags
 }
 
-resource "aws_iam_role" "ssm" {
-  name               = "${local.workspace}_ssm"
+resource "aws_iam_role" "aem_ec2" {
+  name               = "${local.workspace}_aem_ec2"
   assume_role_policy = <<EOF
   {
     "Version": "2012-10-17",
@@ -22,15 +22,20 @@ resource "aws_iam_role" "ssm" {
       "Action": "sts:AssumeRole"
     }
   }
-EOF
-  tags               = local.tags
+  EOF
+  tags = local.tags
 }
 
 resource "aws_iam_role_policy_attachment" "ssm" {
-  role       = aws_iam_role.ssm.name
+  role       = aws_iam_role.aem_ec2.name
   policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
 }
 
+resource "aws_iam_role_policy_attachment" "s3" {
+  role       = aws_iam_role.aem_ec2.name
+  policy_arn = "arn:aws:iam::aws:policy/AmazonS3FullAccess"
+}
+
 output "instance_ip" {
   value = aws_instance.aem_single.public_ip
 }

examples/ssh/aem.tf

@@ -16,6 +16,27 @@ resource "aem_instance" "single" {
     lib_dir     = "aem/home/lib"
     config_file = "aem/default/etc/aem.yml"
   }
+  hook {
+    bootstrap = <<EOF
+      #!/bin/sh
+      sudo yum install -y unzip && \
+      curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
+      unzip -q awscliv2.zip && \
+      sudo ./aws/install --update && \
+      mkdir -p "/home/ec2-user/aemc/aem/home/lib" && \
+      aws s3 cp --recursive --no-progress "s3://aemc/instance/classic/" "/home/ec2-user/aemc/aem/home/lib"
+    EOF
+    initialize = <<EOF
+      #!/bin/sh
+      # sh aemw instance backup restore
+    EOF
+    provision = <<EOF
+      #!/bin/sh
+      sh aemw osgi bundle install --url "https://github.com/neva-dev/felix-search-webconsole-plugin/releases/download/2.0.0/search-webconsole-plugin-2.0.0.jar" && \
+      sh aemw osgi config save --pid "org.apache.sling.jcr.davex.impl.servlets.SlingDavExServlet" --input-string "alias: /crx/server" && \
+      sh aemw package deploy --file "aem/home/lib/aem-service-pkg-6.5.*.0.zip"
+    EOF
+  }
 }
 
 output "aem_instances" {

examples/ssh/aem/default/etc/aem.yml

@@ -221,4 +221,4 @@ output:
     # File path of logs written especially when output format is different than 'text'
     file: aem/home/var/log/aem.log
     # Controls where outputs and logs should be written to when format is 'text' (console|file|both)
-    mode: console
+    mode: both

examples/ssh/aws.tf

@@ -2,8 +2,9 @@ resource "aws_instance" "aem_single" {
   ami                         = "ami-043e06a423cbdca17" // RHEL 8
   instance_type               = "m5.xlarge"
   associate_public_ip_address = true
-  tags                        = local.tags
+  iam_instance_profile        = aws_iam_instance_profile.aem_ec2.name
   key_name                    = aws_key_pair.main.key_name
+  tags                        = local.tags
 }
 
 data "tls_public_key" "main" {
@@ -16,6 +17,32 @@ resource "aws_key_pair" "main" {
   tags       = local.tags
 }
 
+resource "aws_iam_instance_profile" "aem_ec2" {
+  name = "${local.workspace}_aem_ec2"
+  role = aws_iam_role.aem_ec2.name
+  tags = local.tags
+}
+
+resource "aws_iam_role" "aem_ec2" {
+  name               = "${local.workspace}_aem_ec2"
+  assume_role_policy = <<EOF
+  {
+    "Version": "2012-10-17",
+    "Statement": {
+      "Effect": "Allow",
+      "Principal": {"Service": "ec2.amazonaws.com"},
+      "Action": "sts:AssumeRole"
+    }
+  }
+EOF
+  tags = local.tags
+}
+
+resource "aws_iam_role_policy_attachment" "s3" {
+  role       = aws_iam_role.aem_ec2.name
+  policy_arn = "arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess"
+}
+
 output "instance_ip" {
   value = aws_instance.aem_single.public_ip
 }

internal/client/client.go

@@ -16,7 +16,7 @@ type Client struct {
 	connection Connection
 
 	Env    map[string]string
-	EnvDir string
+	EnvDir string // TODO this is more like tmp script dir
 }
 
 func (c Client) TypeName() string {
@@ -42,17 +42,17 @@ func (c Client) Connect() error {
 
 func (c Client) ConnectWithRetry(timeout time.Duration, callback func()) error {
 	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	var err error
 	defer cancel()
 	for {
 		select {
 		case <-ctx.Done():
-			return fmt.Errorf("cannot connect - awaiting timeout reached '%s'", timeout)
+			return fmt.Errorf("cannot connect - awaiting timeout reached '%s': %w", timeout, err)
 		default:
-			err := c.Connect()
-			if err == nil {
+			if err = c.Connect(); err == nil {
 				return nil
 			}
-			time.Sleep(time.Second)
+			time.Sleep(3 * time.Second)
 			callback()
 		}
 	}
@@ -105,6 +105,24 @@ func (c Client) RunShellWithEnv(cmd string) ([]byte, error) {
 	return c.RunShell(fmt.Sprintf("source %s && %s", c.envScriptPath(), cmd))
 }
 
+func (c Client) RunShellScriptWithEnv(dir string, cmdScript string) ([]byte, error) {
+	file, err := os.CreateTemp(os.TempDir(), "tf-provider-aem-script-*.sh")
+	path := file.Name()
+	defer func() { _ = file.Close(); _ = os.Remove(path) }()
+	if err != nil {
+		return nil, fmt.Errorf("cannot create temporary file for remote shell script: %w", err)
+	}
+	if _, err := file.WriteString(cmdScript); err != nil {
+		return nil, fmt.Errorf("cannot write temporary file for remote shell script: %w", err)
+	}
+	remotePath := fmt.Sprintf("%s/%s", c.EnvDir, filepath.Base(file.Name()))
+	defer func() { _ = c.FileDelete(remotePath) }()
+	if err := c.FileCopy(path, remotePath, true); err != nil {
+		return nil, err
+	}
+	return c.RunShellWithEnv(fmt.Sprintf("cd %s && sh %s", dir, remotePath))
+}
+
 func (c Client) RunShell(cmd string) ([]byte, error) {
 	cmdObj, err := c.connection.Command([]string{"sh", "-c", "\"" + cmd + "\""})
 	if err != nil {
@@ -185,6 +203,7 @@ func (c Client) FileDelete(path string) error {
 	return nil
 }
 
+// TODO seems that if file exists it is not skipping copying file
 func (c Client) FileCopy(localPath string, remotePath string, override bool) error {
 	if !override {
 		exists, err := c.FileExists(remotePath)

internal/client/connection_ssh.go

@@ -21,9 +21,8 @@ type SSHConnection struct {
 func (s *SSHConnection) Connect() error {
 	auth, err := goph.Key(s.privateKeyFile, s.passphrase)
 	if err != nil {
-		return fmt.Errorf("SSH: cannot get auth using private key '%s': %w", s.privateKeyFile, err)
+		return fmt.Errorf("ssh: cannot get auth using private key '%s': %w", s.privateKeyFile, err)
 	}
-	// TODO loop until establishment of connection
 	client, err := goph.NewConn(&goph.Config{
 		User:     s.user,
 		Addr:     s.host,
@@ -33,7 +32,7 @@ func (s *SSHConnection) Connect() error {
 		Callback: ssh.InsecureIgnoreHostKey(), // TODO make it secure by default
 	})
 	if err != nil {
-		return fmt.Errorf("SSH: cannot connect to host '%s': %w", s.host, err)
+		return fmt.Errorf("ssh: cannot connect to host '%s': %w", s.host, err)
 	}
 	s.client = client
 	return nil
@@ -44,7 +43,7 @@ func (s *SSHConnection) Disconnect() error {
 		return nil
 	}
 	if err := s.client.Close(); err != nil {
-		return fmt.Errorf("SSH: cannot disconnect from host '%s': %w", s.host, err)
+		return fmt.Errorf("ssh: cannot disconnect from host '%s': %w", s.host, err)
 	}
 	return nil
 }
@@ -53,7 +52,7 @@ func (s *SSHConnection) Command(cmdLine []string) (*goph.Cmd, error) {
 	name, args := s.splitCommandLine(cmdLine)
 	cmd, err := s.client.Command(name, args...)
 	if err != nil {
-		return nil, fmt.Errorf("SSH: cannot create command '%s' for host '%s': %w", strings.Join(cmdLine, " "), s.host, err)
+		return nil, fmt.Errorf("ssh: cannot create command '%s' for host '%s': %w", strings.Join(cmdLine, " "), s.host, err)
 	}
 	return cmd, nil
 }
@@ -69,7 +68,7 @@ func (s *SSHConnection) splitCommandLine(cmdLine []string) (string, []string) {
 
 func (s *SSHConnection) CopyFile(localPath string, remotePath string) error {
 	if err := s.client.Upload(localPath, remotePath); err != nil {
-		return fmt.Errorf("SSH: cannot copy local file '%s' to remote path '%s' on host '%s': %w", localPath, remotePath, s.host, err)
+		return fmt.Errorf("ssh: cannot copy local file '%s' to remote path '%s' on host '%s': %w", localPath, remotePath, s.host, err)
 	}
 	return nil
 }

internal/provider/instance_client.go

@@ -18,11 +18,9 @@ func (ic *InstanceClient) Close() error {
 
 // TODO chown data dir to ssh user or 'aem' user (create him maybe)
 func (ic *InstanceClient) prepareDataDir() error {
-	/* TODO to avoid re-uploading library files (probably temporary)
 	if _, err := ic.cl.RunShell(fmt.Sprintf("rm -fr %s", ic.DataDir())); err != nil {
 		return fmt.Errorf("cannot clean up AEM data directory: %w", err)
 	}
-	*/
 	if _, err := ic.cl.RunShell(fmt.Sprintf("mkdir -p %s", ic.DataDir())); err != nil {
 		return fmt.Errorf("cannot create AEM data directory: %w", err)
 	}
@@ -141,3 +139,36 @@ func (ic *InstanceClient) ReadStatus() (InstanceStatus, error) {
 	}
 	return status, nil
 }
+
+// TODO when create fails this could be run twice; how about protecting it with lock?
+func (ic *InstanceClient) bootstrap() error {
+	return ic.runHook("bootstrap", ic.data.Hook.Bootstrap.ValueString())
+}
+
+// TODO when create fails this could be run twice; how about protecting it with lock?
+func (ic *InstanceClient) initialize() error {
+	return ic.runHook("initialize", ic.data.Hook.Initialize.ValueString())
+}
+
+func (ic *InstanceClient) provision() error {
+	return ic.runHook("provision", ic.data.Hook.Provision.ValueString())
+}
+
+func (ic *InstanceClient) runHook(name string, cmdScript string) error {
+	if cmdScript == "" {
+		return nil
+	}
+
+	tflog.Info(ic.ctx, fmt.Sprintf("Hook '%s' started", name))
+
+	textOut, err := ic.cl.RunShellScriptWithEnv(ic.DataDir(), cmdScript)
+	if err != nil {
+		return fmt.Errorf("unable to run hook '%s' properly: %w", name, err)
+	}
+	textStr := string(textOut) // TODO how about streaming it line by line to tflog ;)
+
+	tflog.Info(ic.ctx, fmt.Sprintf("Hook '%s' finished", name))
+	tflog.Info(ic.ctx, textStr)
+
+	return nil
+}

internal/provider/instance_resource.go

@@ -45,6 +45,11 @@ type InstanceResourceModel struct {
 		LibDir             types.String `tfsdk:"lib_dir"`
 		InstanceId         types.String `tfsdk:"instance_id"`
 	} `tfsdk:"compose"`
+	Hook struct {
+		Bootstrap  types.String `tfsdk:"bootstrap"`
+		Initialize types.String `tfsdk:"initialize"`
+		Provision  types.String `tfsdk:"provision"`
+	} `tfsdk:"hook"`
 	Instances types.List `tfsdk:"instances"`
 }
 
@@ -122,6 +127,25 @@ func (r *InstanceResource) Schema(ctx context.Context, req resource.SchemaReques
 					},
 				},
 			},
+			"hook": schema.SingleNestedBlock{
+				MarkdownDescription: "Scripts executed on the remote AEM machine at key stages of the AEM instance lifecycle",
+				Attributes: map[string]schema.Attribute{
+					"bootstrap": schema.StringAttribute{
+						MarkdownDescription: "Executed once after connecting to the instance. Forces instance recreation if changed. Typically used for: providing AEM library files (quickstart.jar, license.properties, service packs), mounting data volume, etc.",
+						Optional:            true,
+						PlanModifiers:       []planmodifier.String{stringplanmodifier.RequiresReplace()},
+					},
+					"initialize": schema.StringAttribute{
+						MarkdownDescription: "Executed once after initializing AEM Compose but before launching the instance. Forces instance recreation if changed. Can be used for restoring instances from backup.",
+						Optional:            true,
+						PlanModifiers:       []planmodifier.String{stringplanmodifier.RequiresReplace()},
+					},
+					"provision": schema.StringAttribute{
+						MarkdownDescription: "Executed when the instance is launched. Must be idempotent as it is executed always when changed. Typically used for setting up replication agents, installing service packs, etc.",
+						Optional:            true,
+					},
+				},
+			},
 		},
 
 		Attributes: map[string]schema.Attribute{
@@ -187,14 +211,14 @@ func (r *InstanceResource) Configure(ctx context.Context, req resource.Configure
 }
 
 func (r *InstanceResource) Create(ctx context.Context, req resource.CreateRequest, resp *resource.CreateResponse) {
-	r.createOrUpdate(ctx, &req.Plan, &resp.Diagnostics, &resp.State)
+	r.createOrUpdate(ctx, &req.Plan, &resp.Diagnostics, &resp.State, true)
 }
 
 func (r *InstanceResource) Update(ctx context.Context, req resource.UpdateRequest, resp *resource.UpdateResponse) {
-	r.createOrUpdate(ctx, &req.Plan, &resp.Diagnostics, &resp.State)
+	r.createOrUpdate(ctx, &req.Plan, &resp.Diagnostics, &resp.State, false)
 }
 
-func (r *InstanceResource) createOrUpdate(ctx context.Context, plan *tfsdk.Plan, diags *diag.Diagnostics, state *tfsdk.State) {
+func (r *InstanceResource) createOrUpdate(ctx context.Context, plan *tfsdk.Plan, diags *diag.Diagnostics, state *tfsdk.State, create bool) {
 	model := r.newModel()
 
 	// Read Terraform plan data into the model
@@ -221,6 +245,12 @@ func (r *InstanceResource) createOrUpdate(ctx context.Context, plan *tfsdk.Plan,
 		diags.AddError("Unable to prepare AEM data directory", fmt.Sprintf("%s", err))
 		return
 	}
+	if create {
+		if err := ic.bootstrap(); err != nil {
+			diags.AddError("Unable to bootstrap AEM machine", fmt.Sprintf("%s", err))
+			return
+		}
+	}
 	if err := ic.installComposeWrapper(); err != nil {
 		diags.AddError("Unable to install AEM Compose CLI", fmt.Sprintf("%s", err))
 		return
@@ -233,6 +263,12 @@ func (r *InstanceResource) createOrUpdate(ctx context.Context, plan *tfsdk.Plan,
 		diags.AddError("Unable to copy AEM library dir", fmt.Sprintf("%s", err))
 		return
 	}
+	if create {
+		if err := ic.initialize(); err != nil {
+			diags.AddError("Unable to initialize AEM instance", fmt.Sprintf("%s", err))
+			return
+		}
+	}
 	if err := ic.create(); err != nil {
 		diags.AddError("Unable to create AEM instance", fmt.Sprintf("%s", err))
 		return
@@ -241,6 +277,10 @@ func (r *InstanceResource) createOrUpdate(ctx context.Context, plan *tfsdk.Plan,
 		diags.AddError("Unable to launch AEM instance", fmt.Sprintf("%s", err))
 		return
 	}
+	if err := ic.provision(); err != nil {
+		diags.AddError("Unable to provision AEM instance", fmt.Sprintf("%s", err))
+		return
+	}
 
 	tflog.Info(ctx, "Finished setting up AEM instance resource")