Feat: OutputTemplates

app/assets/javascripts/foreman_remote_execution/output_templates.js

@@ -0,0 +1,10 @@
+function show_import_output_template_modal() {
+  var modal_window = $('#importOutputTemplateModal');
+  modal_window.modal({'show': true});
+  modal_window.find('a[rel="popover-modal"]').popover();
+}
+
+function close_import_output_template_modal() {
+  var modal_window = $('#importOutputTemplateModal');
+  modal_window.modal('hide');
+}

app/assets/stylesheets/foreman_remote_execution/template_invocation.scss

@@ -32,7 +32,7 @@ div.terminal {
     min-height: 50px;
   }
 
-  div.line.stderr, div.line.error, div.line.debug {
+  div.line.stderr, div.line.error, div.line.debug, div.line.output_templates, div.line.output_templates_err {
     color: red;
   }
 

app/controllers/api/v2/output_templates_controller.rb

@@ -0,0 +1,61 @@
+module Api
+  module V2
+    class OutputTemplatesController < ::Api::V2::BaseController
+      include ::Api::Version2
+      include ::Foreman::Renderer
+      include ::Foreman::Controller::ProvisioningTemplates
+      include ::Foreman::Controller::Parameters::OutputTemplate
+
+      api :GET, '/output_templates/', N_('List output templates')
+      # location and Organization
+      param_group :taxonomy_scope, ::Api::V2::BaseController
+      # search and pagination allows to display and filter the index page of templates
+      param_group :search_and_pagination, ::Api::V2::BaseController
+      def index
+        # do not show saved runtime templates
+        @output_templates = resource_scope_for_index.filter { |template| !template.snippet }
+      end
+
+      def_param_group :output_template do
+        param :output_template, Hash, :required => true, :action_aware => true do
+          param :name, String, :required => true, :desc => N_('Template name')
+          param :description, String
+          param :template, String, :required => true
+          param :output, String
+          param :snippet, :bool, :allow_nil => true
+          param :locked, :bool, :desc => N_('Whether or not the template is locked for editing')
+          param :effective_user_attributes, Hash, :desc => N_('Effective user options') do
+            param :value, String, :desc => N_('What user should be used to run the script (using sudo-like mechanisms)'), :allowed_nil => true
+            param :overridable, :bool, :desc => N_('Whether it should be allowed to override the effective user from the invocation form.')
+            param :current_user, :bool, :desc => N_('Whether the current user login should be used as the effective user')
+          end
+          param_group :taxonomies, ::Api::V2::BaseController
+        end
+      end
+
+      api :POST, '/output_templates/', N_('Create an output template')
+      param_group :output_template, :as => :create
+      def create
+        @output_template = OutputTemplate.new(output_template_params)
+        process_response @output_template.save
+      end
+
+      api :DELETE, '/output_templates/:id', N_('Delete an output template')
+      param :id, :identifier, :required => true
+      def destroy
+        process_response @output_template.destroy
+      end
+
+      api :POST, '/output_templates/import', N_('Import an output template from ERB')
+      param :template, String, :required => true, :desc => N_('Template ERB')
+      param :overwrite, :bool, :required => false, :desc => N_('Overwrite template if it already exists')
+      def import
+        options = params[:overwrite] ? { :update => true } : { :build_new => true }
+
+        @output_template = OutputTemplate.import_raw(params[:template], options)
+        @output_template ||= OutputTemplate.new
+        process_response @output_template.save
+      end
+    end
+  end
+end

app/controllers/concerns/foreman/controller/parameters/job_template.rb

@@ -15,7 +15,7 @@ def job_template_effective_user_filter
 
     def job_template_params_filter
       Foreman::ParameterFilter.new(::TemplateInput).tap do |filter|
-        filter.permit :job_category, :provider_type, :description_format, :execution_timeout_interval,
+        filter.permit :job_category, :provider_type, :description_format, :execution_timeout_interval, :output_template_ids => [],
           :effective_user_attributes => [job_template_effective_user_filter],
           :template_inputs_attributes => [template_input_params_filter],
           :foreign_input_sets_attributes => [foreign_input_set_params_filter]

app/controllers/concerns/foreman/controller/parameters/output_template.rb

@@ -0,0 +1,29 @@
+module Foreman::Controller::Parameters::OutputTemplate
+  extend ActiveSupport::Concern
+  include Foreman::Controller::Parameters::Taxonomix
+  include ::Foreman::Controller::Parameters::Template
+  include Foreman::Controller::Parameters::TemplateInput
+
+  class_methods do
+    def output_template_effective_user_filter
+      Foreman::ParameterFilter.new(::OutputTemplateEffectiveUser).tap do |filter|
+        filter.permit_by_context(:value, :current_user, :overridable,
+          :nested => true)
+      end
+    end
+
+    def output_template_params_filter
+      Foreman::ParameterFilter.new(::TemplateInput).tap do |filter|
+        filter.permit :description_format,
+          :effective_user_attributes => [output_template_effective_user_filter],
+          :template_inputs_attributes => [template_input_params_filter]
+        add_template_params_filter(filter)
+        add_taxonomix_params_filter(filter)
+      end
+    end
+  end
+
+  def output_template_params
+    self.class.output_template_params_filter.filter_params(params, parameter_filter_context, :output_template)
+  end
+end

app/controllers/output_templates_controller.rb

@@ -0,0 +1,18 @@
+class OutputTemplatesController < ::TemplatesController
+  include ::Foreman::Controller::Parameters::OutputTemplate
+
+  def import
+    contents = params.fetch(:imported_template, {}).fetch(:template, nil).try(:read)
+
+    @template = OutputTemplate.import_raw(contents, :update => ActiveRecord::Type::Boolean.new.deserialize(params[:imported_template][:overwrite]))
+    if @template&.save
+      flash[:success] = _('Output template imported successfully.')
+      redirect_to output_templates_path(:search => "name = \"#{@template.name}\"")
+    else
+      @template ||= OutputTemplate.import_raw(contents, :build_new => true)
+      @template.valid?
+      flash[:warning] = _('Unable to save template. Correct highlighted errors')
+      render :action => 'new'
+    end
+  end
+end

app/controllers/template_invocations_controller.rb

@@ -15,6 +15,8 @@ def show
     @since = params[:since].to_f if params[:since].present?
     @line_sets = @template_invocation_task.main_action.live_output
     @line_sets = @line_sets.drop_while { |o| o['timestamp'].to_f <= @since } if @since
+    @template_output_sets = @line_sets.select { |o| o['output_type'] == 'template_output' || o['output_type'] == 'template_output_err' }
+    @line_sets.select! { |o| o['output_type'] != 'template_output' && o['output_type'] != 'template_output_err' }
     @line_counter = params[:line_counter].to_i
   end
 end

app/controllers/ui_job_wizard_controller.rb

@@ -20,6 +20,7 @@ def template
       :template_inputs => template_inputs,
       :provider_name => job_template.provider.provider_input_namespace,
       :advanced_template_inputs => advanced_template_inputs+provider_inputs,
+      :default_output_templates => job_template.output_templates,
     }
   end
 

app/lib/actions/remote_execution/output_processing_action.rb

@@ -0,0 +1,45 @@
+module Actions
+  module RemoteExecution
+    class OutputProcessing < Dynflow::Action
+
+      def process_proxy_template(output, template, invocation)
+        base = Host.authorized(:view_hosts, Host)
+        # provide host information for the output template rendering
+        host = base.find(invocation.host_id)
+        renderer = InputTemplateRenderer.new(template, host, invocation, nil, false, [], output)
+        processed_output = renderer.render
+        unless processed_output
+          return renderer.error_message.html_safe, false
+        end
+        return processed_output, true
+      end
+
+      def run
+        processed_outputs = []
+        template_invocation = TemplateInvocation.find(input[:template_invocation_id])
+        events = template_invocation.template_invocation_events
+        sq_id = events.max_by { |e| e.sequence_id }.sequence_id + 1
+        output_templates = template_invocation.job_invocation.output_templates
+        output_templates.each_with_index.map do |output_templ, templ_id|
+          for i in 0..events.length - 1 do
+            if events[i][:event].instance_of?(String) && events[i][:event_type] == 'stdout'
+              output, success = process_proxy_template(events[i][:event], output_templ, template_invocation)
+              processed_outputs << {
+                sequence_id: sq_id,
+                template_invocation_id: template_invocation.id,
+                event: output,
+                timestamp: events[i][:timestamp] || Time.zone.now,
+                event_type: success ? 'template_output' : 'template_output_err',
+              }
+              # template invocation id and a sequence combination has to be unique
+              sq_id += 1
+            end
+          end
+        end
+        processed_outputs.each_slice(1000) do |batch|
+          TemplateInvocationEvent.upsert_all(batch, unique_by: [:template_invocation_id, :sequence_id]) # rubocop:disable Rails/SkipsModelValidations
+        end
+      end
+    end
+  end
+end

app/lib/actions/remote_execution/run_host_job.rb

@@ -68,9 +68,12 @@ def inner_plan(job_invocation, host, template_invocation, proxy_selector, option
                                :alternative_names => provider.alternative_names(host) }
         action_options = provider.proxy_command_options(template_invocation, host)
                                  .merge(additional_options)
-
-        plan_delegated_action(proxy, provider.proxy_action_class, action_options, proxy_action_class: ::Actions::RemoteExecution::ProxyAction)
-        plan_self :with_event_logging => true
+        # Defines the order between planned actions.
+        sequence do
+          plan_delegated_action(proxy, provider.proxy_action_class, action_options, proxy_action_class: ::Actions::RemoteExecution::ProxyAction)
+          plan_self :with_event_logging => true
+          plan_action(::Actions::RemoteExecution::OutputProcessing, template_invocation_id: template_invocation.id)
+        end
       end
 
       def finalize(*args)
@@ -274,9 +277,10 @@ def determine_proxy!(proxy_selector, provider, host)
         property :host, object_of: 'Host', desc: "Returns the host"
         property :job_invocation_id, Integer, desc: "Returns the id of the job invocation"
         property :job_invocation, object_of: 'JobInvocation', desc: "Returns the job invocation"
+        property :output, String, desc: "Returns the output of the template invocation"
       end
       class Jail < ::Actions::ObservableAction::Jail
-        allow :host_name, :host_id, :host, :job_invocation_id, :job_invocation
+        allow :host_name, :host_id, :host, :job_invocation_id, :job_invocation, :output
       end
     end
   end

app/lib/actions/remote_execution/run_hosts_job.rb

@@ -185,9 +185,11 @@ def cache_deletion_query(job_invocation_id)
         property :task, object_of: 'Task', desc: 'Returns the task to which this action belongs'
         property :job_invocation_id, Integer, desc: "Returns the id of the job invocation"
         property :job_invocation, object_of: 'JobInvocation', desc: "Returns the job invocation"
+        property :output, String, desc: "Returns the output of the template invocation"
       end
       class Jail < ::Actions::ObservableAction::Jail
-        allow :job_invocation_id, :job_invocation
+        # enables variables in the template
+        allow :job_invocation_id, :job_invocation, :output
       end
     end
   end

app/models/concerns/foreman_remote_execution/taxonomy_extensions.rb

@@ -4,6 +4,7 @@ module TaxonomyExtensions
 
     included do
       has_many :job_templates, :through => :taxable_taxonomies, :source => :taxable, :source_type => 'JobTemplate'
+      has_many :output_templates, :through => :taxable_taxonomies, :source => :taxable, :source_type => 'OutputTemplate'
 
       # TODO: on foreman_version_bump
       # workaround for #11805 - use before_create for setting

app/models/input_template_renderer.rb

@@ -7,12 +7,12 @@ class RenderError < ::Foreman::Exception
 
   delegate :logger, to: Rails
 
-  attr_accessor :template, :host, :invocation, :template_input_values, :error_message, :templates_stack, :current_user
+  attr_accessor :template, :host, :invocation, :template_input_values, :error_message, :templates_stack, :current_user, :output
 
   # takes template object that should be rendered
   # host and template invocation arguments are optional
   # so we can render values based on parameters, facts or user inputs
-  def initialize(template, host = nil, invocation = nil, input_values = nil, preview = false, templates_stack = [])
+  def initialize(template, host = nil, invocation = nil, input_values = nil, preview = false, templates_stack = [], output = "")
     raise Foreman::Exception, N_('Recursive rendering of templates detected') if templates_stack.include?(template)
 
     @host = host
@@ -21,6 +21,8 @@ def initialize(template, host = nil, invocation = nil, input_values = nil, previ
     @template_input_values = input_values
     @preview = preview
     @templates_stack = templates_stack + [template]
+    # gives templates the access to the output variable
+    @output = output
   end
 
   def render
@@ -44,6 +46,7 @@ def render
         templates_stack: templates_stack,
         input_template_instance: self,
         current_user: User.current.try(:login),
+        output: output,
       }
     )
     Foreman::Renderer.render(source, @scope)

app/models/job_invocation.rb

@@ -90,6 +90,10 @@ class JobInvocation < ApplicationRecord
 
   encrypts :password, :key_passphrase, :effective_user_password
 
+  # join table for linking output templates
+  has_many :job_invocation_templates, dependent: :destroy
+  has_many :output_templates, through: :job_invocation_templates
+
   class Jail < Safemode::Jail
     allow :sub_task_for_host, :template_invocations_hosts
   end

app/models/job_invocation_composer.rb

@@ -14,6 +14,8 @@ def params
         :targeting => targeting(ui_params.fetch(:targeting, {})),
         :triggering => triggering,
         :host_ids => ui_params[:host_ids],
+        :output_template_ids => ui_params[:output_template_ids] || [],
+        :runtime_templates => ui_params[:runtime_templates] || [],
         :remote_execution_feature_id => job_invocation_base[:remote_execution_feature_id],
         :description_format => job_invocation_base[:description_format],
         :ssh_user => blank_to_nil(job_invocation_base[:ssh_user]),
@@ -131,7 +133,9 @@ def params
         :concurrency_control => concurrency_control_params,
         :execution_timeout_interval => api_params[:execution_timeout_interval] || template.execution_timeout_interval,
         :time_to_pickup => api_params[:time_to_pickup],
-        :template_invocations => template_invocations_params }.with_indifferent_access
+        :template_invocations => template_invocations_params,
+        :runtime_templates => api_params[:runtime_templates] || [],
+        :output_template_ids => api_params[:output_template_ids] || [] }.with_indifferent_access
     end
 
     def remote_execution_feature_id
@@ -235,6 +239,9 @@ def initialize(job_invocation, params = {})
       elsif params[:failed_only]
         @host_ids = job_invocation.failed_host_ids
       end
+      if params[:output_template_ids]
+        @output_template_ids = params[:output_template_ids]
+      end
     end
 
     def params
@@ -373,7 +380,7 @@ def job_template
 
   attr_accessor :params, :job_invocation, :host_ids, :search_query
   attr_reader :reruns
-  delegate :job_category, :remote_execution_feature_id, :pattern_template_invocations, :template_invocations, :targeting, :triggering, :to => :job_invocation
+  delegate :job_category, :remote_execution_feature_id, :pattern_template_invocations, :template_invocations, :targeting, :triggering, :output_templates, :to => :job_invocation
 
   def initialize(params, set_defaults = false)
     @params = params
@@ -384,6 +391,7 @@ def initialize(params, set_defaults = false)
     compose
 
     @host_ids = validate_host_ids(params[:host_ids])
+    @output_templates_ids = params[:output_template_ids]
     @search_query = job_invocation.targeting.search_query if job_invocation.targeting.bookmark_id.blank?
   end
 
@@ -430,13 +438,32 @@ def compose
     self
   end
 
+  def build_output_templates
+    params[:output_template_ids].map do |output_t|
+      job_invocation.output_templates << OutputTemplate.find(output_t)
+    end
+    params[:runtime_templates].map.with_index do |output_t, index|
+      # Runtime templates need unique name
+      name = DateTime.now.to_i.to_s + " runtime template " + index.to_s
+      # runtime template are not yet saved, they have to be built
+      job_invocation.output_templates.build(:template => output_t, :name => name, :snippet => true)
+    end
+  end
+
   def trigger(raise_on_error = false)
+    # starts the job invocation Dynflow action
     generate_description
     if raise_on_error
       save!
     else
       return false unless save
     end
+    build_output_templates
+    if raise_on_error
+      save!
+    else
+      return false unless save
+    end
     triggering.trigger(::Actions::RemoteExecution::RunHostsJob, job_invocation)
   end
 

app/models/job_invocation_template.rb

@@ -0,0 +1,4 @@
+class JobInvocationTemplate < ApplicationRecord
+  belongs_to :job_invocation
+  belongs_to :output_template
+end

app/models/job_template.rb

@@ -23,6 +23,9 @@ class NonUniqueInputsError < Foreman::Exception
   # rubocop:enable Rails/HasManyOrHasOneDependent
   has_many :remote_execution_features, :dependent => :nullify
 
+  has_many :job_template_output_templates, dependent: :destroy
+  has_many :output_templates, through: :job_template_output_templates
+
   # these can't be shared in parent class, scoped search can't handle STI properly
   # tested with scoped_search 3.2.0
   include Taxonomix

app/models/job_template_output_template.rb

@@ -0,0 +1,4 @@
+class JobTemplateOutputTemplate < ApplicationRecord
+  belongs_to :job_template
+  belongs_to :output_template
+end