diff --git a/app/assets/javascripts/foreman_remote_execution/output_templates.js b/app/assets/javascripts/foreman_remote_execution/output_templates.js new file mode 100644 index 000000000..bd32b8b7f --- /dev/null +++ b/app/assets/javascripts/foreman_remote_execution/output_templates.js @@ -0,0 +1,10 @@ +function show_import_output_template_modal() { + var modal_window = $('#importOutputTemplateModal'); + modal_window.modal({'show': true}); + modal_window.find('a[rel="popover-modal"]').popover(); +} + +function close_import_output_template_modal() { + var modal_window = $('#importOutputTemplateModal'); + modal_window.modal('hide'); +} diff --git a/app/assets/stylesheets/foreman_remote_execution/template_invocation.scss b/app/assets/stylesheets/foreman_remote_execution/template_invocation.scss index bf9bdccfc..f868fed7e 100644 --- a/app/assets/stylesheets/foreman_remote_execution/template_invocation.scss +++ b/app/assets/stylesheets/foreman_remote_execution/template_invocation.scss @@ -32,7 +32,7 @@ div.terminal { min-height: 50px; } - div.line.stderr, div.line.error, div.line.debug { + div.line.stderr, div.line.error, div.line.debug, div.line.output_templates, div.line.output_templates_err { color: red; } diff --git a/app/controllers/api/v2/output_templates_controller.rb b/app/controllers/api/v2/output_templates_controller.rb new file mode 100644 index 000000000..6f2e590fd --- /dev/null +++ b/app/controllers/api/v2/output_templates_controller.rb @@ -0,0 +1,61 @@ +module Api + module V2 + class OutputTemplatesController < ::Api::V2::BaseController + include ::Api::Version2 + include ::Foreman::Renderer + include ::Foreman::Controller::ProvisioningTemplates + include ::Foreman::Controller::Parameters::OutputTemplate + + api :GET, '/output_templates/', N_('List output templates') + # location and Organization + param_group :taxonomy_scope, ::Api::V2::BaseController + # search and pagination allows to display and filter the index page of templates + param_group :search_and_pagination, ::Api::V2::BaseController + def index + # do not show saved runtime templates + @output_templates = resource_scope_for_index.filter { |template| !template.snippet } + end + + def_param_group :output_template do + param :output_template, Hash, :required => true, :action_aware => true do + param :name, String, :required => true, :desc => N_('Template name') + param :description, String + param :template, String, :required => true + param :output, String + param :snippet, :bool, :allow_nil => true + param :locked, :bool, :desc => N_('Whether or not the template is locked for editing') + param :effective_user_attributes, Hash, :desc => N_('Effective user options') do + param :value, String, :desc => N_('What user should be used to run the script (using sudo-like mechanisms)'), :allowed_nil => true + param :overridable, :bool, :desc => N_('Whether it should be allowed to override the effective user from the invocation form.') + param :current_user, :bool, :desc => N_('Whether the current user login should be used as the effective user') + end + param_group :taxonomies, ::Api::V2::BaseController + end + end + + api :POST, '/output_templates/', N_('Create an output template') + param_group :output_template, :as => :create + def create + @output_template = OutputTemplate.new(output_template_params) + process_response @output_template.save + end + + api :DELETE, '/output_templates/:id', N_('Delete an output template') + param :id, :identifier, :required => true + def destroy + process_response @output_template.destroy + end + + api :POST, '/output_templates/import', N_('Import an output template from ERB') + param :template, String, :required => true, :desc => N_('Template ERB') + param :overwrite, :bool, :required => false, :desc => N_('Overwrite template if it already exists') + def import + options = params[:overwrite] ? { :update => true } : { :build_new => true } + + @output_template = OutputTemplate.import_raw(params[:template], options) + @output_template ||= OutputTemplate.new + process_response @output_template.save + end + end + end +end diff --git a/app/controllers/concerns/foreman/controller/parameters/job_template.rb b/app/controllers/concerns/foreman/controller/parameters/job_template.rb index 073a10cba..3c7861d3d 100644 --- a/app/controllers/concerns/foreman/controller/parameters/job_template.rb +++ b/app/controllers/concerns/foreman/controller/parameters/job_template.rb @@ -15,7 +15,7 @@ def job_template_effective_user_filter def job_template_params_filter Foreman::ParameterFilter.new(::TemplateInput).tap do |filter| - filter.permit :job_category, :provider_type, :description_format, :execution_timeout_interval, + filter.permit :job_category, :provider_type, :description_format, :execution_timeout_interval, :output_template_ids => [], :effective_user_attributes => [job_template_effective_user_filter], :template_inputs_attributes => [template_input_params_filter], :foreign_input_sets_attributes => [foreign_input_set_params_filter] diff --git a/app/controllers/concerns/foreman/controller/parameters/output_template.rb b/app/controllers/concerns/foreman/controller/parameters/output_template.rb new file mode 100644 index 000000000..5d782bc34 --- /dev/null +++ b/app/controllers/concerns/foreman/controller/parameters/output_template.rb @@ -0,0 +1,29 @@ +module Foreman::Controller::Parameters::OutputTemplate + extend ActiveSupport::Concern + include Foreman::Controller::Parameters::Taxonomix + include ::Foreman::Controller::Parameters::Template + include Foreman::Controller::Parameters::TemplateInput + + class_methods do + def output_template_effective_user_filter + Foreman::ParameterFilter.new(::OutputTemplateEffectiveUser).tap do |filter| + filter.permit_by_context(:value, :current_user, :overridable, + :nested => true) + end + end + + def output_template_params_filter + Foreman::ParameterFilter.new(::TemplateInput).tap do |filter| + filter.permit :description_format, + :effective_user_attributes => [output_template_effective_user_filter], + :template_inputs_attributes => [template_input_params_filter] + add_template_params_filter(filter) + add_taxonomix_params_filter(filter) + end + end + end + + def output_template_params + self.class.output_template_params_filter.filter_params(params, parameter_filter_context, :output_template) + end +end diff --git a/app/controllers/output_templates_controller.rb b/app/controllers/output_templates_controller.rb new file mode 100644 index 000000000..ae5d356e9 --- /dev/null +++ b/app/controllers/output_templates_controller.rb @@ -0,0 +1,18 @@ +class OutputTemplatesController < ::TemplatesController + include ::Foreman::Controller::Parameters::OutputTemplate + + def import + contents = params.fetch(:imported_template, {}).fetch(:template, nil).try(:read) + + @template = OutputTemplate.import_raw(contents, :update => ActiveRecord::Type::Boolean.new.deserialize(params[:imported_template][:overwrite])) + if @template&.save + flash[:success] = _('Output template imported successfully.') + redirect_to output_templates_path(:search => "name = \"#{@template.name}\"") + else + @template ||= OutputTemplate.import_raw(contents, :build_new => true) + @template.valid? + flash[:warning] = _('Unable to save template. Correct highlighted errors') + render :action => 'new' + end + end +end diff --git a/app/controllers/template_invocations_controller.rb b/app/controllers/template_invocations_controller.rb index 1ddfb0633..baa0f334f 100644 --- a/app/controllers/template_invocations_controller.rb +++ b/app/controllers/template_invocations_controller.rb @@ -15,6 +15,8 @@ def show @since = params[:since].to_f if params[:since].present? @line_sets = @template_invocation_task.main_action.live_output @line_sets = @line_sets.drop_while { |o| o['timestamp'].to_f <= @since } if @since + @template_output_sets = @line_sets.select { |o| o['output_type'] == 'template_output' || o['output_type'] == 'template_output_err' } + @line_sets.select! { |o| o['output_type'] != 'template_output' && o['output_type'] != 'template_output_err' } @line_counter = params[:line_counter].to_i end end diff --git a/app/controllers/ui_job_wizard_controller.rb b/app/controllers/ui_job_wizard_controller.rb index 015ee412a..5ccf39a85 100644 --- a/app/controllers/ui_job_wizard_controller.rb +++ b/app/controllers/ui_job_wizard_controller.rb @@ -20,6 +20,7 @@ def template :template_inputs => template_inputs, :provider_name => job_template.provider.provider_input_namespace, :advanced_template_inputs => advanced_template_inputs+provider_inputs, + :default_output_templates => job_template.output_templates, } end diff --git a/app/lib/actions/remote_execution/output_processing_action.rb b/app/lib/actions/remote_execution/output_processing_action.rb new file mode 100644 index 000000000..ea5706731 --- /dev/null +++ b/app/lib/actions/remote_execution/output_processing_action.rb @@ -0,0 +1,45 @@ +module Actions + module RemoteExecution + class OutputProcessing < Dynflow::Action + + def process_proxy_template(output, template, invocation) + base = Host.authorized(:view_hosts, Host) + # provide host information for the output template rendering + host = base.find(invocation.host_id) + renderer = InputTemplateRenderer.new(template, host, invocation, nil, false, [], output) + processed_output = renderer.render + unless processed_output + return renderer.error_message.html_safe, false + end + return processed_output, true + end + + def run + processed_outputs = [] + template_invocation = TemplateInvocation.find(input[:template_invocation_id]) + events = template_invocation.template_invocation_events + sq_id = events.max_by { |e| e.sequence_id }.sequence_id + 1 + output_templates = template_invocation.job_invocation.output_templates + output_templates.each_with_index.map do |output_templ, templ_id| + for i in 0..events.length - 1 do + if events[i][:event].instance_of?(String) && events[i][:event_type] == 'stdout' + output, success = process_proxy_template(events[i][:event], output_templ, template_invocation) + processed_outputs << { + sequence_id: sq_id, + template_invocation_id: template_invocation.id, + event: output, + timestamp: events[i][:timestamp] || Time.zone.now, + event_type: success ? 'template_output' : 'template_output_err', + } + # template invocation id and a sequence combination has to be unique + sq_id += 1 + end + end + end + processed_outputs.each_slice(1000) do |batch| + TemplateInvocationEvent.upsert_all(batch, unique_by: [:template_invocation_id, :sequence_id]) # rubocop:disable Rails/SkipsModelValidations + end + end + end + end +end diff --git a/app/lib/actions/remote_execution/run_host_job.rb b/app/lib/actions/remote_execution/run_host_job.rb index 5341d170e..b0e0af93f 100644 --- a/app/lib/actions/remote_execution/run_host_job.rb +++ b/app/lib/actions/remote_execution/run_host_job.rb @@ -68,9 +68,12 @@ def inner_plan(job_invocation, host, template_invocation, proxy_selector, option :alternative_names => provider.alternative_names(host) } action_options = provider.proxy_command_options(template_invocation, host) .merge(additional_options) - - plan_delegated_action(proxy, provider.proxy_action_class, action_options, proxy_action_class: ::Actions::RemoteExecution::ProxyAction) - plan_self :with_event_logging => true + # Defines the order between planned actions. + sequence do + plan_delegated_action(proxy, provider.proxy_action_class, action_options, proxy_action_class: ::Actions::RemoteExecution::ProxyAction) + plan_self :with_event_logging => true + plan_action(::Actions::RemoteExecution::OutputProcessing, template_invocation_id: template_invocation.id) + end end def finalize(*args) @@ -274,9 +277,10 @@ def determine_proxy!(proxy_selector, provider, host) property :host, object_of: 'Host', desc: "Returns the host" property :job_invocation_id, Integer, desc: "Returns the id of the job invocation" property :job_invocation, object_of: 'JobInvocation', desc: "Returns the job invocation" + property :output, String, desc: "Returns the output of the template invocation" end class Jail < ::Actions::ObservableAction::Jail - allow :host_name, :host_id, :host, :job_invocation_id, :job_invocation + allow :host_name, :host_id, :host, :job_invocation_id, :job_invocation, :output end end end diff --git a/app/lib/actions/remote_execution/run_hosts_job.rb b/app/lib/actions/remote_execution/run_hosts_job.rb index cb823fe60..84491306f 100644 --- a/app/lib/actions/remote_execution/run_hosts_job.rb +++ b/app/lib/actions/remote_execution/run_hosts_job.rb @@ -185,9 +185,11 @@ def cache_deletion_query(job_invocation_id) property :task, object_of: 'Task', desc: 'Returns the task to which this action belongs' property :job_invocation_id, Integer, desc: "Returns the id of the job invocation" property :job_invocation, object_of: 'JobInvocation', desc: "Returns the job invocation" + property :output, String, desc: "Returns the output of the template invocation" end class Jail < ::Actions::ObservableAction::Jail - allow :job_invocation_id, :job_invocation + # enables variables in the template + allow :job_invocation_id, :job_invocation, :output end end end diff --git a/app/models/concerns/foreman_remote_execution/taxonomy_extensions.rb b/app/models/concerns/foreman_remote_execution/taxonomy_extensions.rb index 8528b44ff..acfef9a89 100644 --- a/app/models/concerns/foreman_remote_execution/taxonomy_extensions.rb +++ b/app/models/concerns/foreman_remote_execution/taxonomy_extensions.rb @@ -4,6 +4,7 @@ module TaxonomyExtensions included do has_many :job_templates, :through => :taxable_taxonomies, :source => :taxable, :source_type => 'JobTemplate' + has_many :output_templates, :through => :taxable_taxonomies, :source => :taxable, :source_type => 'OutputTemplate' # TODO: on foreman_version_bump # workaround for #11805 - use before_create for setting diff --git a/app/models/input_template_renderer.rb b/app/models/input_template_renderer.rb index 19aada6c3..0b005d2b3 100644 --- a/app/models/input_template_renderer.rb +++ b/app/models/input_template_renderer.rb @@ -7,12 +7,12 @@ class RenderError < ::Foreman::Exception delegate :logger, to: Rails - attr_accessor :template, :host, :invocation, :template_input_values, :error_message, :templates_stack, :current_user + attr_accessor :template, :host, :invocation, :template_input_values, :error_message, :templates_stack, :current_user, :output # takes template object that should be rendered # host and template invocation arguments are optional # so we can render values based on parameters, facts or user inputs - def initialize(template, host = nil, invocation = nil, input_values = nil, preview = false, templates_stack = []) + def initialize(template, host = nil, invocation = nil, input_values = nil, preview = false, templates_stack = [], output = "") raise Foreman::Exception, N_('Recursive rendering of templates detected') if templates_stack.include?(template) @host = host @@ -21,6 +21,8 @@ def initialize(template, host = nil, invocation = nil, input_values = nil, previ @template_input_values = input_values @preview = preview @templates_stack = templates_stack + [template] + # gives templates the access to the output variable + @output = output end def render @@ -44,6 +46,7 @@ def render templates_stack: templates_stack, input_template_instance: self, current_user: User.current.try(:login), + output: output, } ) Foreman::Renderer.render(source, @scope) diff --git a/app/models/job_invocation.rb b/app/models/job_invocation.rb index feb366c8f..ef558b277 100644 --- a/app/models/job_invocation.rb +++ b/app/models/job_invocation.rb @@ -90,6 +90,10 @@ class JobInvocation < ApplicationRecord encrypts :password, :key_passphrase, :effective_user_password + # join table for linking output templates + has_many :job_invocation_templates, dependent: :destroy + has_many :output_templates, through: :job_invocation_templates + class Jail < Safemode::Jail allow :sub_task_for_host, :template_invocations_hosts end diff --git a/app/models/job_invocation_composer.rb b/app/models/job_invocation_composer.rb index 9154ef1be..8c33fa965 100644 --- a/app/models/job_invocation_composer.rb +++ b/app/models/job_invocation_composer.rb @@ -14,6 +14,8 @@ def params :targeting => targeting(ui_params.fetch(:targeting, {})), :triggering => triggering, :host_ids => ui_params[:host_ids], + :output_template_ids => ui_params[:output_template_ids] || [], + :runtime_templates => ui_params[:runtime_templates] || [], :remote_execution_feature_id => job_invocation_base[:remote_execution_feature_id], :description_format => job_invocation_base[:description_format], :ssh_user => blank_to_nil(job_invocation_base[:ssh_user]), @@ -131,7 +133,9 @@ def params :concurrency_control => concurrency_control_params, :execution_timeout_interval => api_params[:execution_timeout_interval] || template.execution_timeout_interval, :time_to_pickup => api_params[:time_to_pickup], - :template_invocations => template_invocations_params }.with_indifferent_access + :template_invocations => template_invocations_params, + :runtime_templates => api_params[:runtime_templates] || [], + :output_template_ids => api_params[:output_template_ids] || [] }.with_indifferent_access end def remote_execution_feature_id @@ -235,6 +239,9 @@ def initialize(job_invocation, params = {}) elsif params[:failed_only] @host_ids = job_invocation.failed_host_ids end + if params[:output_template_ids] + @output_template_ids = params[:output_template_ids] + end end def params @@ -373,7 +380,7 @@ def job_template attr_accessor :params, :job_invocation, :host_ids, :search_query attr_reader :reruns - delegate :job_category, :remote_execution_feature_id, :pattern_template_invocations, :template_invocations, :targeting, :triggering, :to => :job_invocation + delegate :job_category, :remote_execution_feature_id, :pattern_template_invocations, :template_invocations, :targeting, :triggering, :output_templates, :to => :job_invocation def initialize(params, set_defaults = false) @params = params @@ -384,6 +391,7 @@ def initialize(params, set_defaults = false) compose @host_ids = validate_host_ids(params[:host_ids]) + @output_templates_ids = params[:output_template_ids] @search_query = job_invocation.targeting.search_query if job_invocation.targeting.bookmark_id.blank? end @@ -430,13 +438,32 @@ def compose self end + def build_output_templates + params[:output_template_ids].map do |output_t| + job_invocation.output_templates << OutputTemplate.find(output_t) + end + params[:runtime_templates].map.with_index do |output_t, index| + # Runtime templates need unique name + name = DateTime.now.to_i.to_s + " runtime template " + index.to_s + # runtime template are not yet saved, they have to be built + job_invocation.output_templates.build(:template => output_t, :name => name, :snippet => true) + end + end + def trigger(raise_on_error = false) + # starts the job invocation Dynflow action generate_description if raise_on_error save! else return false unless save end + build_output_templates + if raise_on_error + save! + else + return false unless save + end triggering.trigger(::Actions::RemoteExecution::RunHostsJob, job_invocation) end diff --git a/app/models/job_invocation_template.rb b/app/models/job_invocation_template.rb new file mode 100644 index 000000000..fb80724b3 --- /dev/null +++ b/app/models/job_invocation_template.rb @@ -0,0 +1,4 @@ +class JobInvocationTemplate < ApplicationRecord + belongs_to :job_invocation + belongs_to :output_template +end diff --git a/app/models/job_template.rb b/app/models/job_template.rb index 3584e26ef..fc70bbf4d 100644 --- a/app/models/job_template.rb +++ b/app/models/job_template.rb @@ -23,6 +23,9 @@ class NonUniqueInputsError < Foreman::Exception # rubocop:enable Rails/HasManyOrHasOneDependent has_many :remote_execution_features, :dependent => :nullify + has_many :job_template_output_templates, dependent: :destroy + has_many :output_templates, through: :job_template_output_templates + # these can't be shared in parent class, scoped search can't handle STI properly # tested with scoped_search 3.2.0 include Taxonomix diff --git a/app/models/job_template_output_template.rb b/app/models/job_template_output_template.rb new file mode 100644 index 000000000..fb6790a80 --- /dev/null +++ b/app/models/job_template_output_template.rb @@ -0,0 +1,4 @@ +class JobTemplateOutputTemplate < ApplicationRecord + belongs_to :job_template + belongs_to :output_template +end diff --git a/app/models/output_template.rb b/app/models/output_template.rb new file mode 100644 index 000000000..835f5e352 --- /dev/null +++ b/app/models/output_template.rb @@ -0,0 +1,57 @@ +class OutputTemplate < ::Template + audited + include Taxonomix + include Authorizable + + scoped_search :on => :id, :complete_enabled => false, :only_explicit => true, :validator => ScopedSearch::Validators::INTEGER + scoped_search :on => :name, :complete_value => true, :default_order => true + scoped_search :on => :locked, :complete_value => {:true => true, :false => false} + scoped_search :on => :snippet, :complete_value => {:true => true, :false => false} + + has_many :audits, :as => :auditable, :class_name => Audited.audit_class.name, :dependent => :nullify + validates :name, :uniqueness => true + validates :template, :presence => true + + has_many :job_invocation_templates, dependent: :destroy + has_many :job_invocations, through: :job_invocation_templates + + has_many :job_template_output_templates, dependent: :destroy + has_many :job_templates, through: :job_template_output_templates + + class << self + # we have to override the base_class because polymorphic associations does not detect it correctly, more details at + # http://apidock.com/rails/ActiveRecord/Associations/ClassMethods/has_many#1010-Polymorphic-has-many-within-inherited-class-gotcha + def base_class + self + end + + # allows importing templates in the controller + def import_raw(contents, options = {}) + metadata = Template.parse_metadata(contents) + import_parsed(metadata['name'], contents, metadata, options) + end + + def import_raw!(contents, options = {}) + template = import_raw(contents, options) + template&.save! + template + end + + def import_parsed(name, text, _metadata, options = {}) + transaction do + # Do not search for duplicates in case we want to always create new template + existing = self.find_by(:name => name) if !options.delete(:build_new) + # Do not save duplicates + return unless options.delete(:update) && existing + + template = existing || self.new(:name => name) + template.import_without_save(text, options) + template + end + end + end + + def default_input_values(ignore_keys) + return {} + end +end diff --git a/app/models/output_template_effective_user.rb b/app/models/output_template_effective_user.rb new file mode 100644 index 000000000..8390a19d2 --- /dev/null +++ b/app/models/output_template_effective_user.rb @@ -0,0 +1,19 @@ +class OutputTemplateEffectiveUser < ApplicationRecord + belongs_to :output_template + before_validation :set_defaults + + def set_defaults + self.overridable = true if self.overridable.nil? + self.current_user = false if self.current_user.nil? + end + + def compute_value + if current_user? + User.current.login + elsif value.present? + value + else + Setting[:remote_execution_effective_user] + end + end +end diff --git a/app/views/api/v2/output_templates/base.json.rabl b/app/views/api/v2/output_templates/base.json.rabl new file mode 100644 index 000000000..841e0d542 --- /dev/null +++ b/app/views/api/v2/output_templates/base.json.rabl @@ -0,0 +1,3 @@ +object @output_template + +attributes :id, :name, :provider_type, :snippet, :description_format, :created_at, :updated_at diff --git a/app/views/api/v2/output_templates/index.json.rabl b/app/views/api/v2/output_templates/index.json.rabl new file mode 100644 index 000000000..2c8737cc2 --- /dev/null +++ b/app/views/api/v2/output_templates/index.json.rabl @@ -0,0 +1,3 @@ +collection @output_templates + +extends 'api/v2/output_templates/base' diff --git a/app/views/job_templates/_custom_tab_headers.html.erb b/app/views/job_templates/_custom_tab_headers.html.erb index 3ceff6995..92ac3008a 100644 --- a/app/views/job_templates/_custom_tab_headers.html.erb +++ b/app/views/job_templates/_custom_tab_headers.html.erb @@ -1,2 +1,3 @@
<%= sort :name, :as => s_("OutputTemplate|Name") %> | +<%= sort :snippet, :as => s_("OutputTemplate|Snippet") %> | +<%= sort :locked, :as => s_("OutputTemplate|Locked"), :default => "DESC" %> | +<%= _('Actions') %> | +
---|---|---|---|
<%= link_to_if_authorized output_template, + hash_for_edit_output_template_path(:id => output_template).merge(:auth_object => output_template, :authorizer => authorizer) %> | +<%= checked_icon output_template.snippet %> | ++ <%= locked_icon output_template.locked?, _("This template is locked for editing.") %> + | ++ <%= action_buttons(*permitted_actions(output_template)) %> + | +