Generate HomeOffice report from template

.rubocop.yml

@@ -110,5 +110,3 @@ RSpec/AnyInstance:
 RSpec/MultipleMemoizedHelpers:
   Exclude:
     - 'spec/models/event_spec.rb'
-
-

Gemfile

@@ -89,3 +89,5 @@ gem "dartsass-rails", "~> 0.5.0"
 gem "importmap-rails", "~> 1.2"
 
 gem "propshaft", "~> 0.8.0"
+
+gem "rubyXL", "~> 3.4"

Gemfile.lock

@@ -428,6 +428,10 @@ GEM
       rubocop-factory_bot (~> 2.22)
     ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
+    rubyXL (3.4.25)
+      nokogiri (>= 1.10.8)
+      rubyzip (>= 1.3.0)
+    rubyzip (2.3.2)
     sanitize (6.1.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
@@ -535,6 +539,7 @@ DEPENDENCIES
   rubocop-performance
   rubocop-rails
   rubocop-rspec
+  rubyXL (~> 3.4)
   scenic
   sentry-rails (~> 5.12)
   shoulda-matchers (~> 5.0)

app/models/report_template.rb

@@ -0,0 +1,22 @@
+# == Schema Information
+#
+# Table name: report_templates
+#
+#  id           :bigint           not null, primary key
+#  config       :jsonb
+#  file         :binary
+#  filename     :string
+#  report_class :string
+#  created_at   :datetime         not null
+#  updated_at   :datetime         not null
+#
+class ReportTemplate < ApplicationRecord
+  validates(:file, presence: true)
+  validates(:filename, presence: true)
+  validates(:report_class, presence: true, uniqueness: true)
+  validates(:config, presence: true)
+
+  validate do |record|
+    HomeOfficeReportConfigValidator.new(record).validate
+  end
+end

app/models/reports/applications.rb

@@ -1,6 +1,6 @@
 module Reports
   class Applications < Base
-    def csv
+    def generate
       CSV.generate do |csv|
         csv << header
         rows.find_each(batch_size: 50) { |row| csv << columns(row) }

app/models/reports/base.rb

@@ -7,13 +7,23 @@ def initialize(**kwargs)
 
     attr_reader :name, :kwargs
 
+    class << self
+      def file_ext(value)
+        @file_ext = value
+      end
+
+      def get_file_ext
+        @file_ext || "csv"
+      end
+    end
+
     def filename
       current_time = Time.zone.now.strftime("%Y%m%d-%H%M%S")
 
-      "#{name}-#{current_time}.csv"
+      "#{name}-#{current_time}.#{self.class.get_file_ext}"
     end
 
-    def csv; end
+    def generate; end
 
     def post_generation_hook; end
   end

app/models/reports/home_office.rb → app/models/reports/home_office_csv.rb

@@ -1,8 +1,8 @@
-# frozen_string_literal: true
-
 module Reports
-  class HomeOffice < Base
-    def csv
+  class HomeOfficeCsv < Base
+    file_ext "csv"
+
+    def generate
       CSV.generate do |csv|
         csv << header
         rows.each { |row| csv << row }

app/models/reports/home_office_excel.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Reports
+  class HomeOfficeExcel < Base
+    file_ext "xlsx"
+
+    HEADER_MAPPINGS_KEY = "header_mappings"
+    WORKSHEET_NAME_KEY = "worksheet_name"
+
+    def generate
+      cell_coords.each { worksheet.add_cell(*_1) }
+      workbook.stream.string
+    end
+
+    def post_generation_hook
+      base_query.update_all(home_office_csv_downloaded_at: Time.zone.now) # rubocop:disable Rails/SkipsModelValidations
+    end
+
+  private
+
+    def workbook
+      @workbook ||= ::RubyXL::Parser.parse_buffer(template.file)
+    end
+
+    def template
+      @template ||= ReportTemplate.find_by!(report_class: self.class.name)
+    end
+
+    def worksheet
+      @worksheet ||= workbook[worksheet_name]
+    end
+
+    def worksheet_name
+      template.config.fetch(WORKSHEET_NAME_KEY)
+    end
+
+    def header_mappings
+      template.config.fetch(HEADER_MAPPINGS_KEY)
+    end
+
+    def headers_with_column_index
+      @headers_with_column_index ||= worksheet[0]
+        .cells
+        .each
+        .with_index
+        .map { |v, i| [v.value, i] }
+    end
+
+    def sheet_col_number(header_mapping)
+      _, col_number = headers_with_column_index.detect { |(header, _)| header == header_mapping }
+
+      col_number
+    end
+
+    def cell_coords
+      dataset.each.with_index.flat_map do |cols, sheet_row_number|
+        header_mappings.each.with_index.map do |(header_mapping, _), col_idx|
+          [sheet_row_number + 1, sheet_col_number(header_mapping), cols[col_idx].to_s]
+        end
+      end
+    end
+
+    def base_query
+      @base_query ||= Application
+        .joins(:application_progress)
+        .includes(:applicant)
+        .where.not(application_progresses: { initial_checks_completed_at: nil })
+        .where(
+          application_progresses: {
+            home_office_checks_completed_at: nil,
+            rejection_completed_at: nil,
+          },
+          home_office_csv_downloaded_at: nil,
+        )
+    end
+
+    def dataset
+      base_query.pluck(*dataset_fields)
+    end
+
+    def dataset_fields
+      header_mappings.values.map do |cols|
+        if cols.size == 1
+          cols.first
+        else
+          Arel.sql("CONCAT(#{cols.join(', \' \', ')})")
+        end
+      end
+    end
+  end
+end

app/models/reports/payroll.rb

@@ -2,7 +2,7 @@
 
 module Reports
   class Payroll < Base
-    def csv
+    def generate
       CSV.generate do |csv|
         csv << header
         rows.each { |row| csv << row }

app/models/reports/qa_report.rb

@@ -5,7 +5,7 @@ def initialize(...)
       @name = [@name, status].join("-")
     end
 
-    def csv
+    def generate
       CSV.generate do |csv|
         csv << header
         rows.each { |row| csv << row }

app/models/reports/standing_data.rb

@@ -2,7 +2,7 @@
 
 module Reports
   class StandingData < Base
-    def csv
+    def generate
       CSV.generate do |csv|
         csv << header
         rows.each { |row| csv << row }

app/services/report.rb

@@ -1,6 +1,7 @@
 class Report
   REGISTERED_REPORTS = {
-    home_office: Reports::HomeOffice,
+    home_office: Reports::HomeOfficeCsv,
+    home_office_excel: Reports::HomeOfficeExcel,
     standing_data: Reports::StandingData,
     payroll: Reports::Payroll,
     applications: Reports::Applications,
@@ -16,6 +17,9 @@ def self.call(...)
 
   def initialize(report_id, **kwargs)
     @kwargs = kwargs&.symbolize_keys || {}
+    if Flipper.enabled?(:home_office_excel) && report_id.to_sym == :home_office
+      report_id = :home_office_excel
+    end
     @report_class = REGISTERED_REPORTS.with_indifferent_access.fetch(report_id)
   rescue KeyError
     raise(ArgumentError, "Invalid report id #{report_id}")
@@ -26,7 +30,7 @@ def initialize(report_id, **kwargs)
   delegate :post_generation_hook, to: :report
 
   def data
-    @data ||= report.csv
+    @data ||= report.generate
   end
 
 private

app/validators/home_office_report_config_validator.rb

@@ -0,0 +1,52 @@
+# Example config
+# config:
+#    {
+#      worksheet_name: "Data",
+#      header_mapping: {
+#        "ID (Mandatory)"               => %w[urn],
+#        "Full Name/ Organisation Name" => %w[applicants.given_name applicants.middle_name applicants.family_name],
+#        "DOB"                          => %w[applicants.date_of_birth],
+#        "Nationality"                  => %w[applicants.nationality],
+#        "Passport Number"              => %w[applicants.passport_number],
+#      }
+#    }
+#
+
+class HomeOfficeReportConfigValidator
+  def initialize(record)
+    @record = record
+  end
+
+  def validate
+    return if record.report_class != Reports::HomeOfficeExcel.name
+
+    validate_workbook
+    validate_config_worksheet_name
+    validate_worksheet
+    validate_config_header_mappings
+  end
+
+private
+
+  attr_reader :record, :workbook
+
+  def validate_workbook
+    @workbook = ::RubyXL::Parser.parse_buffer(record.file.dup)
+  rescue StandardError
+    record.errors.add(:file, :ho_invalid)
+  end
+
+  def validate_worksheet
+    return if workbook.blank?
+
+    record.errors.add(:config, :ho_invalid_worksheet_name) if workbook[record.config.fetch(Reports::HomeOfficeExcel::WORKSHEET_NAME_KEY, nil)].blank?
+  end
+
+  def validate_config_worksheet_name
+    record.errors.add(:config, :ho_missing_worksheet_name) if record.config.fetch(Reports::HomeOfficeExcel::WORKSHEET_NAME_KEY, nil).blank?
+  end
+
+  def validate_config_header_mappings
+    record.errors.add(:config, :ho_missing_header_mappings) if record.config.fetch(Reports::HomeOfficeExcel::HEADER_MAPPINGS_KEY, nil).blank?
+  end
+end

app/views/layouts/admin.html.erb

@@ -38,7 +38,7 @@
       <%= header.with_navigation_item(text: "Users", href: users_path, active: request.path.include?('/system-admin/users')) if current_user.has_role?(:admin) %>
       <%= header.with_navigation_item(text: "Settings", href: edit_settings_path, active: request.path.include?('/system-admin/settings')) if current_user.has_role?(:admin) %>
       <%= header.with_navigation_item(text: "Audits", href: audits_path, active: request.path.include?('/system-admin/audits')) if current_user.has_role?(:admin) %>
-      <%= header.with_navigation_item(text: "Feature Flags", href: '/system_admin/features') if current_user.has_role?(:super_admin) %>
+      <%= header.with_navigation_item(text: "Feature Flags", href: '/system_admin/features/') if current_user.has_role?(:super_admin) %>
       <%= header.with_navigation_item(text: "Logout", href: destroy_user_session_path, active: false) %>
     <% end %>
 

app/views/system_admin/reports/index.html.erb

@@ -1,7 +1,12 @@
 <div class="home-office">
   <h2 class="govuk-heading-m">Home office report</h2>
   <p class="govuk-body">
+  <% if Flipper.enabled? :home_office_excel %>
+    Download a home office Excel file of applicants who have passed initial checks and can now be sent to the Home Office for verification.
+  <% else %>
     Download a CSV file of applicants who have passed initial checks and can now be sent to the Home Office for verification.
+
+  <% end %>
   </p>
 
   <p>

config/brakeman.ignore

@@ -138,6 +138,29 @@
       ],
       "note": "This is a false positive because the field argument in the extract_dates method is provided by the Step class."
     },
+    {
+      "warning_type": "SQL Injection",
+      "warning_code": 0,
+      "fingerprint": "a656b2b6c44e404c3fc2d9410ba8c247a14332dbbdbb015904d14fdbc46e75da",
+      "check_name": "SQL",
+      "message": "Possible SQL injection",
+      "file": "app/models/reports/home_office_excel.rb",
+      "line": 86,
+      "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
+      "code": "Arel.sql(\"CONCAT(#{cols.join(\", ' ', \")})\")",
+      "render_path": null,
+      "location": {
+        "type": "method",
+        "class": "Reports::HomeOfficeExcel",
+        "method": "dataset_fields"
+      },
+      "user_input": "cols.join(\", ' ', \")",
+      "confidence": "Medium",
+      "cwe_id": [
+        89
+      ],
+      "note": ""
+    },
     {
       "warning_type": "SQL Injection",
       "warning_code": 0,
@@ -185,6 +208,6 @@
       "note": "This is a false positive because the field argument in the method is provided by the Step class required_fields."
     }
   ],
-  "updated": "2023-10-02 14:11:36 +0100",
+  "updated": "2023-10-31 14:11:54 +0000",
   "brakeman_version": "6.0.1"
 }

config/locales/en.yml

@@ -50,3 +50,14 @@ en:
     no_account: 'You have not yet an account!'
   errors:
     access_denied: 'You do not have permission to access this page'
+  activerecord:
+    errors:
+      models:
+        report_template:
+          attributes:
+            file:
+              ho_invalid: "File parsing error"
+            config:
+              ho_missing_header_mappings: "config.header_mappings must be present"
+              ho_missing_worksheet_name: "config.worksheet_name must be present"
+              ho_invalid_worksheet_name: "config.worksheet_name not present in file"

db/migrate/20231004134346_create_report_templates.rb

@@ -0,0 +1,14 @@
+class CreateReportTemplates < ActiveRecord::Migration[7.0]
+  def change
+    create_table :report_templates do |t|
+      t.string :report_class
+      t.string :filename
+      t.binary :file
+      t.jsonb :config
+
+      t.timestamps
+    end
+
+    add_index :report_templates, :report_class, unique: true
+  end
+end