HomeOffice report uses Excel template

This template now loads a report template from the database along with
the header mappings and sheet to update.
A validator is also added to has all the required info to generate
report successfully

Gemfile

@@ -82,3 +82,5 @@ gem "sidekiq", "~> 6.5"
 gem "sidekiq-cron", "~> 1.10"
 
 gem "mail-notify", "~> 1.1"
+
+gem "rubyXL", "~> 3.4"

Gemfile.lock

@@ -394,6 +394,10 @@ GEM
       rubocop-factory_bot (~> 2.22)
     ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
+    rubyXL (3.4.25)
+      nokogiri (>= 1.10.8)
+      rubyzip (>= 1.3.0)
+    rubyzip (2.3.2)
     sanitize (6.1.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
@@ -504,6 +508,7 @@ DEPENDENCIES
   rubocop-performance
   rubocop-rails
   rubocop-rspec
+  rubyXL (~> 3.4)
   scenic
   sentry-rails (~> 5.11)
   shoulda-matchers (~> 5.0)

app/models/reports/home_office.rb

@@ -2,41 +2,66 @@
 
 module Reports
   class HomeOffice < Base
-    def csv
-      CSV.generate do |csv|
-        csv << header
-        rows.each { |row| csv << row }
-      end
+    file_ext "xlsx"
+
+    HEADER_MAPPINGS_KEY = "header_mappings"
+    WORKSHEET_NAME_KEY = "worksheet_name"
+
+    def generate
+      cell_coords.each { worksheet.add_cell(*_1) }
+      workbook.stream.string
     end
 
     def post_generation_hook
-      applications.update_all(home_office_csv_downloaded_at: Time.zone.now) # rubocop:disable Rails/SkipsModelValidations
+      base_query.update_all(home_office_csv_downloaded_at: Time.zone.now) # rubocop:disable Rails/SkipsModelValidations
     end
 
   private
 
-    def rows
-      applications.map do |application|
-        [
-          application.urn,
-          application.applicant.full_name,
-          application.applicant.date_of_birth,
-          nil,
-          application.applicant.nationality,
-          nil,
-          application.applicant.passport_number,
-          nil,
-          nil,
-          nil,
-          nil,
-          nil,
-          nil,
-        ]
+    def workbook
+      @workbook ||= ::RubyXL::Parser.parse_buffer(template.file)
+    end
+
+    def template
+      @template ||= ReportTemplate.find_by!(report_class: self.class.name)
+    end
+
+    def worksheet
+      @worksheet ||= workbook[worksheet_name]
+    end
+
+    def worksheet_name
+      template.config.fetch(WORKSHEET_NAME_KEY)
+    end
+
+    def header_mappings
+      template.config.fetch(HEADER_MAPPINGS_KEY)
+    end
+
+    def headers_with_column_index
+      @headers_with_column_index ||= worksheet[0]
+        .cells
+        .each
+        .with_index
+        .map { |v, i| [v.value, i] }
+    end
+
+    def sheet_col_number(header_mapping)
+      _, col_number = headers_with_column_index.detect { |(header, _)| header == header_mapping }
+
+      col_number
+    end
+
+    def cell_coords
+      dataset.each.with_index.flat_map do |cols, sheet_row_number|
+        header_mappings.each.with_index.map do |(header_mapping, _), col_idx|
+          [sheet_row_number + 1, sheet_col_number(header_mapping), cols[col_idx]]
+        end
       end
     end
 
-    def applications
-      @applications ||= Application
+    def base_query
+      @base_query ||= Application
         .joins(:application_progress)
         .includes(:applicant)
         .where.not(application_progresses: { initial_checks_completed_at: nil })
@@ -49,22 +74,18 @@ def applications
         )
     end
 
-    def header
-      [
-        "ID",
-        "Full Name",
-        "DOB",
-        "Gender",
-        "Nationality",
-        "Place of Birth",
-        "Passport Number",
-        "National Insurance Number",
-        "Address",
-        "Postcode",
-        "Email",
-        "Telephone",
-        "Reference",
-      ]
+    def dataset
+      base_query.pluck(*dataset_fields)
+    end
+
+    def dataset_fields
+      header_mappings.values.map do |cols|
+        if cols.size == 1
+          cols.first
+        else
+          Arel.sql("CONCAT(#{cols.join(', \' \', ')})")
+        end
+      end
     end
   end
 end

app/validators/home_office_report_config_validator.rb

@@ -0,0 +1,52 @@
+# Example config
+# config:
+#    {
+#      worksheet_name: "Data",
+#      header_mapping: {
+#        "ID (Mandatory)"               => %w[urn],
+#        "Full Name/ Organisation Name" => %w[applicants.given_name applicants.middle_name applicants.family_name],
+#        "DOB"                          => %w[applicants.date_of_birth],
+#        "Nationality"                  => %w[applicants.nationality],
+#        "Passport Number"              => %w[applicants.passport_number],
+#      }
+#    }
+#
+
+class HomeOfficeReportConfigValidator
+  def initialize(record)
+    @record = record
+  end
+
+  def validate
+    return if record.report_class != Reports::HomeOffice.name
+
+    validate_workbook
+    validate_config_worksheet_name
+    validate_worksheet
+    validate_config_header_mappings
+  end
+
+private
+
+  attr_reader :record, :workbook
+
+  def validate_workbook
+    @workbook = ::RubyXL::Parser.parse_buffer(record.file.dup)
+  rescue StandardError
+    record.errors.add(:file, :ho_invalid)
+  end
+
+  def validate_worksheet
+    return if workbook.blank?
+
+    record.errors.add(:config, :ho_invalid_worksheet_name) if workbook[record.config.fetch(Reports::HomeOffice::WORKSHEET_NAME_KEY, nil)].blank?
+  end
+
+  def validate_config_worksheet_name
+    record.errors.add(:config, :ho_missing_worksheet_name) if record.config.fetch(Reports::HomeOffice::WORKSHEET_NAME_KEY, nil).blank?
+  end
+
+  def validate_config_header_mappings
+    record.errors.add(:config, :ho_missing_header_mappings) if record.config.fetch(Reports::HomeOffice::HEADER_MAPPINGS_KEY, nil).blank?
+  end
+end

config/locales/en.yml

@@ -55,8 +55,8 @@ en:
         report_template:
           attributes:
             file:
-              invalid: "File parsing error"
+              ho_invalid: "File parsing error"
             config:
-              missing_header_mappings: "config.header_mappings must be present"
-              missing_worksheet_name: "config.worksheet_name must be present"
-              invalid_worksheet_name: "config.worksheet_name not present in file"
+              ho_missing_header_mappings: "config.header_mappings must be present"
+              ho_missing_worksheet_name: "config.worksheet_name must be present"
+              ho_invalid_worksheet_name: "config.worksheet_name not present in file"

spec/factories/report_templates.rb

@@ -27,13 +27,31 @@
       report_class { "Reports::HomeOffice" }
       config do
         {
-          "worksheet_name" => "Data",
+          "worksheet_name" => "TestData",
           "header_mappings" => {
             "Column A" => %w[urn],
             "bar" => %w[applicants.given_name applicants.family_name],
           },
         }
       end
     end
+
+    factory :mocked_home_office_report_template do
+      file { Rails.root.join("spec/fixtures/test_homeoffice_template.xlsx").read }
+      filename { "test_homeoffice_template.xlsx" }
+      report_class { "Reports::HomeOffice" }
+      config do
+        {
+          "worksheet_name" => "Data",
+          "header_mappings" => {
+            "ID (Mandatory)" => %w[urn],
+            "Full Name/ Organisation Name" => %w[applicants.given_name applicants.middle_name applicants.family_name],
+            "DOB" => %w[applicants.date_of_birth],
+            "Nationality" => %w[applicants.nationality],
+            "Passport Number" => %w[applicants.passport_number],
+          },
+        }
+      end
+    end
   end
 end

spec/features/admin_console/reports_spec.rb

@@ -54,7 +54,7 @@ def then_the_standing_data_csv_report_is_downloaded
   end
 
   def then_the_home_office_csv_report_is_downloaded
-    expect(page.response_headers["Content-Type"]).to match(/application\/vnd.openxmlformats-officedocument.spreadsheetml.sheet/)
+    expect(page.response_headers["Content-Type"]).to match(/application\/octet-stream/)
     expect(page.response_headers["Content-Disposition"]).to include "attachment"
     expect(page.response_headers["Content-Disposition"]).to match(/filename="reports-home-office.*/)
   end
@@ -78,6 +78,7 @@ def then_the_qa_report_csv_report_is_downloaded
   end
 
   def and_i_click_on_the_home_office_csv_link
+    create(:mocked_home_office_report_template)
     within ".home-office" do
       click_on "Download"
     end

spec/models/reports/home_office_spec.rb

@@ -6,12 +6,16 @@ module Reports
   describe HomeOffice do
     include ActiveSupport::Testing::TimeHelpers
 
+    before { create(:mocked_home_office_report_template) }
+
     subject(:report) { described_class.new }
 
+    let(:headers) { report.send(:worksheet)[0].cells.map(&:value) }
+
     it "returns the filename of the Report" do
       frozen_time = Time.zone.local(2023, 7, 17, 12, 30, 45)
       travel_to frozen_time do
-        expected_name = "reports-home-office-20230717-123045.csv"
+        expected_name = "reports-home-office-20230717-123045.xlsx"
 
         report = described_class.new
         actual_name = report.filename
@@ -20,11 +24,12 @@ module Reports
       end
     end
 
-    describe "#csv" do
+    describe "#dataset" do
+      let(:dataset) { report.send(:dataset) }
+
       it "returns applicants who have completed initial checks but not home office checks" do
         app = create(:application, application_progress: build(:application_progress, :home_office_pending))
-
-        expect(report.csv).to include(app.urn)
+        expect(dataset.first).to include(app.urn)
       end
 
       it "does not return rejected applicants" do
@@ -33,73 +38,58 @@ module Reports
                          rejection_completed_at: Time.zone.now)
         app = create(:application, application_progress: progress)
 
-        expect(report.csv).not_to include(app.urn)
+        expect(dataset).not_to include(app.urn)
       end
 
       it "does not return applicants who have not completed initial checks" do
         app = create(:application, application_progress: build(:application_progress, initial_checks_completed_at: nil))
-
-        expect(report.csv).not_to include(app.urn)
+        expect(dataset).not_to include(app.urn)
       end
 
       it "does not return applicants who have completed home office checks" do
         app = create(:application, application_progress: build(:application_progress, :initial_checks_completed, home_office_checks_completed_at: Time.zone.now))
 
-        expect(report.csv).not_to include(app.urn)
+        expect(dataset).not_to include(app.urn)
       end
 
       it "returns the data in CSV format" do
         application = create(:application, application_progress: build(:application_progress, :home_office_pending))
 
-        expect(report.csv).to include([
-          application.urn,
-          application.applicant.full_name,
+        expect(dataset).to contain_exactly([
           application.applicant.date_of_birth,
-          nil,
           application.applicant.nationality,
-          nil,
+          application.urn,
           application.applicant.passport_number,
-          nil,
-          nil,
-          nil,
-          nil,
-          nil,
-          nil,
-        ].join(","))
+          application.applicant.full_name,
+        ])
       end
 
       it "returns the header in CSV format" do
         expected_header = [
-          "ID",
-          "Full Name",
           "DOB",
-          "Gender",
+          "Dummy 1",
+          "Dummy 2",
+          "Full Name/ Organisation Name",
+          "ID (Mandatory)",
           "Nationality",
-          "Place of Birth",
           "Passport Number",
-          "National Insurance Number",
-          "Address",
-          "Postcode",
-          "Email",
-          "Telephone",
-          "Reference",
-        ].join(",")
-
-        expect(report.csv).to include(expected_header)
+        ]
+
+        expect(headers).to match_array(expected_header)
       end
 
       context "includes applications from the csv before invoking `post_generation_hook`" do
         let(:app) { create(:application, application_progress: build(:application_progress, :home_office_pending)) }
-        let(:csv) { report.csv }
+        let(:dataset) { report.send(:dataset) }
 
         before { app }
 
-        it { expect(csv).to include(app.urn) }
+        it { expect(dataset.first).to include(app.urn) }
 
         context "excludes applications from the csv after invoking `post_generation_hook`" do
           before { report.post_generation_hook }
 
-          it { expect(csv).not_to include(app.urn) }
+          it { expect(dataset.first).to be_nil }
         end
       end
     end