From a3f31ee5b6936fb198b165d1dd6a5766e3309c14 Mon Sep 17 00:00:00 2001
From: fumimowdan <tecknuovo@zemis.co.uk>
Date: Wed, 4 Oct 2023 14:31:48 +0100
Subject: [PATCH] HomeOffice report uses Excel template

This template now loads a report template from the database along with
the header mappings and sheet to update.
A validator is also added to has all the required info to generate
report successfully
---
 Gemfile                                       |   2 +
 Gemfile.lock                                  |   5 +
 app/models/reports/home_office.rb             | 103 +++++++++++-------
 .../home_office_report_config_validator.rb    |  52 +++++++++
 config/locales/en.yml                         |   8 +-
 spec/factories/report_templates.rb            |  20 +++-
 spec/features/admin_console/reports_spec.rb   |   3 +-
 spec/fixtures/test_homeoffice_template.xlsx   | Bin 6202 -> 7601 bytes
 spec/models/reports/home_office_spec.rb       |  62 +++++------
 ...ome_office_report_config_validator_spec.rb |  70 ++++++++++++
 10 files changed, 242 insertions(+), 83 deletions(-)
 create mode 100644 app/validators/home_office_report_config_validator.rb
 create mode 100644 spec/validators/home_office_report_config_validator_spec.rb

diff --git a/Gemfile b/Gemfile
index e9fd37cf..a9a95870 100644
--- a/Gemfile
+++ b/Gemfile
@@ -82,3 +82,5 @@ gem "sidekiq", "~> 6.5"
 gem "sidekiq-cron", "~> 1.10"
 
 gem "mail-notify", "~> 1.1"
+
+gem "rubyXL", "~> 3.4"
diff --git a/Gemfile.lock b/Gemfile.lock
index 7171c789..1fecf659 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -394,6 +394,10 @@ GEM
       rubocop-factory_bot (~> 2.22)
     ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
+    rubyXL (3.4.25)
+      nokogiri (>= 1.10.8)
+      rubyzip (>= 1.3.0)
+    rubyzip (2.3.2)
     sanitize (6.1.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
@@ -504,6 +508,7 @@ DEPENDENCIES
   rubocop-performance
   rubocop-rails
   rubocop-rspec
+  rubyXL (~> 3.4)
   scenic
   sentry-rails (~> 5.11)
   shoulda-matchers (~> 5.0)
diff --git a/app/models/reports/home_office.rb b/app/models/reports/home_office.rb
index 291eb5ea..b6ab2e2b 100644
--- a/app/models/reports/home_office.rb
+++ b/app/models/reports/home_office.rb
@@ -2,41 +2,66 @@
 
 module Reports
   class HomeOffice < Base
-    def csv
-      CSV.generate do |csv|
-        csv << header
-        rows.each { |row| csv << row }
-      end
+    file_ext "xlsx"
+
+    HEADER_MAPPINGS_KEY = "header_mappings"
+    WORKSHEET_NAME_KEY = "worksheet_name"
+
+    def generate
+      cell_coords.each { worksheet.add_cell(*_1) }
+      workbook.stream.string
     end
 
     def post_generation_hook
-      applications.update_all(home_office_csv_downloaded_at: Time.zone.now) # rubocop:disable Rails/SkipsModelValidations
+      base_query.update_all(home_office_csv_downloaded_at: Time.zone.now) # rubocop:disable Rails/SkipsModelValidations
     end
 
   private
 
-    def rows
-      applications.map do |application|
-        [
-          application.urn,
-          application.applicant.full_name,
-          application.applicant.date_of_birth,
-          nil,
-          application.applicant.nationality,
-          nil,
-          application.applicant.passport_number,
-          nil,
-          nil,
-          nil,
-          nil,
-          nil,
-          nil,
-        ]
+    def workbook
+      @workbook ||= ::RubyXL::Parser.parse_buffer(template.file)
+    end
+
+    def template
+      @template ||= ReportTemplate.find_by!(report_class: self.class.name)
+    end
+
+    def worksheet
+      @worksheet ||= workbook[worksheet_name]
+    end
+
+    def worksheet_name
+      template.config.fetch(WORKSHEET_NAME_KEY)
+    end
+
+    def header_mappings
+      template.config.fetch(HEADER_MAPPINGS_KEY)
+    end
+
+    def headers_with_column_index
+      @headers_with_column_index ||= worksheet[0]
+        .cells
+        .each
+        .with_index
+        .map { |v, i| [v.value, i] }
+    end
+
+    def sheet_col_number(header_mapping)
+      _, col_number = headers_with_column_index.detect { |(header, _)| header == header_mapping }
+
+      col_number
+    end
+
+    def cell_coords
+      dataset.each.with_index.flat_map do |cols, sheet_row_number|
+        header_mappings.each.with_index.map do |(header_mapping, _), col_idx|
+          [sheet_row_number + 1, sheet_col_number(header_mapping), cols[col_idx]]
+        end
       end
     end
 
-    def applications
-      @applications ||= Application
+    def base_query
+      @base_query ||= Application
         .joins(:application_progress)
         .includes(:applicant)
         .where.not(application_progresses: { initial_checks_completed_at: nil })
@@ -49,22 +74,18 @@ def applications
         )
     end
 
-    def header
-      [
-        "ID",
-        "Full Name",
-        "DOB",
-        "Gender",
-        "Nationality",
-        "Place of Birth",
-        "Passport Number",
-        "National Insurance Number",
-        "Address",
-        "Postcode",
-        "Email",
-        "Telephone",
-        "Reference",
-      ]
+    def dataset
+      base_query.pluck(*dataset_fields)
+    end
+
+    def dataset_fields
+      header_mappings.values.map do |cols|
+        if cols.size == 1
+          cols.first
+        else
+          Arel.sql("CONCAT(#{cols.join(', \' \', ')})")
+        end
+      end
     end
   end
 end
diff --git a/app/validators/home_office_report_config_validator.rb b/app/validators/home_office_report_config_validator.rb
new file mode 100644
index 00000000..07646c51
--- /dev/null
+++ b/app/validators/home_office_report_config_validator.rb
@@ -0,0 +1,52 @@
+# Example config
+# config:
+#    {
+#      worksheet_name: "Data",
+#      header_mapping: {
+#        "ID (Mandatory)"               => %w[urn],
+#        "Full Name/ Organisation Name" => %w[applicants.given_name applicants.middle_name applicants.family_name],
+#        "DOB"                          => %w[applicants.date_of_birth],
+#        "Nationality"                  => %w[applicants.nationality],
+#        "Passport Number"              => %w[applicants.passport_number],
+#      }
+#    }
+#
+
+class HomeOfficeReportConfigValidator
+  def initialize(record)
+    @record = record
+  end
+
+  def validate
+    return if record.report_class != Reports::HomeOffice.name
+
+    validate_workbook
+    validate_config_worksheet_name
+    validate_worksheet
+    validate_config_header_mappings
+  end
+
+private
+
+  attr_reader :record, :workbook
+
+  def validate_workbook
+    @workbook = ::RubyXL::Parser.parse_buffer(record.file.dup)
+  rescue StandardError
+    record.errors.add(:file, :ho_invalid)
+  end
+
+  def validate_worksheet
+    return if workbook.blank?
+
+    record.errors.add(:config, :ho_invalid_worksheet_name) if workbook[record.config.fetch(Reports::HomeOffice::WORKSHEET_NAME_KEY, nil)].blank?
+  end
+
+  def validate_config_worksheet_name
+    record.errors.add(:config, :ho_missing_worksheet_name) if record.config.fetch(Reports::HomeOffice::WORKSHEET_NAME_KEY, nil).blank?
+  end
+
+  def validate_config_header_mappings
+    record.errors.add(:config, :ho_missing_header_mappings) if record.config.fetch(Reports::HomeOffice::HEADER_MAPPINGS_KEY, nil).blank?
+  end
+end
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 9911cdf7..6560f938 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -55,8 +55,8 @@ en:
         report_template:
           attributes:
             file:
-              invalid: "File parsing error"
+              ho_invalid: "File parsing error"
             config:
-              missing_header_mappings: "config.header_mappings must be present"
-              missing_worksheet_name: "config.worksheet_name must be present"
-              invalid_worksheet_name: "config.worksheet_name not present in file"
+              ho_missing_header_mappings: "config.header_mappings must be present"
+              ho_missing_worksheet_name: "config.worksheet_name must be present"
+              ho_invalid_worksheet_name: "config.worksheet_name not present in file"
diff --git a/spec/factories/report_templates.rb b/spec/factories/report_templates.rb
index 897eb4a8..470014cc 100644
--- a/spec/factories/report_templates.rb
+++ b/spec/factories/report_templates.rb
@@ -27,7 +27,7 @@
       report_class { "Reports::HomeOffice" }
       config do
         {
-          "worksheet_name" => "Data",
+          "worksheet_name" => "TestData",
           "header_mappings" => {
             "Column A" => %w[urn],
             "bar" => %w[applicants.given_name applicants.family_name],
@@ -35,5 +35,23 @@
         }
       end
     end
+
+    factory :mocked_home_office_report_template do
+      file { Rails.root.join("spec/fixtures/test_homeoffice_template.xlsx").read }
+      filename { "test_homeoffice_template.xlsx" }
+      report_class { "Reports::HomeOffice" }
+      config do
+        {
+          "worksheet_name" => "Data",
+          "header_mappings" => {
+            "ID (Mandatory)" => %w[urn],
+            "Full Name/ Organisation Name" => %w[applicants.given_name applicants.middle_name applicants.family_name],
+            "DOB" => %w[applicants.date_of_birth],
+            "Nationality" => %w[applicants.nationality],
+            "Passport Number" => %w[applicants.passport_number],
+          },
+        }
+      end
+    end
   end
 end
diff --git a/spec/features/admin_console/reports_spec.rb b/spec/features/admin_console/reports_spec.rb
index 55a2dea5..8156272e 100644
--- a/spec/features/admin_console/reports_spec.rb
+++ b/spec/features/admin_console/reports_spec.rb
@@ -54,7 +54,7 @@ def then_the_standing_data_csv_report_is_downloaded
   end
 
   def then_the_home_office_csv_report_is_downloaded
-    expect(page.response_headers["Content-Type"]).to match(/application\/vnd.openxmlformats-officedocument.spreadsheetml.sheet/)
+    expect(page.response_headers["Content-Type"]).to match(/application\/octet-stream/)
     expect(page.response_headers["Content-Disposition"]).to include "attachment"
     expect(page.response_headers["Content-Disposition"]).to match(/filename="reports-home-office.*/)
   end
@@ -78,6 +78,7 @@ def then_the_qa_report_csv_report_is_downloaded
   end
 
   def and_i_click_on_the_home_office_csv_link
+    create(:mocked_home_office_report_template)
     within ".home-office" do
       click_on "Download"
     end
diff --git a/spec/fixtures/test_homeoffice_template.xlsx b/spec/fixtures/test_homeoffice_template.xlsx
index c99d75b4b8511c05ccb283a8cb065c27a9c5a30a..6d8b2fd5fc29333440e708e5934b25e62112f4fd 100644
GIT binary patch
delta 5721
zcmZu#byQT*x1M2yLApV@yG3e{lpK*kkrD(16cCVR1|5c!l8_;k5D)~B8XD=8uAxDs
zk?zh1uJztef9u`*&s}GI`>cKUx4*s5K9!<{5F!{9bb|_jkB<+CH!URM0^-D*qHhCb
zQKQ;DF%m_Fk1`Y^%4yCIBX}Y11MEvm>@uB+h}*BcTyrzZ!IpvgVuV$|T}5kIT06?(
zf>rAm&2nKT3nq5BVh~?zsbbFof5{2G)jaj%jvu-Aa(QZB6;tp#R#oDueo{^h<+$7Z
zqr4ROX@UYEu&k~mx+fq^^^%->5hd!TiwY*9k)3#)ioq3Qm3eqrMpbHEY|F+eR~jJq
zbR-3?>cC_2yep-q<h%vW*IQQQrP(hgP*I|I4~jfW^7bbRA&-wn$f)DDDqT|SfCJP*
zGsLaziF9yC4$A8gvds{OX3xB+&d1M7V#HL6AqMGcx>lDvxd}>n2s@D<bH%XlDEa1h
z`fin#EI;>+<_)npK{dLU&#FL$d}7bjofySP5htTW{LlS;uml!@mFy&el<2##nh&bP
z(96*G6JrZ|F$_x}O+N|jp4N}kBrQl9>+wNjsNTcQ=j?438?InPaxGpMVZ<4~f)_{K
zq3q>j$IbAcwdt-iip4!P$2biz)SJ(qp+`a8ibPZ_?hnXZpSc{=@y?w+^||+8cLh&r
zw7!v?MX+*L&@y}ZB0lDe264%(yX^B{J;5K&;R^5Gw%gFTe@2Y12F$Fd?xu~fxW$!s
zaybYYan3F1U`tez`{GO1pGZiYsq^adV228a4u6);Khit(Og^TS>G1}_p!8sTHv#!1
zN*n-S6$AkMoh8MdpkAEoG||A5MhzEDa@#&MQe7FGz_Dd)wk6yPr%|UN4{eH$nVets
z)G$R0(lpQq3-!QW^dx=fJU_~c?q_~S{gr0hUOb!wCgYOk5q5OCzS4avT@=CB!^v;h
zd}uxwIcByD>`oQ}n%o0)2+v|=ka3)nB5BV&{OG9=1QOJ)ZW2|^cA$4bsCfR5U0DNj
z8PAkkh#Nm5ZsO4dw$0$|R^lwslYydTKEHydIRqU9)!|1nc%<q<<T*lV!?RiwYhJ%K
z2>F@X(x~l{Yi!qBET{c|Wx;4cgiT+})nHZXMlRp?yr{-WaD;Ol-quS>YGql`si;S$
z1l&SVI>k)(^~jNiO~>s7B2+S@9h-rafqrMbi5%kqwXn;lAUwA&$eb|gzuV&AhUDLk
z#V0Iq=cd%QplLq2J%f4Mc%;x6GYwq3oWeQUD3j!dU{lRK;WVea8I}{o8f{El>K4PL
zXew)0GY8!Rd`1F&=x-UAoJ2BGCx*6g=b2XHy>&}LrH&}8nA0jV6kq*zm9&>)uA~f_
zvwsaPD{B@wl44x*A%2Z{HI)#NHNT&ZQLWzPj)d%U5!k|=@9bI*6a{mr4c{%XUz{sO
z#V(Car7@=T$g9q83M@9X_L?H~(_VZX&7;p~)@d-+G%RlKZ{J(D_a20*#vbE5_1x*<
zkr*S@d*6dy2@fAe#bdylhe+8-d#Y4TF$7iJ7uN>H&WHX%Z2|T><FdI4P-HNhH=i_k
z2$C>ysl<y(!_mrI@8d8^ICZM6ki7RLUdZThOhWshSt3hecF@^T!;T~7$tp(Xb97~F
zCUR+GvTsaBNaCq!POMEch`)6{dYJr!?iBWcJ3DGaBjFb6{72UVnE9;}l&lN_<uXXp
z;*^np6LM_hjYVmc3du$<Psv%+RRU8L-O^pLn?c5pEbT-YwiPE4T8*VD(+sj};rU_X
zmQzyB^K8T1%`<RVIoH=a*3ejSzi8qrIQdJ79#JKsx>&(Fh{4$lK9jmo_VMcU_Fh{+
zoZ;~o$8*#g|CV#OyZ3DViXEZ0!VrzVPPY!!Ijx1=dzEt8PSnt45JwpuMH6nEld9XS
zstZewAv^+g_f#*Q?RqZ_399xaGR4<Ci@m@QO%-^LEXn`G+LiWGJM=K*e(*wqAjZj=
z&Tn5ABaUQr(7+!Yl2#WipGP<edPjg|Em~;jPaJM*m+mhvuk;DV&p5_;6>+823-G^s
zz0xTcN?aK3U|(YZb)AqVs@*K;Tk0Darl3}MWR%=IX79<JciXu#d5l5mAq`K=*K6sH
zh-lmuRPTB4oOGvw2DsD&RuuQy+AVkodCgBvd5l%%m~yvaCT!f9#GYTkPn|bpwdRt{
zg<UG$QnFnX)awGlY$<wL*&YB^1E|zcWd&+ajyg-C$&QmC!JnI4yoo--<#)F7!~1L6
zw+?ZpQoL?G|Nex{q~p*$nA1Y6hTQpMzli~Bv`emUt(H+5eTmt;Dg53nC}4bR8CIPw
zA;e67xFt>lO}qD5y|B&B?wptQ*DPm`pf_nDQD;`s5q>@HjdJUx<rkg(;mU;GBFO<L
zXYB<8rBvaYi=lq`^^nT6lh+peC%QlVQ*~eKE)7YkvspwbKk`t=S}i|K>!hY<pnSP4
zl@1>I?xDv|Qm%-ZH@pMaKgzP2k656}FpewTQexEiL;2pXPak@Grov_&u2^JBK(%S}
z`H4n&IYWnSE}x8ZvG5(RNaAmM0eF}q%Ek|xPYmATvErBlEqqB#3~;HZ3fwOIoRp|&
z$|G5Z^7UUHb$=SrCMmWU#VxT9`TgqPVB{gX4{|1`l}Ounl6t40Nk5o}L%gDv4^7;}
zSy8_@lx<W&g17u>*xqkr>?gtApw5%%)y|jTJ%=G8_+e#sZywSyb=qB$l~REY)z4?1
z)^D|FmTHd8B;el-OsMvkq-&Ts$Dy|6bgxy7U4>L;F|KJ#^N2ga0+*7H3YLGGho{3r
z{W%;YoU=>X_|u15`cWPDX^5P!xtIP?)*#&Z3Rvq4P6Erh9x4N)0;gh}WqIYaa&V@z
zgY#rL{sSR>WLVg6qkqNtw4X1^pHNV&PR0Jsr_q-BW?`>p?%yHdU~b6`^2AR(lD9sL
zW~keHd>VB#e>T8ec#L>f>>JYU)lBm{r27iANv?8%cpm2aQRt+r5AnQ><pedp=}P!(
zS`tpGgmir629R(gR6J+)`DKG+vn1?u;sB`W2lw@b1NI?6uStUHo+H#ljkr~Ji$emQ
zKp7ej-?^Vvke=!SF3~qqDmMI_ugSmvc!S$X5StuABPrZHTick!eN<q$K_|WFA%37<
zd~E+Pva^+=3+-W<c%%3LgqR>84w22$T$lE~p%KuRTs-1PY7OLgskHF626kHTWo3Kx
zU8P5k&0^=t=!2MiOV(i&_JnOVN+DCiOr-=S(9UXDl`S->tRW=kZvpRgK68sjv+xb9
zf1a&>Iy`ps8C|#m`)z>~@kdDS=%0yx4b7Hc?-<?HdZsIC7bFv$A9^{gwoVqu8Fg$j
zC7vCTZ@irlrEt*tTZ90YZAhLHS#Pv$Yh9;sCK!0Jho2MUQri#*-D4F`y1Y>I#BjH3
z*{;LPQTfW7_?Bb>;+LfTg6t7BwdI&Q`b+L}0hU$Y)Gu+a>q!cZQVjbI0APdk|EVXp
zuj@&&zV@pLX|kHl0y4KH+m}X;>u>b*-;cUiDjI{GD{&9h!eUedW-*-BQpfv{40$GS
zn7OM)^T+-~saMkJoGGYy_R-r8(rpO3$R8H<R(LU;lX7R<7%vnv-VX}R5J3Cl4CU>a
z-+PhsfRlczcVhrx#>b$*(iQR0`zU=(^apZY?tZkta7at~FI6>LwXdjbs(vfEPpVuq
z;<@C)`%gq!dl5h4q8@Xh%kcW|lZEYRH^kcZdk;v>=xUP~+I7BAbT)b|TYWTl^J)8S
zVmuc}N2>kTWPJp=DdUjQ^H*&86z7K7hxoae3L)B`@`wnF1o#$&Y2L&zvknDpshz(^
z_f}rj!@g;abm9B}xY5^FnJy8M5I+#$64O_Dw@)x7$Y6UxdyV}~{ZtZD*R+f%E~9>1
z0Y!6((=T|sOwB@c+R(~3v0qIVz*zwTU#XW%M?q=N@|1Q>eTiNfIYKm`WenVg&u=l+
zmgnczb0%iF5Pk-w3i8BaB5+Wdpvm#WHaN2uPnMzvOt{7G=1d2Xe^<*z;8Ag;#TF6F
zQJfu?0heS>syiJqGfqw_pTEg3Lg>}H<lcnAg}ZSFK|Hi~9!n#ro&5(e_#nUGyE$Vf
ztxc}ycIMv4Z%xde9um8@HT^D=5<ppFM#8Chn{*yH_%_e>?=?w$>OpZ2Jm9tOG+4Z+
zerOedK7Y4vBcu&V)jx?_|5y+2wXta;a`){g_kG(NZ5Yi>PXd%|(lotAXw5YEWoef-
z=^-~no#~t<K#svag|(xC5c}(->x*mN9eErxNt?;6Z6k{H_jbbpCUq1p-i$@k#2=Z|
z1<ioprAdxm=y%w=S0qr+#N|o8Be--}DAL*-DD8$~7qR)VW`Rn+#X+k%DMO|TSt3Fe
zBIE6Rin}kO;(~EE3^HH8pHy$$SsNrDg8pXcheU(~rYvqRt)1DAzk{Qi*|u0dF3;zl
zHoU~(h9ObikB+FZpm~6YQ$%H;0bkPXmc6$Ma~RW9m+t#j$9t$SOmf)bD6uk~ba3eb
zf|qe#rD2{vrzIy#hGtbkco*hdmf(~$oyR&h7n`9#=vXyLKb>WTX%$cUjT^HeVEvOn
zT76cSIwgpOu~dKPSG4bJKGgfx$bP|_A>aZcq61nrJHD4KyPJR&IrUD$-r&vibUKNM
zSs7@&^ghhNIeoMYEU&v+JNLXUD7sBybBx5GDnD6X_|uVsf#zV7baO)2rHb3c?X6x<
z=CTH@&lbt>wTmi^OWbQ+dq@M`=3Jc<((wOF*DJ(AD0IVvCQ+ido3BILJQK<o@Y6D@
zSTjsp;Xb_XiFOVYW5BX<GctTx+i%kMAx33$mzN}{Z~GRE&0vbdUwphpj5lQG5`Vc9
z{PGa-jKt*A4$**_1}Mvr=I<RD1u$Z7dHBW*8SlK1PlYB94}7(>%SJ0D)@;Dy5Erbr
z-+=ocQJJF>1U3UytRAgCm;DD06AW(-7LoU8MGQ3}T7fcrb}OR%M;pZ%pq(NlLi<!l
z=XF`|cI32Y%yg}m7`V7v?7j@RxKgakg=F#K8((1p1`!^^u0Y)Bb_Oc`IzOiVRp&Ya
zE_{Z9GXMZU;lCyuRRc5$DHkecLJK5D_I48)%4XnYF+fB9ep2YJ;c{S}2GYmPvy7q5
zqMNg&wq}4G&STW3N-~vljPA<#fP4btA<C!`Xt9!@@yS03_h^0iW@F?~2Zxt`gm;c9
zZuTAL^C4{|D&H>=TKz4f3Q$e=ApQQf!zK8a{f2LCO-?a`U_D~@;C>Xn3{xb_m&fBb
zzam`<sRqEn_T_?r&Z&@)uLRh~&cr+SYre@yZYt9~;QfZox>>;&+@@yifQc&PpI6<D
zbX?XhB3V?KR;76ZV+)(eEyB2q<CS$RGQBMAynP&+-yoOrgj$9prhhxBrH<hfV5(6$
zXmml7N_SHwz@aF}x6Isbju&@9dJ(W+ZCjy(*pO4r_pb)SXWS%>=9!5PnBpJOocjvl
z<+sXt75hz;L$lRbdOVwU?l%fE#tyjEOarsybMH1;0U+m=T$k)aq}KsoSL}_|1_A*7
zSMB8QQ4&o?#)ay<$~dy>UF!FnA-gs}BTas4Jz+iN&-l&V(i4MZkhpcp^{F{dZR1q1
zBLp%x7C$idRPPr_$xLY%j4GfF)V|`^DXt=t6Nn0vv&<H>c?@mq0}>@H81Ov`7^45=
zmZ$iH<cF&omMl}eq)?yy9r1)uDz_8Ca+`Y|ZYjwM1xlc!*5;-(Nz+TgdaR#u=o1yQ
z@Q5WFBTWB82lt^{R75JcI2m`O3E1CYjr$r+-t>vR&}vv$YEWHPj996eNypeGq-x4$
zuPQXY>TLb(sF%{|@t6qqiJ5`zKByE1@A27Nz+1fYy~<1o#vJ54tDeU4z(7}{F_?4U
zkm}NFiXL^~oVOcy7f4HM{$a7BlHp)JGN6e%c41gZ4!o?;1!@0ar(>jR8NadglBDHq
zT26TWZA7!!Vek8A;vb{hyT>&Al?;rUBFa%BkyW2L?5iXvB6ZBmSZr?ym;Kc5%L>}+
zMRnC(P4r;=U_(+?>nj!}u7>*muy8%e7m|yjwE1#bWAGdjJe0>tn8x!uC}_nOpqil_
zb!Gr!={fZIxR-sM)m^(-rh2fgwtuE-h`8ua0$UH_6T7Kht`21@HBoC~i6wks-ZJI!
z-mq^Zg>oD|w}cH#`V$MrBW4LgRlCH&<-8HWOzb9R{dN0H_l}Z(NCkf<eG^>+KG}Cv
z6;6L(g>18WTI0?D(qe+ybXFJZ=Mk!!dSePZtd?n<?c0NXBh(h=s^vn1=5XUc$-PN3
zm{t`Npi#Xurg8h-<YCdq3c^FG?+dNBuezK7Pl3N4djWoJP;%Qu*pS<aNeo{P=#suc
z%3zu0IvS#`#3#pBMIhlSB3IG4`G2=y=p~Bb>!8$Aa-q5(KhbGorShICuz}wRbm0E+
z!Q=JT&R1wsFWyW^ODxo<dwq&9SvYKXrAQ6QzL2!3Mk0}zZ=IZ5FPru_LJb~F0lV*-
z*1>qU>lM~jBAKY@_=gWvxxhu{TeTEzfV94E2FXhhzaD`D*Z#~vTr;dJl`Cr739j}*
zpE*VcIge6AuMrSb<|18cNG6_40@T|V%0#plg6SU-sv&vNhFp2lgr%#1e5%W7VWjI@
zP|>n;tmwx4OWgDhQXJ(UR?c-y+xDqUSSp-FVa9T5#^_t67jYeZip-W2ey@85GE4H?
z!^*;V@@OkgN1qJ3>Eh#=6n`NiTYHDwVTwC#9ga$*hVl<eo%Fo2b3*1wWhiL;0x;Uf
z>GpDaoPkg1E-6{EQ1h0Mi&NKQs6oTyL7cin$!gCUoq2iB_(-1=BL@>WpAe(t>|M`$
zS=?m4`=3ww*Qh}I{n>1_NXDbIar@=!IQ7m~w(?=5;@GoN?t*J#WCv+(J6@48j|U*V
zVg`sq1^C|!O0+$-6mh(%s=@!=O`$Q=qAb@d%J+pr<E|1rhXMc~zM}O%k0<()nvwMy
z^Y?^I_aDI3mj)i%mxdY*qapvZ+22Rse}Rl>9~z!NoBiEJ|HTB5TyG}!=ZO5T|EdOm
zw;uX`TyUpApAa(&{<p#3rIYF(gHM!bQd;&uzWl9h@_#@=)M#B=mOr4s6Q1lJP(KYi
yikA60=P|U5IQVqv9$L0PCVx=@p!~;VnhxDZNlgQT;^JMe6J5Vo-8AygKm89xgg4p%

delta 4381
zcmZ8k1yoes79P5Lkd%=aLb{O|U=#<WBt#Gf5a|vDWI!0CLvl!EXh8{wkQhoN6cLba
zK{}<R^MP;u@9BT;I`^(~?mqk6v-kJyyLTC+R*X_dgMbhSASERQwB0GA1moei-9by@
z$)n`7x)Y_!AKc4>#Z&`N_ha}4J$rMOZgSoD`bt*P)&EIh-cx{Oh=Dl8G2SnjwLFsp
zTLo5obut)v1=oQaAh+-${_1jNV?ou~>+KLEb~`EBDB{7lXHzy{{fRwKsz_*v45O;g
z%Mw>d0%$cDV9*yEXuf7>!au~-*?Dfhl!KyWDfXzs!X8mrz#jALRRJrlqmZ0D&}u4V
z{X3<tNIO0go6d*zy=NBniMDlq4f#u{j3&leDjI$#>Ha~KZ@ERlzaGcX-VJ)hw9OW*
z5}7Aq<wW@nKV~oG15&;f;@nzf&_2&AY21;TJ^HYaP8T>j1$||sqYbUZZ0&IAzeB}}
z=Q3ij>#9%v=x!vxZ`v}KSaW)MtqT0fiT*D40;e|U)y+01ppG%;3%Frxw%f`aKp%_L
zRX)3~>SFtSSwOajB!SRAZ(9A+ak@Gr``RIObEH1KwIsuw*gkRqW>$ce+b)_|0^tZ$
z3ua*z9d=0kk2=qC_9G9%w3LY>V^Q*a;b3vx!IX?I5)0!hpgz7yD^u#$OcJ$T->&0O
zd(H_L&?6{0NT*J#mgTaUnu^VJmtawCuw$Y2dE^aQ@nfb`ZP@+kqjgW);ETs;kkqn{
z?G9MjFDf*2pR{d3vsSM1$V6UU5nhS<+_^FL{(wwJgM@S&^Uj3?9{?yKpauZ`L`Fpr
z!8iO%Xv9&2(MVzjR0+7wQ>0{#%WdT+zCC-ZJ;fTEaEuTaFbphE_hon^u2x1MGK<(E
zN^&Mm(pKb);>jB;4}c|6AsmXyb0@s6Dl*_thr%H~C`NIwrp{>Q1j0`MGH0bHf{+|s
zH8>!O6gCPskuR76M|5etNyTIl!!z);+SsplDVB`^W~WhlV6@T83?4>?>n^mDX({IB
zy5a01;#fhQcX;C0(zuVRfMZ^EN(WJ!HIG!)D`zEhyhpu#>X*lDy_4FhjD;q6J{P{;
zgMR%rcTxZ?Isvy{w^?^QeHkDPu4d4y32PEJv%x$0bUP%Dy3=+}KP6C&Vy@qb2I2P5
zW3vuY&#8=Byqj#)XJ%$&5*OUK7XRFW?xAJ_#QOue+*t1uD-Gzo2ovG7pQV9YAnt=C
z9hiab%8+-|XT}cgJp9VRku6Z13Fc$CpW4p5EH{Nb%R1pZ?N>Lq8nk}$5REG8LlPAL
zVS$5vNOB;xmaP))EWEfrIpLb_F|A;n#2oYd*>@C*?0n9dU4hGBJY7xC&L<(JdwBZj
zUeWM&q=NZXD^6E+GbJ)etFP;J&0DNad$VIvTIS1SIJ<Z`N0zlDTjr~x_|r*!OBvu-
zg4oQuzGCG%9Sj0|C&V?dF2P%i?-$a?M+Vd7i+OWBv&r%9`e<<eQG8*qk@0C6-4QwU
zwdGS3%+4v7RLlR1HleX%N@tn1%_g(1aD88;=L2ccUCz*h*FV_xH|!?QsIr?v3;EKj
z8kK5ezWK>LG-Au?v(^}Dk?2Yw5bs)!mGewzOBf@amyvghev^0iwHip?SIJ}d9!0dl
zuq~K%E4)J-mvf)c=EvOrTZY3Ys(qb{wl;f#DEzNI$IEj&YaWx-t!{bAWXIl$ke^C<
zW7Jo_><DVJ5W4{QX}>R}@6=@v4J5I48bI|NGth272@hPTA-RsZdn5Z=?8xwFGIu$S
zK~pjZ<x#!YFS4v!ILumEqa`(UK5X!O-4F$y8;}oUv%{;_?N*+woG}{sXCstq$e)>5
zqA;N9)!MO}{o0iwyZ3(e>zO!|g-;#au4Sc6X0HtB;mL#N4$0dA-F1QbeDnBcunI|6
zxpQx3AI~(-#V&tDqeYVc2l6Rw<M7htRIhPmue5U)qB!;F_{!dU8Le5$4K3U0MJ^3u
z=Gc+AL&@7%HDHD-zEe~Zcow=maz1XxaYl2=*DDCqmIwj>;E3dJx?a*0B_ZPN?1Z^L
z?lLY%>o<kHw$oEkVvX~3NxIL|?Sr^h5+wH>|5zm|5gt5K%402icyv|Ia5xHH>R;N%
zveKC*$y5MB86>RX%J2uc)Oq{E-I53x9IC?`72Zvn-ENgHWKPOD;PP3JIZ8cE=4gVB
zCOlE4y<n}rd;xZ5WC4uAzQF`6kSh?SM#@89nmScZCx)b4XZe=jUs02D3r^LXgrp({
zhO{3F&v~{bIl?GY1~~l?<j#=ynQQmGJ9`N7$F^8>>z$zrX4_|kSb7{DMaZi;oyvi`
z74<JfW4DX9hiQx6APaX9ghQzl26v`22tM;`5F9jchQ!6Z5^X}2iV!6w=W6#~opRgP
zBKhDO>?`H5E~hcOH!oz358kon3uTI0lE)eIo~hJqru-m*`0!Q*w~!wWCX7*wFrP~W
zekW_|UD5o`3i?q>ct{q~)XfSxTX^%pR?NbXs|8|K!_n9KGgKo3(jC)W?8qnHv49&3
zT%%(awG+lH25h04JoXgy%x-tA3eYjUk1d<iJA!g29M@%vH;o>L*8}ulrK{@;+;|n!
z{jg>AgK<N~(;b$TEDE23rQQBgBJXY9Sjg^ua(iz->0NYxc{nM2HAB|PC1DGPOd0`Y
zbIE?u5u<_nuRBzb3kOxYB*7RPDHhb{k~9~7Eg!-qUhSjg9~mjSi^|v9GNi9D38~uJ
zw}q-KK^hn7H%M+-vZ>f@N5zu9f9CQw*T~@qtn`Z6@|gUaml}TIoc;pLvxQHks?URT
z%!3Be`s3L*%cb_GpIJMHM~WieuUGIy@3%|2x=y4|re2+TkTK~!JQXgE71`j`bq;>)
zc(sZ~pzOOl%93)nj7??MkRrxzxCqR=h2NozYfbvp^=8Xm6cV2)KV#GVL8pR&`9Ub&
z%e&p8-HSi<sZ^cnN{1IKew_t%xPHtYIA0KG8K4Fb1+Zae{XY+|LJ&_X=tM^*Q}|6F
zaDT4m^=0ESg}nwQI+`*WPU@f_A0eEvWz_>`Zn4o))HkMh;#o>YIZ1@tuL8%m#+2sW
z)fYeZI7(vJ`RvG&&Miy*4+i6Fs*Qzmo@N*qteAhfNx{((Gb(+Ey)_gZkpr7gLXcU8
z=`<j_MnK2eK@~t|$04)5P=liuiDw@Ub%TyK!zqRnur->Xtelm7lExNi2Ugpfl+NaF
zo~$%t(i~-+zs_k-8@Z+r6`QIa*h9M+VoyOnd%u~Bz9)l-zF;np^k2+<PA`l~_~y_e
zMj5u@N1l5;!^39zLarh4CKj;|_@q12(<+m&SKz7?8!a57=fw|U`Lefv409OT_?2fC
z@BQK<?0#aFSj|ZEBeBhT-<sMW&pHPV_wt0yQV>JY312CmXu^7>&7ymiNoz3Y)K0vO
zATIAF-|C(h%rs@bBp*yL#+{EE^CGqzzn|q57L0oIxe;{LCC4{Sygh=n>v3wFWN%-9
z#g8Hp36h=VgQJ8k*Z93Py7}QB1xfRYu`B7FyGmaN7Oh-F<e6Q`>iDwjt-l@$jZaAw
zsepc(9phaFEw}m7<uL#NP=1kU{F!6G4KF4y8@hsl0~Ob+PAE<ZpE`|<ds<z=#icP0
zXF3VoC799lD(K0#L>ujFlms5|k|S|*850^OXC+oOZvFb;A6}BzN?&-YKc+r-_N=y1
zRfju`-*7r$qqrv4&}=X(HwVwQ!g?`1Mv|#d%k;s~f<A(CwH$*HiMH=c8&&s`ZE;&9
z2j65~HM@Rg^U9;7o!aebgeq3x2pf(Ok=mQ{SH_t#&${0w_alAb8-1^14Flg$OnX?P
z+~r8;eGq%NrM<x9prkmAVo<s`8wu%+(Vg+;iz6A~{e4#S4Ms4^R69rv=(lee*QmVg
zrov|aMe0`@n77;RS<!sVeS7;cml~Ao^N;!KAFF-WZz${+Ew5=9e58KCSlK}vVF3)7
z@#NG`%U&Ghs#kgLsmWMe$Y^CpbERuLD$%jSksvmI$UGgu=FM#_Fw8P0S0L~y*%c<i
zh78%BtCdN(L8BFclAg^hCWJ=J3JQHJ3%n7EXwiv{Su!`;?A3GC8e{>oD-+QtR;1o0
z1XWVMcy9~x=#0=KuO4<YkdMnf`WmS_BBeFJgBjS@-dx?De#<5eEHq^#Nh}Cc;g@Y<
z2R@h0^20qxs<~_!P$X*4sL?0Ly(kc*?@#TqI!c~-xPfRvDZbO1ELZ(8i!UU53xaCN
zIEmk^u_&B43R%o*K1TAf-1{jYX!E@f<R_e!y3z1CZK5Y0I(Mej2L{XXaZ?eSS`$Ee
z&l=J_;Ho2D&ea>BCR3$8(r0VIMOoT3)0yX~D1u8DY#}^9SzMTZ{NHdxPhJrvP!nK7
z(=xH5zVRJb?vD5``$*Nw!NNO6iM8BA$V;2mte;XHIZbtduUpI*JpL&pwM0r+GD>ht
z+O2$t+#;S#M#e5P;UOJ;8wX{qT($H1jtFzcV-~O0-3I#2l~(g6UOmIX^rqA%^|;p`
z@OwjQ<nzvNy=Evlo~cqe>a=`MG%Ky$#E*Mjg3>Ul?f?{3JQH53*f%E2DCc&{Z6v7A
zG>jw4m?nD1$7UT?c5K{TcWCJ@+cVGP595(2;uQ_*EAO1}9f_!Kc+i!vUDM&tanR3?
z!*%%o=9#caJk8++_Fn!Ad;jCvCD#B97v0Jg&ClF?nclB4gHdhwA=)isw3EkzTv&du
zaXik%_XM$Y*L7{I)-)_wm8|MspYD&4=`#ky21$7Z8;{L~mJ_<F%kRpVV|uQuGC)<x
z`15f()-fC@)yqULzp0Eqwrv7u8{*jSPrlN)#a;x{3#-IqydF}OaI|T!VZo2+rEm9u
zD)MOx(sUFkK1O93c~KFW4lCkDLst!Cn{*!!?!n)(>smT|d_!YRUzEpSo9(606g`tU
z@p_}%4JM=Tj{XzI4{NS}s0M3K6`Fj?ZAR&#{PI@QMA%|gpmKTE(3CC<Yx_OTTboVD
z+Qcep<e=AX@h<tkl5f~c5%P}#j#bHcftWHFt_V<H?2dY3p%!v<bL!BP%EfLsbcg8+
zTL5>LILk<D#`zskKWe)-=KD1Cq?fcd()b6>WT=AV0T27*{X^1&kikzORMykp3?Ll0
zy!3O@Vy08>8bitfzJk!2CvuyXm7`-wVeMpB`^LyviM|tI8`n#GGAvRE+F!WHg!nQ<
z;Ng=40sqwFXbh_?Rofl7;s5F~^fap&=yK+krf_8PWu-<B08ssQ<Ucn@v=|%vwM&^l
zvqR>8UjKeViP2fqjAG)K8RF%h|KAdB_~)PL?^P7&GB&>7n*R*ptbb{qq(aBjv!jjK
zDSr$88UKKP2?A))q3m2_m*4rDq3AMpMtnkAbUQnU^m6hNz32`0EBLSJ&?HwmeruZ^
z0tBR7XxX~xm$?0Zo1jO-uX1W!F8tG;e+#|87$THiT|BH^JRa&J-K?LO{?`CR7p4aQ
PC@!u~7bZ1hyd3=ptqI2b

diff --git a/spec/models/reports/home_office_spec.rb b/spec/models/reports/home_office_spec.rb
index 673b275b..7441e9b2 100644
--- a/spec/models/reports/home_office_spec.rb
+++ b/spec/models/reports/home_office_spec.rb
@@ -6,12 +6,16 @@ module Reports
   describe HomeOffice do
     include ActiveSupport::Testing::TimeHelpers
 
+    before { create(:mocked_home_office_report_template) }
+
     subject(:report) { described_class.new }
 
+    let(:headers) { report.send(:worksheet)[0].cells.map(&:value) }
+
     it "returns the filename of the Report" do
       frozen_time = Time.zone.local(2023, 7, 17, 12, 30, 45)
       travel_to frozen_time do
-        expected_name = "reports-home-office-20230717-123045.csv"
+        expected_name = "reports-home-office-20230717-123045.xlsx"
 
         report = described_class.new
         actual_name = report.filename
@@ -20,11 +24,12 @@ module Reports
       end
     end
 
-    describe "#csv" do
+    describe "#dataset" do
+      let(:dataset) { report.send(:dataset) }
+
       it "returns applicants who have completed initial checks but not home office checks" do
         app = create(:application, application_progress: build(:application_progress, :home_office_pending))
-
-        expect(report.csv).to include(app.urn)
+        expect(dataset.first).to include(app.urn)
       end
 
       it "does not return rejected applicants" do
@@ -33,73 +38,58 @@ module Reports
                          rejection_completed_at: Time.zone.now)
         app = create(:application, application_progress: progress)
 
-        expect(report.csv).not_to include(app.urn)
+        expect(dataset).not_to include(app.urn)
       end
 
       it "does not return applicants who have not completed initial checks" do
         app = create(:application, application_progress: build(:application_progress, initial_checks_completed_at: nil))
-
-        expect(report.csv).not_to include(app.urn)
+        expect(dataset).not_to include(app.urn)
       end
 
       it "does not return applicants who have completed home office checks" do
         app = create(:application, application_progress: build(:application_progress, :initial_checks_completed, home_office_checks_completed_at: Time.zone.now))
 
-        expect(report.csv).not_to include(app.urn)
+        expect(dataset).not_to include(app.urn)
       end
 
       it "returns the data in CSV format" do
         application = create(:application, application_progress: build(:application_progress, :home_office_pending))
 
-        expect(report.csv).to include([
-          application.urn,
-          application.applicant.full_name,
+        expect(dataset).to contain_exactly([
           application.applicant.date_of_birth,
-          nil,
           application.applicant.nationality,
-          nil,
+          application.urn,
           application.applicant.passport_number,
-          nil,
-          nil,
-          nil,
-          nil,
-          nil,
-          nil,
-        ].join(","))
+          application.applicant.full_name,
+        ])
       end
 
       it "returns the header in CSV format" do
         expected_header = [
-          "ID",
-          "Full Name",
           "DOB",
-          "Gender",
+          "Dummy 1",
+          "Dummy 2",
+          "Full Name/ Organisation Name",
+          "ID (Mandatory)",
           "Nationality",
-          "Place of Birth",
           "Passport Number",
-          "National Insurance Number",
-          "Address",
-          "Postcode",
-          "Email",
-          "Telephone",
-          "Reference",
-        ].join(",")
-
-        expect(report.csv).to include(expected_header)
+        ]
+
+        expect(headers).to match_array(expected_header)
       end
 
       context "includes applications from the csv before invoking `post_generation_hook`" do
         let(:app) { create(:application, application_progress: build(:application_progress, :home_office_pending)) }
-        let(:csv) { report.csv }
+        let(:dataset) { report.send(:dataset) }
 
         before { app }
 
-        it { expect(csv).to include(app.urn) }
+        it { expect(dataset.first).to include(app.urn) }
 
         context "excludes applications from the csv after invoking `post_generation_hook`" do
           before { report.post_generation_hook }
 
-          it { expect(csv).not_to include(app.urn) }
+          it { expect(dataset.first).to be_nil }
         end
       end
     end
diff --git a/spec/validators/home_office_report_config_validator_spec.rb b/spec/validators/home_office_report_config_validator_spec.rb
new file mode 100644
index 00000000..dad124bd
--- /dev/null
+++ b/spec/validators/home_office_report_config_validator_spec.rb
@@ -0,0 +1,70 @@
+require "rails_helper"
+
+describe HomeOfficeReportConfigValidator do
+  subject(:validator) { described_class.new(report_template) }
+
+  let(:report_template) { build(:home_office_report_template, config:) }
+
+  before { validator.validate }
+
+  context "when not a home office report_template skip validation" do
+    let(:report_template) { build(:report_template) }
+
+    it { expect(report_template.errors[:file]).to be_blank }
+    it { expect(report_template.errors[:config]).to be_blank }
+  end
+
+  context "returns error on file" do
+    let(:report_template) { build(:home_office_report_template, file: "bad_file") }
+
+    it { expect(report_template.errors[:file]).not_to be_blank }
+  end
+
+  context "returns error on config" do
+    context "when missing worksheet_name" do
+      let(:config) do
+        {
+          "header_mappings" => {
+            "Column A" => %w[urn],
+          },
+        }
+      end
+
+      it { expect(report_template.errors[:config]).not_to be_blank }
+    end
+
+    context "when worksheet_name does not exist is file" do
+      let(:config) do
+        {
+          "worksheet_name" => "unknown",
+          "header_mappings" => {
+            "Column A" => %w[urn],
+          },
+        }
+      end
+
+      it { expect(report_template.errors[:config]).not_to be_blank }
+    end
+
+    context "when missing header_mappgins" do
+      let(:config) do
+        {
+          "worksheet_name" => "Data",
+        }
+      end
+
+      it { expect(report_template.errors[:config]).not_to be_blank }
+    end
+
+    context "when header_mappgins blank" do
+      let(:config) do
+        {
+          "worksheet_name" => "Data",
+          "header_mappings" => {},
+        }
+      end
+
+      it { expect(report_template.errors[:config]).not_to be_blank }
+    end
+  end
+end