fix: ASCOR pathways importer so No data text is respected

Vizzuality · Dec 6, 2023 · 55cdf50 · 55cdf50
1 parent 8c3cc45
commit 55cdf50
Show file tree

Hide file tree

Showing 6 changed files with 12 additions and 5 deletions.
diff --git a/app/models/ascor/pathway.rb b/app/models/ascor/pathway.rb
@@ -18,7 +18,7 @@
 #  updated_at             :datetime         not null
 #  trend_source           :string
 #  trend_year             :integer
-#  recent_emission_level  :float
+#  recent_emission_level  :string
 #  recent_emission_source :string
 #  recent_emission_year   :integer
 #

diff --git a/app/services/csv_import/ascor_pathways.rb b/app/services/csv_import/ascor_pathways.rb
@@ -19,7 +19,11 @@ def import
         pathway.trend_source = row[:source_metric_ep1aii] if row.header?(:source_metric_ep1aii)
         pathway.trend_year = row[:year_metric_ep1aii] if row.header?(:year_metric_ep1aii)
         if row.header?(:metric_ep1ai)
-          pathway.recent_emission_level = string_to_float(row[:metric_ep1ai], thousands_separator: ',')
+          pathway.recent_emission_level = string_to_float(
+            row[:metric_ep1ai],
+            thousands_separator: ',',
+            ignored_texts: ['no data']
+          )
         end
         pathway.recent_emission_source = row[:source_metric_ep1ai] if row.header?(:source_metric_ep1ai)
         pathway.recent_emission_year = row[:year_metric_ep1ai] if row.header?(:year_metric_ep1ai)

diff --git a/app/services/csv_import/helpers/emissions.rb b/app/services/csv_import/helpers/emissions.rb
@@ -15,7 +15,8 @@ def emission_headers?(row)
         row.headers.grep(EMISSION_YEAR_PATTERN).any?
       end
 
-      def string_to_float(string, thousands_separator: ',')
+      def string_to_float(string, thousands_separator: ',', ignored_texts: [])
+        return string if ignored_texts.include?(string.downcase)
         return nil if string.blank?
         return string.to_f unless string.is_a?(String)
 

diff --git a/spec/commands/csv_data_upload_spec.rb b/spec/commands/csv_data_upload_spec.rb
@@ -1533,6 +1533,7 @@
     )
 
     pathway = ASCOR::Pathway.joins(:country).find_by ascor_countries: {iso: 'USA'}
+    pathway_2 = ASCOR::Pathway.joins(:country).find_by ascor_countries: {iso: 'JPN'}
 
     expect(pathway.publication_date).to eq(Date.new(2023, 12))
     expect(pathway.assessment_date).to eq(Date.new(2023, 10, 30))
@@ -1574,6 +1575,7 @@
       '2029' => 4311.9,
       '2030' => 4073.4
     )
+    expect(pathway_2.recent_emission_level).to eq('No data')
   end
 
   it 'import CSV file with ASCOR assessment indicators data' do

diff --git a/spec/factories/ascor_pathways.rb b/spec/factories/ascor_pathways.rb
@@ -18,7 +18,7 @@
 #  updated_at             :datetime         not null
 #  trend_source           :string
 #  trend_year             :integer
-#  recent_emission_level  :float
+#  recent_emission_level  :string
 #  recent_emission_source :string
 #  recent_emission_year   :integer
 #

diff --git a/spec/support/fixtures/files/ascor_pathways.csv b/spec/support/fixtures/files/ascor_pathways.csv
@@ -1,3 +1,3 @@
 Id,Country,Emissions metric,Emissions boundary,Units,Assessment date,Publication date,Last historical year,metric EP1.a.i,source metric EP1.a.i,year metric EP1.a.i,metric EP1.a.ii 1-year,metric EP1.a.ii 3-year,metric EP1.a.ii 5-year,source metric EP1.a.ii,year metric EP1.a.ii,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021,2022,2023,2024,2025,2026,2027,2028,2029,2030
 ,United States,Absolute,Production - excluding LULUCF,MtCO2e,10/30/23,2023-12,2021,-	792.3 ,https://zenodo.org/record/7727475,2021,3.9%,-2.3%,-0.9%,https://zenodo.org/record/7727475,2021,NA,"7,380.0","7,480.0","7,260.0","6,800.0","7,020.0","6,860.0","6,620.0","6,800.0","6,860.0","6,700.0","6,550.0","6,510.0","6,700.0","6,580.0","5,990.0","6,220.0","5,981.5","5,743.0","5,504.5","5,266.0","5,027.5","4,789.0","4,550.4","4,311.9","4,073.4"
-,Japan,Intensity per GDP-PPP,Consumption - excluding LULUCF,tCO2/Million I US$,10/30/23,2023-12,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+,Japan,Intensity per GDP-PPP,Consumption - excluding LULUCF,tCO2/Million I US$,10/30/23,2023-12,,No data,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,