CLDC-2933 Fixes to missing address data import task (#2018)

* feat: set encoding by bom for file IO before parsing

* feat: set bom encoding for illness task as well

* feat: use StringIOs in testing and fix row count bug found

* feat: revert toheaders: false

lib/tasks/correct_illness_from_csv.rake

@@ -20,7 +20,9 @@ namespace :correct_illness do
     raise "Usage: rake correct_illness:correct_illness_from_csv['csv_file_name']" if file_name.blank?
 
     s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
-    illness_csv = CSV.parse(s3_service.get_file_io(file_name), headers: false)
+    file_io = s3_service.get_file_io(file_name)
+    file_io.set_encoding_by_bom
+    illness_csv = CSV.parse(file_io, headers: false)
 
     illness_csv.each_with_index do |row, index|
       next if index < 3

lib/tasks/import_address_from_csv.rake

@@ -6,7 +6,9 @@ namespace :data_import do
     raise "Usage: rake data_import:import_lettings_addresses_from_csv['csv_file_name']" if file_name.blank?
 
     s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
-    addresses_csv = CSV.parse(s3_service.get_file_io(file_name), headers: true)
+    file_io = s3_service.get_file_io(file_name)
+    file_io.set_encoding_by_bom
+    addresses_csv = CSV.parse(file_io, headers: true)
     contains_issue_type = addresses_csv.headers.include?("Issue type")
 
     addresses_csv.each do |row|
@@ -65,7 +67,9 @@ namespace :data_import do
     raise "Usage: rake data_import:import_sales_addresses_from_csv['csv_file_name']" if file_name.blank?
 
     s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
-    addresses_csv = CSV.parse(s3_service.get_file_io(file_name), headers: true)
+    file_io = s3_service.get_file_io(file_name)
+    file_io.set_encoding_by_bom
+    addresses_csv = CSV.parse(file_io, headers: true)
     contains_issue_type = addresses_csv.headers.include?("Issue type")
 
     addresses_csv.each do |row|

spec/lib/tasks/correct_address_from_csv_spec.rb

@@ -91,11 +91,11 @@ RSpec.describe "data_import" do
       before do
         allow(storage_service).to receive(:get_file_io)
         .with("addresses_reimport_123.csv")
-        .and_return(replace_entity_ids(lettings_log, lettings_logs[0], lettings_logs[1], lettings_logs[2], File.open("./spec/fixtures/files/addresses_reimport.csv").read))
+        .and_return(StringIO.new(replace_entity_ids(lettings_log, lettings_logs[0], lettings_logs[1], lettings_logs[2], File.open("./spec/fixtures/files/addresses_reimport.csv").read)))
 
         allow(storage_service).to receive(:get_file_io)
         .with("all_addresses_reimport_123.csv")
-        .and_return(replace_entity_ids(lettings_log, lettings_logs[0], lettings_logs[1], lettings_logs[2], File.open("./spec/fixtures/files/addresses_reimport_all_logs.csv").read))
+        .and_return(StringIO.new(replace_entity_ids(lettings_log, lettings_logs[0], lettings_logs[1], lettings_logs[2], File.open("./spec/fixtures/files/addresses_reimport_all_logs.csv").read)))
       end
 
       context "when the file contains issue type column" do
@@ -312,11 +312,11 @@ RSpec.describe "data_import" do
       before do
         allow(storage_service).to receive(:get_file_io)
         .with("addresses_reimport_123.csv")
-        .and_return(replace_entity_ids(sales_log, sales_logs[0], sales_logs[1], sales_logs[2], File.open("./spec/fixtures/files/sales_addresses_reimport.csv").read))
+        .and_return(StringIO.new(replace_entity_ids(sales_log, sales_logs[0], sales_logs[1], sales_logs[2], File.open("./spec/fixtures/files/sales_addresses_reimport.csv").read)))
 
         allow(storage_service).to receive(:get_file_io)
         .with("all_addresses_reimport_123.csv")
-        .and_return(replace_entity_ids(sales_log, sales_logs[0], sales_logs[1], sales_logs[2], File.open("./spec/fixtures/files/sales_addresses_reimport_all_logs.csv").read))
+        .and_return(StringIO.new(replace_entity_ids(sales_log, sales_logs[0], sales_logs[1], sales_logs[2], File.open("./spec/fixtures/files/sales_addresses_reimport_all_logs.csv").read)))
       end
 
       context "when the file contains issue type column" do

spec/lib/tasks/correct_illness_from_csv_spec.rb

@@ -96,7 +96,7 @@ RSpec.describe "correct_illness" do
       before do
         allow(storage_service).to receive(:get_file_io)
         .with("illness_123.csv")
-        .and_return(replace_entity_ids(lettings_log, second_lettings_log, third_lettings_log, File.open("./spec/fixtures/files/illness_update.csv").read))
+        .and_return(StringIO.new(replace_entity_ids(lettings_log, second_lettings_log, third_lettings_log, File.open("./spec/fixtures/files/illness_update.csv").read)))
       end
 
       it "sets illness to yes and sets correct illness type" do