From 61adc430e4849165f273ce1c6b7300235ce08531 Mon Sep 17 00:00:00 2001 From: natdeanlewissoftwire <94526761+natdeanlewissoftwire@users.noreply.github.com> Date: Mon, 6 Nov 2023 14:13:23 +0000 Subject: [PATCH] CLDC-2933 Fixes to missing address data import task (#2018) * feat: set encoding by bom for file IO before parsing * feat: set bom encoding for illness task as well * feat: use StringIOs in testing and fix row count bug found * feat: revert toheaders: false --- lib/tasks/correct_illness_from_csv.rake | 4 +++- lib/tasks/import_address_from_csv.rake | 8 ++++++-- spec/lib/tasks/correct_address_from_csv_spec.rb | 8 ++++---- spec/lib/tasks/correct_illness_from_csv_spec.rb | 2 +- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/lib/tasks/correct_illness_from_csv.rake b/lib/tasks/correct_illness_from_csv.rake index 2fc3eec74..92bbb8921 100644 --- a/lib/tasks/correct_illness_from_csv.rake +++ b/lib/tasks/correct_illness_from_csv.rake @@ -20,7 +20,9 @@ namespace :correct_illness do raise "Usage: rake correct_illness:correct_illness_from_csv['csv_file_name']" if file_name.blank? s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"]) - illness_csv = CSV.parse(s3_service.get_file_io(file_name), headers: false) + file_io = s3_service.get_file_io(file_name) + file_io.set_encoding_by_bom + illness_csv = CSV.parse(file_io, headers: false) illness_csv.each_with_index do |row, index| next if index < 3 diff --git a/lib/tasks/import_address_from_csv.rake b/lib/tasks/import_address_from_csv.rake index 57496a56c..8d0c1de9e 100644 --- a/lib/tasks/import_address_from_csv.rake +++ b/lib/tasks/import_address_from_csv.rake @@ -6,7 +6,9 @@ namespace :data_import do raise "Usage: rake data_import:import_lettings_addresses_from_csv['csv_file_name']" if file_name.blank? s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"]) - addresses_csv = CSV.parse(s3_service.get_file_io(file_name), headers: true) + file_io = s3_service.get_file_io(file_name) + file_io.set_encoding_by_bom + addresses_csv = CSV.parse(file_io, headers: true) contains_issue_type = addresses_csv.headers.include?("Issue type") addresses_csv.each do |row| @@ -65,7 +67,9 @@ namespace :data_import do raise "Usage: rake data_import:import_sales_addresses_from_csv['csv_file_name']" if file_name.blank? s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"]) - addresses_csv = CSV.parse(s3_service.get_file_io(file_name), headers: true) + file_io = s3_service.get_file_io(file_name) + file_io.set_encoding_by_bom + addresses_csv = CSV.parse(file_io, headers: true) contains_issue_type = addresses_csv.headers.include?("Issue type") addresses_csv.each do |row| diff --git a/spec/lib/tasks/correct_address_from_csv_spec.rb b/spec/lib/tasks/correct_address_from_csv_spec.rb index 294ce535b..58bc2e080 100644 --- a/spec/lib/tasks/correct_address_from_csv_spec.rb +++ b/spec/lib/tasks/correct_address_from_csv_spec.rb @@ -91,11 +91,11 @@ RSpec.describe "data_import" do before do allow(storage_service).to receive(:get_file_io) .with("addresses_reimport_123.csv") - .and_return(replace_entity_ids(lettings_log, lettings_logs[0], lettings_logs[1], lettings_logs[2], File.open("./spec/fixtures/files/addresses_reimport.csv").read)) + .and_return(StringIO.new(replace_entity_ids(lettings_log, lettings_logs[0], lettings_logs[1], lettings_logs[2], File.open("./spec/fixtures/files/addresses_reimport.csv").read))) allow(storage_service).to receive(:get_file_io) .with("all_addresses_reimport_123.csv") - .and_return(replace_entity_ids(lettings_log, lettings_logs[0], lettings_logs[1], lettings_logs[2], File.open("./spec/fixtures/files/addresses_reimport_all_logs.csv").read)) + .and_return(StringIO.new(replace_entity_ids(lettings_log, lettings_logs[0], lettings_logs[1], lettings_logs[2], File.open("./spec/fixtures/files/addresses_reimport_all_logs.csv").read))) end context "when the file contains issue type column" do @@ -312,11 +312,11 @@ RSpec.describe "data_import" do before do allow(storage_service).to receive(:get_file_io) .with("addresses_reimport_123.csv") - .and_return(replace_entity_ids(sales_log, sales_logs[0], sales_logs[1], sales_logs[2], File.open("./spec/fixtures/files/sales_addresses_reimport.csv").read)) + .and_return(StringIO.new(replace_entity_ids(sales_log, sales_logs[0], sales_logs[1], sales_logs[2], File.open("./spec/fixtures/files/sales_addresses_reimport.csv").read))) allow(storage_service).to receive(:get_file_io) .with("all_addresses_reimport_123.csv") - .and_return(replace_entity_ids(sales_log, sales_logs[0], sales_logs[1], sales_logs[2], File.open("./spec/fixtures/files/sales_addresses_reimport_all_logs.csv").read)) + .and_return(StringIO.new(replace_entity_ids(sales_log, sales_logs[0], sales_logs[1], sales_logs[2], File.open("./spec/fixtures/files/sales_addresses_reimport_all_logs.csv").read))) end context "when the file contains issue type column" do diff --git a/spec/lib/tasks/correct_illness_from_csv_spec.rb b/spec/lib/tasks/correct_illness_from_csv_spec.rb index ef7e82a65..e7b9f3c72 100644 --- a/spec/lib/tasks/correct_illness_from_csv_spec.rb +++ b/spec/lib/tasks/correct_illness_from_csv_spec.rb @@ -96,7 +96,7 @@ RSpec.describe "correct_illness" do before do allow(storage_service).to receive(:get_file_io) .with("illness_123.csv") - .and_return(replace_entity_ids(lettings_log, second_lettings_log, third_lettings_log, File.open("./spec/fixtures/files/illness_update.csv").read)) + .and_return(StringIO.new(replace_entity_ids(lettings_log, second_lettings_log, third_lettings_log, File.open("./spec/fixtures/files/illness_update.csv").read))) end it "sets illness to yes and sets correct illness type" do