CLDC-2933 missing address data import (#2021)

* feat: set encoding by bom for file IO before parsing

* feat: set bom encoding for illness task as well

* feat: use StringIOs in testing and fix row count bug found

* feat: revert toheaders: false

* feat: timecop around tests

* feat: timecop around tests

* feat: fix flaky merge test

* feat: skip over logs that fail validations

* feat: test error added correctly and fix typo

lib/tasks/import_address_from_csv.rake

@@ -55,8 +55,11 @@ namespace :data_import do
       lettings_log.send("process_postcode_changes!")
       lettings_log.values_updated_at = Time.zone.now
 
-      lettings_log.save!
+      if lettings_log.save
         Rails.logger.info("Updated lettings log #{lettings_log_id}, with address: #{[lettings_log.address_line1, lettings_log.address_line2, lettings_log.town_or_city, lettings_log.county, lettings_log.postcode_full].join(', ')}")
+      else
+        Rails.logger.error("Validation failed for lettings log with ID #{lettings_log.id}: #{lettings_log.errors.full_messages.join(', ')}}")
+      end
     end
   end
 
@@ -116,8 +119,11 @@ namespace :data_import do
       sales_log.send("process_postcode_changes!")
       sales_log.values_updated_at = Time.zone.now
 
-      sales_log.save!
+      if sales_log.save
         Rails.logger.info("Updated sales log #{sales_log_id}, with address: #{[sales_log.address_line1, sales_log.address_line2, sales_log.town_or_city, sales_log.county, sales_log.postcode_full].join(', ')}")
+      else
+        Rails.logger.error("Validation failed for sales log with ID #{sales_log.id}: #{sales_log.errors.full_messages.join(', ')}}")
+      end
     end
   end
 end

spec/lib/tasks/correct_address_from_csv_spec.rb

@@ -184,6 +184,14 @@ RSpec.describe "data_import" do
         it "raises an error when no path is given" do
           expect { task.invoke(nil) }.to raise_error(RuntimeError, "Usage: rake data_import:import_lettings_addresses_from_csv['csv_file_name']")
         end
+
+        it "logs an error if a validation fails" do
+          lettings_log.ppcodenk = 0
+          lettings_log.ppostcode_full = "invalid_format"
+          lettings_log.save!(validate: false)
+          expect(Rails.logger).to receive(:error).with(/Validation failed for lettings log with ID #{lettings_log.id}: Ppostcode full/)
+          task.invoke(addresses_csv_path)
+        end
       end
 
       context "when the file does not contain issue type column" do
@@ -272,6 +280,14 @@ RSpec.describe "data_import" do
         it "raises an error when no path is given" do
           expect { task.invoke(nil) }.to raise_error(RuntimeError, "Usage: rake data_import:import_lettings_addresses_from_csv['csv_file_name']")
         end
+
+        it "logs an error if a validation fails" do
+          lettings_log.ppcodenk = 0
+          lettings_log.ppostcode_full = "invalid_format"
+          lettings_log.save!(validate: false)
+          expect(Rails.logger).to receive(:error).with(/Validation failed for lettings log with ID #{lettings_log.id}: Ppostcode full/)
+          task.invoke(addresses_csv_path)
+        end
       end
     end
   end
@@ -405,6 +421,14 @@ RSpec.describe "data_import" do
         it "raises an error when no path is given" do
           expect { task.invoke(nil) }.to raise_error(RuntimeError, "Usage: rake data_import:import_sales_addresses_from_csv['csv_file_name']")
         end
+
+        it "logs an error if a validation fails" do
+          sales_log.ppcodenk = 0
+          sales_log.ppostcode_full = "invalid_format"
+          sales_log.save!(validate: false)
+          expect(Rails.logger).to receive(:error).with(/Validation failed for sales log with ID #{sales_log.id}: Ppostcode full/)
+          task.invoke(addresses_csv_path)
+        end
       end
 
       context "when the file does not contain issue type column" do
@@ -493,6 +517,14 @@ RSpec.describe "data_import" do
         it "raises an error when no path is given" do
           expect { task.invoke(nil) }.to raise_error(RuntimeError, "Usage: rake data_import:import_sales_addresses_from_csv['csv_file_name']")
         end
+
+        it "logs an error if a validation fails" do
+          sales_log.ppcodenk = 0
+          sales_log.ppostcode_full = "invalid_format"
+          sales_log.save!(validate: false)
+          expect(Rails.logger).to receive(:error).with(/Validation failed for sales log with ID #{sales_log.id}: Ppostcode full/)
+          task.invoke(addresses_csv_path)
+        end
       end
     end
   end