More lettings import validations (#1479)

* clear age2 if it's out of range

* Clear beds if over max

* Clear charges if there are carehome charges given

* Clear charges if scharge is under 0

* Clear tshortfall if it is not positive

* Clear referral if it's an invalid temporary accommodation

* Clear charges if pscharge is outside the range

* Clear charges if supcharg is outside the range

* Refactor clearing charges

* Clear charges is scharge is outside the range

* Clear location and scheme if location is inactive during the tenancy start date

* Remove care home charge if it's outside the range

* Refactor validation error information infor a hash

* Clear age out of range for all people

* Remove unused constant

* Update import logging messages

* Extract earnings validation and remove to_set from hash

* Remove to_delete

* Update logging

* Update bulk upload errors

app/models/validations/financial_validations.rb

@@ -79,7 +79,7 @@ module Validations::FinancialValidations
         record.errors.add :tshortfall, I18n.t("validations.financial.tshortfall.more_than_rent")
         record.errors.add :brent, I18n.t("validations.financial.rent.less_than_shortfall")
       elsif record.wtshortfall < 0.01
-        record.errors.add :tshortfall, I18n.t("validations.financial.tshortfall.must_be_positive")
+        record.errors.add :tshortfall, :must_be_positive, message: I18n.t("validations.financial.tshortfall.must_be_positive")
       end
     end
 
@@ -89,7 +89,7 @@ module Validations::FinancialValidations
 
     answered_questions = [record.tcharge, record.chcharge].concat(record.household_charge && record.household_charge == 1 ? [record.household_charge] : [])
     if answered_questions.count(&:present?) > 1
-      record.errors.add :tcharge, I18n.t("validations.financial.charges.complete_1_of_3") if record.tcharge.present?
+      record.errors.add :tcharge, :complete_1_of_3, message: I18n.t("validations.financial.charges.complete_1_of_3") if record.tcharge.present?
       record.errors.add :chcharge, I18n.t("validations.financial.charges.complete_1_of_3") if record.chcharge.present?
       record.errors.add :household_charge, I18n.t("validations.financial.charges.complete_1_of_3") if record.household_charge.present?
     end
@@ -123,7 +123,7 @@ module Validations::FinancialValidations
         max_chcharge = [record.form.get_question("chcharge", record).prefix, max_chcharge].join("") if record.form.get_question("chcharge", record).present?
         min_chcharge = [record.form.get_question("chcharge", record).prefix, min_chcharge].join("") if record.form.get_question("chcharge", record).present?
         record.errors.add :period, I18n.t("validations.financial.carehome.out_of_range", period:, min_chcharge:, max_chcharge:)
-        record.errors.add :chcharge, I18n.t("validations.financial.carehome.out_of_range", period:, min_chcharge:, max_chcharge:)
+        record.errors.add :chcharge, :out_of_range, message: I18n.t("validations.financial.carehome.out_of_range", period:, min_chcharge:, max_chcharge:)
       end
     end
   end
@@ -174,7 +174,7 @@ private
       maximum = CHARGE_MAXIMUMS.dig(charge, PROVIDER_TYPE[provider_type], NEEDSTYPE_VALUES[record.needstype])
 
       if maximum.present? && record[:period].present? && record[charge].present? && !weekly_value_in_range(record, charge, 0.0, maximum)
-        record.errors.add charge, I18n.t("validations.financial.rent.#{charge}.#{PROVIDER_TYPE[provider_type]}.#{NEEDSTYPE_VALUES[record.needstype]}")
+        record.errors.add charge, :outside_the_range, message: I18n.t("validations.financial.rent.#{charge}.#{PROVIDER_TYPE[provider_type]}.#{NEEDSTYPE_VALUES[record.needstype]}")
       end
     end
   end

app/models/validations/property_validations.rb

@@ -36,7 +36,7 @@ module Validations::PropertyValidations
 
     if record.is_relet_to_temp_tenant? && REFERRAL_INVALID_TMP.include?(record.referral)
       record.errors.add :rsnvac, I18n.t("validations.property.rsnvac.referral_invalid")
-      record.errors.add :referral, I18n.t("validations.household.referral.rsnvac_non_temp")
+      record.errors.add :referral, :referral_invalid, message: I18n.t("validations.household.referral.rsnvac_non_temp")
     end
 
     if record.renewal.present? && record.renewal.zero? && record.rsnvac == 14
@@ -72,7 +72,7 @@ module Validations::PropertyValidations
     end
 
     if record.beds.present? && record.beds > 12
-      record.errors.add :beds, I18n.t("validations.property.beds.over_max")
+      record.errors.add :beds, :over_max, message: I18n.t("validations.property.beds.over_max")
     end
   end
 

app/models/validations/shared_validations.rb

@@ -47,7 +47,7 @@ module Validations::SharedValidations
 
     if location_inactive_status.present?
       date, scope, deactivation_date = location_inactive_status.values_at(:date, :scope, :deactivation_date)
-      record.errors.add field, I18n.t("validations.setup.startdate.location.#{scope}", postcode: record.location.postcode, date:, deactivation_date:)
+      record.errors.add field, :not_active, message: I18n.t("validations.setup.startdate.location.#{scope}", postcode: record.location.postcode, date:, deactivation_date:)
     end
   end
 
@@ -120,9 +120,9 @@ private
     max = [question.prefix, number_with_delimiter(question.max, delimiter: ","), question.suffix].join("") if question.max
 
     if min && max
-      record.errors.add question.id.to_sym, I18n.t("validations.numeric.within_range", field:, min:, max:)
+      record.errors.add question.id.to_sym, :outside_the_range, message: I18n.t("validations.numeric.within_range", field:, min:, max:)
     elsif min
-      record.errors.add question.id.to_sym, I18n.t("validations.numeric.above_min", field:, min:)
+      record.errors.add question.id.to_sym, :under_min, message: I18n.t("validations.numeric.above_min", field:, min:)
     end
   end
 end

app/models/validations/tenancy_validations.rb

@@ -30,7 +30,7 @@ module Validations::TenancyValidations
     conditions.each do |condition|
       next unless condition[:condition]
 
-      record.errors.add :tenancylength, condition[:error]
+      record.errors.add :tenancylength, :tenancylength_invalid, message: condition[:error]
       record.errors.add :tenancy, condition[:error]
     end
   end

app/services/bulk_upload/lettings/year2022/row_parser.rb

@@ -346,7 +346,7 @@ class BulkUpload::Lettings::Year2022::RowParser
 
       fields.each do |field|
         unless errors.include?(field)
-          errors.add(field, error.type)
+          errors.add(field, error.message)
         end
       end
     end

app/services/bulk_upload/lettings/year2023/row_parser.rb

@@ -349,7 +349,7 @@ class BulkUpload::Lettings::Year2023::RowParser
 
       fields.each do |field|
         unless errors.include?(field)
-          errors.add(field, error.type)
+          errors.add(field, error.message)
         end
       end
     end

app/services/imports/lettings_logs_import_service.rb

@@ -279,78 +279,69 @@ module Imports
           attributes.delete(error.attribute.to_s)
         end
         @logs_overridden << lettings_log.old_id
-        save_lettings_log(attributes, previous_status)
-      elsif lettings_log.errors.of_kind?(:referral, :internal_transfer_non_social_housing)
-        @logger.warn("Log #{lettings_log.old_id}: Removing internal transfer referral since previous tenancy is a non social housing")
-        @logs_overridden << lettings_log.old_id
-        attributes.delete("referral")
-        save_lettings_log(attributes, previous_status)
-      elsif lettings_log.errors.of_kind?(:referral, :internal_transfer_fixed_or_lifetime)
-        @logger.warn("Log #{lettings_log.old_id}: Removing internal transfer referral since previous tenancy is fixed terms or lifetime")
-        @logs_overridden << lettings_log.old_id
-        attributes.delete("referral")
-        save_lettings_log(attributes, previous_status)
-      elsif lettings_log.errors.of_kind?(:earnings, :under_hard_min)
-        @logger.warn("Log #{lettings_log.old_id}: Where the income is 0, set earnings and income to blank and set incref to refused")
+        return save_lettings_log(attributes, previous_status)
+      end
+
+      errors = {
+        %i[chcharge out_of_range] => %w[chcharge],
+        %i[referral internal_transfer_non_social_housing] => %w[referral],
+        %i[referral internal_transfer_fixed_or_lifetime] => %w[referral],
+        %i[tenancylength tenancylength_invalid] => %w[tenancylength tenancy],
+        %i[prevten over_20_foster_care] => %w[prevten age1],
+        %i[prevten non_temp_accommodation] => %w[prevten rsnvac],
+        %i[joint not_joint_tenancy] => %w[joint],
+        %i[offered over_20] => %w[offered],
+        %i[earnings over_hard_max] => %w[ecstat1],
+        %i[tshortfall no_outstanding_charges] => %w[tshortfall hbrentshortfall],
+        %i[beds over_max] => %w[beds],
+        %i[tcharge complete_1_of_3] => %w[brent scharge pscharge supcharg tcharge],
+        %i[scharge under_min] => %w[brent scharge pscharge supcharg tcharge],
+        %i[tshortfall must_be_positive] => %w[tshortfall tshortfall_known],
+        %i[referral referral_invalid] => %w[referral],
+        %i[pscharge outside_the_range] => %w[brent scharge pscharge supcharg tcharge],
+        %i[supcharg outside_the_range] => %w[brent scharge pscharge supcharg tcharge],
+        %i[scharge outside_the_range] => %w[brent scharge pscharge supcharg tcharge],
+        %i[location_id not_active] => %w[location_id scheme_id],
+      }
+
+      (2..8).each do |person|
+        errors[[:"age#{person}", :outside_the_range]] = ["age#{person}", "age#{person}_known"]
+      end
+
+      errors.each do |(error, fields)|
+        next unless lettings_log.errors.of_kind?(*error)
+
+        attribute, _type = error
+        fields.each do |field|
+          @logger.warn("Log #{lettings_log.old_id}: Removing #{field} with error: #{lettings_log.errors[attribute].join(', ')}")
+          attributes.delete(field)
+        end
         @logs_overridden << lettings_log.old_id
+        return save_lettings_log(attributes, previous_status)
+      end
 
+      if lettings_log.errors.of_kind?(:earnings, :under_hard_min)
+        @logger.warn("Log #{lettings_log.old_id}: Removing earnings with error: #{lettings_log.errors[:earnings].join(', ')}")
+        @logger.warn("Log #{lettings_log.old_id}: Removing incfreq with error: #{lettings_log.errors[:earnings].join(', ')}")
+        @logs_overridden << lettings_log.old_id
         attributes.delete("earnings")
         attributes.delete("incfreq")
         attributes["incref"] = 1
         attributes["net_income_known"] = 2
-        save_lettings_log(attributes, previous_status)
-      elsif lettings_log.errors.include?(:tenancylength) && lettings_log.errors.include?(:tenancy)
-        @logger.warn("Log #{lettings_log.old_id}: Removing tenancylength as invalid")
-        @logs_overridden << lettings_log.old_id
-        attributes.delete("tenancylength")
-        attributes.delete("tenancy")
-        save_lettings_log(attributes, previous_status)
-      elsif lettings_log.errors.of_kind?(:prevten, :over_20_foster_care)
-        @logger.warn("Log #{lettings_log.old_id}: Removing age1 and prevten as incompatible")
-        @logs_overridden << lettings_log.old_id
-        attributes.delete("prevten")
-        attributes.delete("age1")
-        save_lettings_log(attributes, previous_status)
-      elsif lettings_log.errors.of_kind?(:prevten, :non_temp_accommodation)
-        @logger.warn("Log #{lettings_log.old_id}: Removing vacancy reason and previous tenancy since this accommodation is not temporary")
-        @logs_overridden << lettings_log.old_id
-        attributes.delete("prevten")
-        attributes.delete("rsnvac")
-        save_lettings_log(attributes, previous_status)
-      elsif lettings_log.errors.of_kind?(:joint, :not_joint_tenancy)
-        @logger.warn("Log #{lettings_log.old_id}: Removing joint tenancy as there is only 1 person in the household")
-        @logs_overridden << lettings_log.old_id
-        attributes.delete("joint")
-        save_lettings_log(attributes, previous_status)
-      elsif lettings_log.errors.of_kind?(:offered, :over_20)
-        @logger.warn("Log #{lettings_log.old_id}: Removing offered as the value is above the maximum of 20")
-        @logs_overridden << lettings_log.old_id
-        attributes.delete("offered")
-        save_lettings_log(attributes, previous_status)
-      elsif lettings_log.errors.of_kind?(:earnings, :over_hard_max)
-        @logger.warn("Log #{lettings_log.old_id}: Removing working situation because income is too high for it")
-        @logs_overridden << lettings_log.old_id
-        attributes.delete("ecstat1")
-        save_lettings_log(attributes, previous_status)
-      elsif lettings_log.errors.of_kind?(:tshortfall, :no_outstanding_charges)
-        @logger.warn("Log #{lettings_log.old_id}: Removing tshortfall as there are no outstanding charges")
-        @logs_overridden << lettings_log.old_id
-        attributes.delete("tshortfall")
-        attributes.delete("hbrentshortfall")
-        save_lettings_log(attributes, previous_status)
-      else
-        @logger.error("Log #{lettings_log.old_id}: Failed to import")
-        lettings_log.errors.each do |error|
-          @logger.error("Validation error: Field #{error.attribute}:")
-          @logger.error("\tOwning Organisation: #{lettings_log.owning_organisation&.name}")
-          @logger.error("\tManaging Organisation: #{lettings_log.managing_organisation&.name}")
-          @logger.error("\tOld CORE ID: #{lettings_log.old_id}")
-          @logger.error("\tOld CORE: #{attributes[error.attribute.to_s]&.inspect}")
-          @logger.error("\tNew CORE: #{lettings_log.read_attribute(error.attribute)&.inspect}")
-          @logger.error("\tError message: #{error.type}")
-        end
-        raise exception
+        return save_lettings_log(attributes, previous_status)
+      end
+
+      @logger.error("Log #{lettings_log.old_id}: Failed to import")
+      lettings_log.errors.each do |error|
+        @logger.error("Validation error: Field #{error.attribute}:")
+        @logger.error("\tOwning Organisation: #{lettings_log.owning_organisation&.name}")
+        @logger.error("\tManaging Organisation: #{lettings_log.managing_organisation&.name}")
+        @logger.error("\tOld CORE ID: #{lettings_log.old_id}")
+        @logger.error("\tOld CORE: #{attributes[error.attribute.to_s]&.inspect}")
+        @logger.error("\tNew CORE: #{lettings_log.read_attribute(error.attribute)&.inspect}")
+        @logger.error("\tError message: #{error.type}")
       end
+      raise exception
     end
 
     def compute_differences(lettings_log, attributes)

spec/services/imports/lettings_logs_import_service_spec.rb

@@ -198,7 +198,7 @@ RSpec.describe Imports::LettingsLogsImportService do
 
         it "intercepts the relevant validation error" do
           expect(logger).to receive(:warn).with(/Removing field age2 from log triggering validation: Answer cannot be over 16/)
-          expect(logger).to receive(:warn).with(/Removing field age2 from log triggering validation: Person 2’s age must be between/)
+          expect(logger).to receive(:warn).with(/Removing field age2 from log triggering validation: outside_the_range/)
           expect(logger).to receive(:warn).with(/Removing field ecstat2 from log triggering validation: Answer cannot be ‘child under 16’/)
           expect { lettings_log_service.send(:create_log, lettings_log_xml) }
             .not_to raise_error
@@ -223,7 +223,8 @@ RSpec.describe Imports::LettingsLogsImportService do
         end
 
         it "intercepts the relevant validation error" do
-          expect(logger).to receive(:warn).with(/Where the income is 0, set earnings and income to blank and set incref to refused/)
+          expect(logger).to receive(:warn).with(/Removing earnings with error: Net income cannot be less than £10 per week given the tenant’s working situation/)
+          expect(logger).to receive(:warn).with(/Removing incfreq with error: Net income cannot be less than £10 per week given the tenant’s working situation/)
           expect { lettings_log_service.send(:create_log, lettings_log_xml) }
             .not_to raise_error
         end
@@ -249,7 +250,8 @@ RSpec.describe Imports::LettingsLogsImportService do
         end
 
         it "intercepts the relevant validation error" do
-          expect(logger).to receive(:warn).with(/Removing tenancylength as invalid/)
+          expect(logger).to receive(:warn).with(/Removing tenancylength with error: Enter a tenancy length between 2 and 99 years for a tenancy of this type/)
+          expect(logger).to receive(:warn).with(/Removing tenancy with error: Enter a tenancy length between 2 and 99 years for a tenancy of this type/)
           expect { lettings_log_service.send(:create_log, lettings_log_xml) }
             .not_to raise_error
         end
@@ -274,7 +276,8 @@ RSpec.describe Imports::LettingsLogsImportService do
         end
 
         it "intercepts the relevant validation error" do
-          expect(logger).to receive(:warn).with(/Removing age1 and prevten as incompatible/)
+          expect(logger).to receive(:warn).with(/Removing prevten with error: Answer cannot be a children’s home or foster care as the lead tenant is 20 or older/)
+          expect(logger).to receive(:warn).with(/Removing age1 with error: Answer cannot be a children’s home or foster care as the lead tenant is 20 or older/)
           expect { lettings_log_service.send(:create_log, lettings_log_xml) }
             .not_to raise_error
         end
@@ -327,7 +330,7 @@ RSpec.describe Imports::LettingsLogsImportService do
         end
 
         it "intercepts the relevant validation error" do
-          expect(logger).to receive(:warn).with(/Removing internal transfer referral since previous tenancy is a non social housing/)
+          expect(logger).to receive(:warn).with(/Removing referral with error: Answer cannot be internal transfer as the household situation immediately before this letting was Residential care home/)
           expect { lettings_log_service.send(:create_log, lettings_log_xml) }
             .not_to raise_error
         end
@@ -349,7 +352,7 @@ RSpec.describe Imports::LettingsLogsImportService do
           end
 
           it "intercepts the relevant validation error" do
-            expect(logger).to receive(:warn).with(/Removing internal transfer referral since previous tenancy is fixed terms or lifetime/)
+            expect(logger).to receive(:warn).with(/Removing referral with error: Answer cannot be internal transfer as it’s the same landlord on the tenancy agreement and the household had either a fixed-term or lifetime local authority general needs tenancy immediately before this letting/)
             expect { lettings_log_service.send(:create_log, lettings_log_xml) }
               .not_to raise_error
           end
@@ -379,12 +382,13 @@ RSpec.describe Imports::LettingsLogsImportService do
         end
 
         it "intercepts the relevant validation error" do
-          expect(logger).to receive(:warn).with(/Removing vacancy reason and previous tenancy since this accommodation is not temporary/)
+          expect(logger).to receive(:warn).with(/Removing prevten with error: Answer cannot be non-temporary accommodation as this is a re-let to a tenant who occupied the same property as temporary accommodation/)
+          expect(logger).to receive(:warn).with(/Removing rsnvac with error: Answer cannot be non-temporary accommodation as this is a re-let to a tenant who occupied the same property as temporary accommodation/)
           expect { lettings_log_service.send(:create_log, lettings_log_xml) }
             .not_to raise_error
         end
 
-        it "clears out the referral answer" do
+        it "clears out the vacancy reason answer" do
           allow(logger).to receive(:warn)
 
           lettings_log_service.send(:create_log, lettings_log_xml)
@@ -402,12 +406,12 @@ RSpec.describe Imports::LettingsLogsImportService do
         end
 
         it "intercepts the relevant validation error" do
-          expect(logger).to receive(:warn).with(/Removing offered as the value is above the maximum of 20/)
+          expect(logger).to receive(:warn).with(/Removing offered with error: Enter a number between 0 and 20 for the amount of times the property has been re-let/)
           expect { lettings_log_service.send(:create_log, lettings_log_xml) }
             .not_to raise_error
         end
 
-        it "clears out the referral answer" do
+        it "clears out the number offered answer" do
           allow(logger).to receive(:warn)
 
           lettings_log_service.send(:create_log, lettings_log_xml)
@@ -424,12 +428,12 @@ RSpec.describe Imports::LettingsLogsImportService do
         end
 
         it "intercepts the relevant validation error" do
-          expect(logger).to receive(:warn).with(/Removing working situation because income is too high for it/)
+          expect(logger).to receive(:warn).with(/Removing ecstat1 with error: Net income cannot be greater than £890 per week given the tenant’s working situation/)
           expect { lettings_log_service.send(:create_log, lettings_log_xml) }
             .not_to raise_error
         end
 
-        it "clears out the referral answer" do
+        it "clears out the working situation answer" do
           allow(logger).to receive(:warn)
 
           lettings_log_service.send(:create_log, lettings_log_xml)
@@ -441,6 +445,339 @@ RSpec.describe Imports::LettingsLogsImportService do
         end
       end
 
+      context "and age over the max" do
+        before do
+          lettings_log_xml.at_xpath("//xmlns:P2Age").content = "121"
+          lettings_log_xml.at_xpath("//xmlns:P2Eco").content = "7"
+        end
+
+        it "intercepts the relevant validation error" do
+          expect(logger).to receive(:warn).with(/Removing age2 with error: Person 2’s age must be between 0 and 120/)
+          expect(logger).to receive(:warn).with(/Removing age2_known with error: Person 2’s age must be between 0 and 120/)
+          expect { lettings_log_service.send(:create_log, lettings_log_xml) }
+            .not_to raise_error
+        end
+
+        it "clears out the age answer" do
+          allow(logger).to receive(:warn)
+
+          lettings_log_service.send(:create_log, lettings_log_xml)
+          lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
+
+          expect(lettings_log).not_to be_nil
+          expect(lettings_log.age2).to be_nil
+          expect(lettings_log.age2_known).to be_nil
+        end
+      end
+
+      context "and age 3 over the max" do
+        before do
+          lettings_log_xml.at_xpath("//xmlns:P3Age").content = "121"
+          lettings_log_xml.at_xpath("//xmlns:P3Eco").content = "7"
+          lettings_log_xml.at_xpath("//xmlns:HHMEMB").content = "3"
+        end
+
+        it "intercepts the relevant validation error" do
+          expect(logger).to receive(:warn).with(/Removing age3 with error: Person 3’s age must be between 0 and 120/)
+          expect(logger).to receive(:warn).with(/Removing age3_known with error: Person 3’s age must be between 0 and 120/)
+          expect { lettings_log_service.send(:create_log, lettings_log_xml) }
+            .not_to raise_error
+        end
+
+        it "clears out the age answer" do
+          allow(logger).to receive(:warn)
+
+          lettings_log_service.send(:create_log, lettings_log_xml)
+          lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
+
+          expect(lettings_log).not_to be_nil
+          expect(lettings_log.age3).to be_nil
+          expect(lettings_log.age3_known).to be_nil
+        end
+      end
+
+      context "and beds over the max" do
+        before do
+          lettings_log_xml.at_xpath("//xmlns:Q22").content = "13"
+        end
+
+        it "intercepts the relevant validation error" do
+          expect(logger).to receive(:warn).with(/Removing beds with error: Number of bedrooms cannot be more than 12/)
+          expect { lettings_log_service.send(:create_log, lettings_log_xml) }
+            .not_to raise_error
+        end
+
+        it "clears out the bedrooms answer" do
+          allow(logger).to receive(:warn)
+
+          lettings_log_service.send(:create_log, lettings_log_xml)
+          lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
+
+          expect(lettings_log).not_to be_nil
+          expect(lettings_log.beds).to be_nil
+        end
+      end
+
+      context "and carehome charges and other charges are entered" do
+        let(:lettings_log_id) { "0b4a68df-30cc-474a-93c0-a56ce8fdad3b" }
+
+        before do
+          lettings_log_xml.at_xpath("//xmlns:Q18b").content = "20"
+          lettings_log_xml.at_xpath("//xmlns:Q18c").content = ""
+          lettings_log_xml.at_xpath("//xmlns:Q18ai").content = ""
+          lettings_log_xml.at_xpath("//xmlns:Q18aii").content = "0"
+          lettings_log_xml.at_xpath("//xmlns:Q18aiii").content = ""
+          lettings_log_xml.at_xpath("//xmlns:Q18aiv").content = ""
+          lettings_log_xml.at_xpath("//xmlns:Q18av").content = ""
+        end
+
+        it "intercepts the relevant validation error" do
+          expect(logger).to receive(:warn).with(/Removing brent with error: Enter a total charge that is at least £10 per week, Answer either the ‘household rent and charges’ question or ‘is this accommodation a care home‘, or select ‘no’ for ‘does the household pay rent or charges for the accommodation?/)
+          expect(logger).to receive(:warn).with(/Removing scharge with error: Enter a total charge that is at least £10 per week, Answer either the ‘household rent and charges’ question or ‘is this accommodation a care home‘, or select ‘no’ for ‘does the household pay rent or charges for the accommodation?/)
+          expect(logger).to receive(:warn).with(/Removing pscharge with error: Enter a total charge that is at least £10 per week, Answer either the ‘household rent and charges’ question or ‘is this accommodation a care home‘, or select ‘no’ for ‘does the household pay rent or charges for the accommodation?/)
+          expect(logger).to receive(:warn).with(/Removing supcharg with error: Enter a total charge that is at least £10 per week, Answer either the ‘household rent and charges’ question or ‘is this accommodation a care home‘, or select ‘no’ for ‘does the household pay rent or charges for the accommodation?/)
+          expect(logger).to receive(:warn).with(/Removing tcharge with error: Enter a total charge that is at least £10 per week, Answer either the ‘household rent and charges’ question or ‘is this accommodation a care home‘, or select ‘no’ for ‘does the household pay rent or charges for the accommodation?/)
+          expect { lettings_log_service.send(:create_log, lettings_log_xml) }
+            .not_to raise_error
+        end
+
+        it "clears out the charges answers" do
+          allow(logger).to receive(:warn)
+
+          lettings_log_service.send(:create_log, lettings_log_xml)
+          lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
+
+          expect(lettings_log).not_to be_nil
+          expect(lettings_log.tcharge).to be_nil
+        end
+      end
+
+      context "and scharge is under 0" do
+        let(:lettings_log_id) { "0b4a68df-30cc-474a-93c0-a56ce8fdad3b" }
+
+        before do
+          lettings_log_xml.at_xpath("//xmlns:Q18aii").content = "-1"
+        end
+
+        it "intercepts the relevant validation error" do
+          expect(logger).to receive(:warn).with(/Removing brent with error: Enter an amount above 0, Enter a value for the service charge between £0 and £480 per week if the landlord is a private registered provider and it is a supported housing letting, Service charge must be at least £0 every week/)
+          expect(logger).to receive(:warn).with(/Removing scharge with error: Enter an amount above 0, Enter a value for the service charge between £0 and £480 per week if the landlord is a private registered provider and it is a supported housing letting, Service charge must be at least £0 every week/)
+          expect(logger).to receive(:warn).with(/Removing pscharge with error: Enter an amount above 0, Enter a value for the service charge between £0 and £480 per week if the landlord is a private registered provider and it is a supported housing letting, Service charge must be at least £0 every week/)
+          expect(logger).to receive(:warn).with(/Removing supcharg with error: Enter an amount above 0, Enter a value for the service charge between £0 and £480 per week if the landlord is a private registered provider and it is a supported housing letting, Service charge must be at least £0 every week/)
+          expect(logger).to receive(:warn).with(/Removing tcharge with error: Enter an amount above 0, Enter a value for the service charge between £0 and £480 per week if the landlord is a private registered provider and it is a supported housing letting, Service charge must be at least £0 every week/)
+          expect { lettings_log_service.send(:create_log, lettings_log_xml) }
+            .not_to raise_error
+        end
+
+        it "clears out the charges answers" do
+          allow(logger).to receive(:warn)
+
+          lettings_log_service.send(:create_log, lettings_log_xml)
+          lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
+
+          expect(lettings_log).not_to be_nil
+          expect(lettings_log.brent).to be_nil
+          expect(lettings_log.scharge).to be_nil
+          expect(lettings_log.pscharge).to be_nil
+          expect(lettings_log.supcharg).to be_nil
+          expect(lettings_log.tcharge).to be_nil
+        end
+      end
+
+      context "and tshortfall is not positive" do
+        let(:lettings_log_id) { "0b4a68df-30cc-474a-93c0-a56ce8fdad3b" }
+
+        before do
+          lettings_log_xml.at_xpath("//xmlns:Q18d").content = "1"
+          lettings_log_xml.at_xpath("//xmlns:Q6Ben").content = "1"
+          lettings_log_xml.at_xpath("//xmlns:Q18dyes").content = "0"
+        end
+
+        it "intercepts the relevant validation error" do
+          expect(logger).to receive(:warn).with(/Removing tshortfall with error: Enter a value over £0.01 as you told us there is an outstanding amount/)
+          expect(logger).to receive(:warn).with(/Removing tshortfall_known with error: Enter a value over £0.01 as you told us there is an outstanding amount/)
+          expect { lettings_log_service.send(:create_log, lettings_log_xml) }
+            .not_to raise_error
+        end
+
+        it "clears out the tshortfall answer" do
+          allow(logger).to receive(:warn)
+
+          lettings_log_service.send(:create_log, lettings_log_xml)
+          lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
+
+          expect(lettings_log).not_to be_nil
+          expect(lettings_log.tshortfall).to be_nil
+          expect(lettings_log.tshortfall_known).to be_nil
+        end
+      end
+
+      context "and it has temporary referral in non temporary accommodation" do
+        before do
+          lettings_log_xml.at_xpath("//xmlns:Q27").content = "9"
+          lettings_log_xml.at_xpath("//xmlns:Q16").content = "8"
+        end
+
+        it "intercepts the relevant validation error" do
+          expect(logger).to receive(:warn).with(/Removing referral with error: Answer cannot be this source of referral as this is a re-let to tenant who occupied the same property as temporary accommodation/)
+          expect { lettings_log_service.send(:create_log, lettings_log_xml) }
+            .not_to raise_error
+        end
+
+        it "clears out the referral answer" do
+          allow(logger).to receive(:warn)
+
+          lettings_log_service.send(:create_log, lettings_log_xml)
+          lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
+
+          expect(lettings_log).not_to be_nil
+          expect(lettings_log.referral).to be_nil
+        end
+      end
+
+      context "and pscharge is out of range" do
+        before do
+          lettings_log_xml.at_xpath("//xmlns:Q17").content = "1"
+          lettings_log_xml.at_xpath("//xmlns:Q18aiii").content = "36"
+        end
+
+        it "intercepts the relevant validation error" do
+          expect(logger).to receive(:warn).with(/Removing brent with error: Enter a value for the personal service charge between £0 and £30 per week if the landlord is a private registered provider and it is a general needs letting/)
+          expect(logger).to receive(:warn).with(/Removing scharge with error: Enter a value for the personal service charge between £0 and £30 per week if the landlord is a private registered provider and it is a general needs letting/)
+          expect(logger).to receive(:warn).with(/Removing pscharge with error: Enter a value for the personal service charge between £0 and £30 per week if the landlord is a private registered provider and it is a general needs letting/)
+          expect(logger).to receive(:warn).with(/Removing supcharg with error: Enter a value for the personal service charge between £0 and £30 per week if the landlord is a private registered provider and it is a general needs letting/)
+          expect(logger).to receive(:warn).with(/Removing tcharge with error: Enter a value for the personal service charge between £0 and £30 per week if the landlord is a private registered provider and it is a general needs letting/)
+          expect { lettings_log_service.send(:create_log, lettings_log_xml) }
+            .not_to raise_error
+        end
+
+        it "clears out the charges answers" do
+          allow(logger).to receive(:warn)
+
+          lettings_log_service.send(:create_log, lettings_log_xml)
+          lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
+
+          expect(lettings_log).not_to be_nil
+          expect(lettings_log.pscharge).to be_nil
+          expect(lettings_log.tcharge).to be_nil
+        end
+      end
+
+      context "and supcharg is out of range" do
+        before do
+          lettings_log_xml.at_xpath("//xmlns:Q17").content = "1"
+          lettings_log_xml.at_xpath("//xmlns:Q18aiv").content = "46"
+        end
+
+        it "intercepts the relevant validation error" do
+          expect(logger).to receive(:warn).with(/Removing brent with error: Enter a value for the support charge between £0 and £40 per week if the landlord is a private registered provider and it is a general needs letting/)
+          expect(logger).to receive(:warn).with(/Removing scharge with error: Enter a value for the support charge between £0 and £40 per week if the landlord is a private registered provider and it is a general needs letting/)
+          expect(logger).to receive(:warn).with(/Removing pscharge with error: Enter a value for the support charge between £0 and £40 per week if the landlord is a private registered provider and it is a general needs letting/)
+          expect(logger).to receive(:warn).with(/Removing supcharg with error: Enter a value for the support charge between £0 and £40 per week if the landlord is a private registered provider and it is a general needs letting/)
+          expect(logger).to receive(:warn).with(/Removing tcharge with error: Enter a value for the support charge between £0 and £40 per week if the landlord is a private registered provider and it is a general needs letting/)
+          expect { lettings_log_service.send(:create_log, lettings_log_xml) }
+            .not_to raise_error
+        end
+
+        it "clears out the charges answers" do
+          allow(logger).to receive(:warn)
+
+          lettings_log_service.send(:create_log, lettings_log_xml)
+          lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
+
+          expect(lettings_log).not_to be_nil
+          expect(lettings_log.supcharg).to be_nil
+          expect(lettings_log.tcharge).to be_nil
+        end
+      end
+
+      context "and scharge is out of range" do
+        before do
+          lettings_log_xml.at_xpath("//xmlns:Q17").content = "1"
+          lettings_log_xml.at_xpath("//xmlns:Q18aii").content = "156"
+        end
+
+        it "intercepts the relevant validation error" do
+          expect(logger).to receive(:warn).with(/Removing brent with error: Enter a value for the service charge between £0 and £155 per week if the landlord is a private registered provider and it is a general needs letting/)
+          expect(logger).to receive(:warn).with(/Removing scharge with error: Enter a value for the service charge between £0 and £155 per week if the landlord is a private registered provider and it is a general needs letting/)
+          expect(logger).to receive(:warn).with(/Removing pscharge with error: Enter a value for the service charge between £0 and £155 per week if the landlord is a private registered provider and it is a general needs letting/)
+          expect(logger).to receive(:warn).with(/Removing supcharg with error: Enter a value for the service charge between £0 and £155 per week if the landlord is a private registered provider and it is a general needs letting/)
+          expect(logger).to receive(:warn).with(/Removing tcharge with error: Enter a value for the service charge between £0 and £155 per week if the landlord is a private registered provider and it is a general needs letting/)
+          expect { lettings_log_service.send(:create_log, lettings_log_xml) }
+            .not_to raise_error
+        end
+
+        it "clears out the charges answers" do
+          allow(logger).to receive(:warn)
+
+          lettings_log_service.send(:create_log, lettings_log_xml)
+          lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
+
+          expect(lettings_log).not_to be_nil
+          expect(lettings_log.scharge).to be_nil
+          expect(lettings_log.tcharge).to be_nil
+        end
+      end
+
+      context "and location is not active during the period" do
+        let(:lettings_log_id) { "0b4a68df-30cc-474a-93c0-a56ce8fdad3b" }
+
+        before do
+          location = Location.find_by(old_visible_id: "10")
+          FactoryBot.create(:location_deactivation_period, deactivation_date: Time.zone.local(2021, 10, 10), location:)
+        end
+
+        it "intercepts the relevant validation error" do
+          expect(logger).to receive(:warn).with(/Differences found when saving log/)
+          expect(logger).to receive(:warn).with(/Removing location_id with error: The location LS16 6FT was deactivated on 10 October 2021 and was not available on the day you entered./)
+          expect(logger).to receive(:warn).with(/Removing scheme_id with error: The location LS16 6FT was deactivated on 10 October 2021 and was not available on the day you entered./)
+          expect { lettings_log_service.send(:create_log, lettings_log_xml) }
+            .not_to raise_error
+        end
+
+        it "clears out the location answers" do
+          allow(logger).to receive(:warn)
+
+          lettings_log_service.send(:create_log, lettings_log_xml)
+          lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
+
+          expect(lettings_log).not_to be_nil
+          expect(lettings_log.location).to be_nil
+          expect(lettings_log.scheme).to be_nil
+        end
+      end
+
+      context "and carehome charges are out of range" do
+        let(:lettings_log_id) { "0b4a68df-30cc-474a-93c0-a56ce8fdad3b" }
+
+        before do
+          scheme1.update!(registered_under_care_act: 2)
+          lettings_log_xml.at_xpath("//xmlns:Q18b").content = "2000"
+          lettings_log_xml.at_xpath("//xmlns:Q17").content = "1"
+          lettings_log_xml.at_xpath("//xmlns:Q18ai").content = ""
+          lettings_log_xml.at_xpath("//xmlns:Q18aii").content = ""
+          lettings_log_xml.at_xpath("//xmlns:Q18aiii").content = ""
+          lettings_log_xml.at_xpath("//xmlns:Q18aiv").content = ""
+          lettings_log_xml.at_xpath("//xmlns:Q18av").content = ""
+        end
+
+        it "intercepts the relevant validation error" do
+          expect(logger).to receive(:warn).with(/Removing chcharge with error: Household rent and other charges must be between £10 and £1000 if paying weekly for 52 weeks/)
+          expect { lettings_log_service.send(:create_log, lettings_log_xml) }
+            .not_to raise_error
+        end
+
+        it "clears out the chcharge answer" do
+          allow(logger).to receive(:warn)
+
+          lettings_log_service.send(:create_log, lettings_log_xml)
+          lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
+
+          expect(lettings_log).not_to be_nil
+          expect(lettings_log.chcharge).to be_nil
+        end
+      end
+
       context "and the net income soft validation is triggered (net_income_value_check)" do
         before do
           lettings_log_xml.at_xpath("//xmlns:Q8a").content = "1 Weekly"
@@ -655,7 +992,7 @@ RSpec.describe Imports::LettingsLogsImportService do
       end
 
       it "intercepts the relevant validation error" do
-        expect(logger).to receive(:warn).with(/Removing joint tenancy as there is only 1 person in the household/)
+        expect(logger).to receive(:warn).with(/Removing joint with error: This cannot be a joint tenancy as you've told us there's only one person in the household/)
         expect { lettings_log_service.send(:create_log, lettings_log_xml) }
           .not_to raise_error
       end
@@ -698,7 +1035,8 @@ RSpec.describe Imports::LettingsLogsImportService do
       end
 
       it "intercepts the relevant validation error" do
-        expect(logger).to receive(:warn).with(/Removing tshortfall as there are no outstanding charges/)
+        expect(logger).to receive(:warn).with(/Removing tshortfall with error: You cannot answer the outstanding amount question if you don’t have outstanding rent or charges/)
+        expect(logger).to receive(:warn).with(/Removing hbrentshortfall with error: You cannot answer the outstanding amount question if you don’t have outstanding rent or charges/)
         expect { lettings_log_service.send(:create_log, lettings_log_xml) }
           .not_to raise_error
       end