Merge branch 'main' into DO_NOT_MERGE

# Conflicts:
#	spec/services/imports/lettings_logs_import_service_spec.rb

.github/workflows/staging_pipeline.yml

@@ -229,4 +229,4 @@ jobs:
           cf set-env $APP_NAME S3_CONFIG $S3_CONFIG
           cf set-env $APP_NAME CSV_DOWNLOAD_PAAS_INSTANCE $CSV_DOWNLOAD_PAAS_INSTANCE
           cf set-env $APP_NAME SENTRY_DSN $SENTRY_DSN
-          cf push $APP_NAME --strategy rolling
+          cf push $APP_NAME --strategy rolling -t 180

app/models/form/lettings/pages/pscharge_value_check.rb

@@ -0,0 +1,24 @@
+class Form::Lettings::Pages::PschargeValueCheck < ::Form::Page
+  def initialize(id, hsh, subsection)
+    super
+    @id = "pscharge_value_check"
+    @depends_on = [{ "pscharge_over_soft_max?" => true }]
+    @title_text = {
+      "translation" => "soft_validations.pscharge.over_soft_max_title",
+      "arguments" => [{
+        "key" => "pscharge",
+        "label" => true,
+        "i18n_template" => "pscharge",
+      }],
+    }
+    @informative_text = I18n.t("soft_validations.charges.informative_text")
+  end
+
+  def questions
+    @questions ||= [Form::Lettings::Questions::PschargeValueCheck.new(nil, nil, self)]
+  end
+
+  def interruption_screen_question_ids
+    %w[period needstype pscharge]
+  end
+end

app/models/form/lettings/pages/reason_for_leaving_last_settled_home_renewal.rb

@@ -6,6 +6,9 @@ class Form::Lettings::Pages::ReasonForLeavingLastSettledHomeRenewal < ::Form::Pa
   end
 
   def questions
-    @questions ||= [Form::Lettings::Questions::ReasonRenewal.new(nil, nil, self)]
+    @questions ||= [
+      Form::Lettings::Questions::ReasonRenewal.new(nil, nil, self),
+      Form::Lettings::Questions::Reasonother.new(nil, nil, self),
+    ]
   end
 end

app/models/form/lettings/pages/referral_value_check.rb

@@ -0,0 +1,22 @@
+class Form::Lettings::Pages::ReferralValueCheck < ::Form::Page
+  def initialize(id, hsh, subsection)
+    super
+    @id = "referral_value_check"
+    @depends_on = [{ "la_referral_for_general_needs?" => true }]
+    @title_text = {
+      "translation" => "soft_validations.referral.title_text",
+    }
+    @informative_text = {
+      "translation" => "soft_validations.referral.hint_text",
+      "arguments" => [],
+    }
+  end
+
+  def questions
+    @questions ||= [Form::Lettings::Questions::ReferralValueCheck.new(nil, nil, self)]
+  end
+
+  def interruption_screen_question_ids
+    %w[needstype referral]
+  end
+end

app/models/form/lettings/pages/scharge_value_check.rb

@@ -0,0 +1,24 @@
+class Form::Lettings::Pages::SchargeValueCheck < ::Form::Page
+  def initialize(id, hsh, subsection)
+    super
+    @id = "scharge_value_check"
+    @depends_on = [{ "scharge_over_soft_max?" => true }]
+    @title_text = {
+      "translation" => "soft_validations.scharge.over_soft_max_title",
+      "arguments" => [{
+        "key" => "scharge",
+        "label" => true,
+        "i18n_template" => "scharge",
+      }],
+    }
+    @informative_text = I18n.t("soft_validations.charges.informative_text")
+  end
+
+  def questions
+    @questions ||= [Form::Lettings::Questions::SchargeValueCheck.new(nil, nil, self)]
+  end
+
+  def interruption_screen_question_ids
+    %w[period needstype scharge]
+  end
+end

app/models/form/lettings/pages/supcharg_value_check.rb

@@ -0,0 +1,24 @@
+class Form::Lettings::Pages::SupchargValueCheck < ::Form::Page
+  def initialize(id, hsh, subsection)
+    super
+    @id = "supcharg_value_check"
+    @depends_on = [{ "supcharg_over_soft_max?" => true }]
+    @title_text = {
+      "translation" => "soft_validations.supcharg.over_soft_max_title",
+      "arguments" => [{
+        "key" => "supcharg",
+        "label" => true,
+        "i18n_template" => "supcharg",
+      }],
+    }
+    @informative_text = I18n.t("soft_validations.charges.informative_text")
+  end
+
+  def questions
+    @questions ||= [Form::Lettings::Questions::SupchargValueCheck.new(nil, nil, self)]
+  end
+
+  def interruption_screen_question_ids
+    %w[period needstype supcharg]
+  end
+end

app/models/form/lettings/questions/pscharge_value_check.rb

@@ -0,0 +1,15 @@
+class Form::Lettings::Questions::PschargeValueCheck < ::Form::Question
+  def initialize(id, hsh, page)
+    super
+    @id = "pscharge_value_check"
+    @check_answer_label = "Personal service charge confirmation"
+    @header = "Are you sure?"
+    @type = "interruption_screen"
+    @check_answers_card_number = 0
+    @answer_options = ANSWER_OPTIONS
+    @hidden_in_check_answers = { "depends_on" => [{ "pscharge_value_check" => 0 }, { "pscharge_value_check" => 1 }] }
+    @hint_text = I18n.t("soft_validations.charges.hint_text")
+  end
+
+  ANSWER_OPTIONS = { "0" => { "value" => "Yes" }, "1" => { "value" => "No" } }.freeze
+end

app/models/form/lettings/questions/reason_renewal.rb

@@ -9,10 +9,27 @@ class Form::Lettings::Questions::ReasonRenewal < ::Form::Question
     @hint_text = "You told us this letting is a renewal. We have removed some options because of this."
     @answer_options = ANSWER_OPTIONS
     @question_number = 77
+    @conditional_for = {
+      "reasonother" => [
+        20,
+      ],
+    }
   end
 
   ANSWER_OPTIONS = {
     "40" => { "value" => "End of assured shorthold tenancy (no fault)" },
     "42" => { "value" => "End of fixed term tenancy (no fault)" },
+    "20" => {
+      "value" => "Other",
+    },
+    "47" => {
+      "value" => "Tenant prefers not to say",
+    },
+    "divider" => {
+      "value" => true,
+    },
+    "28" => {
+      "value" => "Don’t know",
+    },
   }.freeze
 end

app/models/form/lettings/questions/referral_prp.rb

@@ -21,6 +21,9 @@ class Form::Lettings::Questions::ReferralPrp < ::Form::Question
     "3" => {
       "value" => "Nominated by a local housing authority",
     },
+    "4" => {
+      "value" => "Referred by local authority housing department",
+    },
     "8" => {
       "value" => "Re-located through official housing mobility scheme",
     },
@@ -45,6 +48,9 @@ class Form::Lettings::Questions::ReferralPrp < ::Form::Question
     "13" => {
       "value" => "Youth offending team",
     },
+    "17" => {
+      "value" => "Children’s Social Care",
+    },
     "16" => {
       "value" => "Other",
     },

app/models/form/lettings/questions/referral_supported_housing.rb

@@ -45,6 +45,9 @@ class Form::Lettings::Questions::ReferralSupportedHousing < ::Form::Question
     "13" => {
       "value" => "Youth offending team",
     },
+    "17" => {
+      "value" => "Children’s Social Care",
+    },
     "16" => {
       "value" => "Other",
     },

app/models/form/lettings/questions/referral_supported_housing_prp.rb

@@ -24,6 +24,7 @@ class Form::Lettings::Questions::ReferralSupportedHousingPrp < ::Form::Question
     "12" => { "value" => "Police, probation or prison" },
     "7" => { "value" => "Voluntary agency" },
     "13" => { "value" => "Youth offending team" },
+    "17" => { "value" => "Children’s Social Care" },
     "16" => { "value" => "Other" },
   }.freeze
 end

app/models/form/lettings/questions/referral_value_check.rb

@@ -0,0 +1,14 @@
+class Form::Lettings::Questions::ReferralValueCheck < ::Form::Question
+  def initialize(id, hsh, page)
+    super
+    @id = "referral_value_check"
+    @check_answer_label = "Referral confirmation"
+    @header = "Are you sure?"
+    @type = "interruption_screen"
+    @check_answers_card_number = 0
+    @answer_options = ANSWER_OPTIONS
+    @hidden_in_check_answers = { "depends_on" => [{ "referral_value_check" => 0 }, { "referral_value_check" => 1 }] }
+  end
+
+  ANSWER_OPTIONS = { "0" => { "value" => "Yes" }, "1" => { "value" => "No" } }.freeze
+end

app/models/form/lettings/questions/scharge_value_check.rb

@@ -0,0 +1,15 @@
+class Form::Lettings::Questions::SchargeValueCheck < ::Form::Question
+  def initialize(id, hsh, page)
+    super
+    @id = "scharge_value_check"
+    @check_answer_label = "Service charge confirmation"
+    @header = "Are you sure?"
+    @type = "interruption_screen"
+    @check_answers_card_number = 0
+    @answer_options = ANSWER_OPTIONS
+    @hidden_in_check_answers = { "depends_on" => [{ "scharge_value_check" => 0 }, { "scharge_value_check" => 1 }] }
+    @hint_text = I18n.t("soft_validations.charges.hint_text")
+  end
+
+  ANSWER_OPTIONS = { "0" => { "value" => "Yes" }, "1" => { "value" => "No" } }.freeze
+end

app/models/form/lettings/questions/supcharg_value_check.rb

@@ -0,0 +1,15 @@
+class Form::Lettings::Questions::SupchargValueCheck < ::Form::Question
+  def initialize(id, hsh, page)
+    super
+    @id = "supcharg_value_check"
+    @check_answer_label = "Support charge confirmation"
+    @header = "Are you sure?"
+    @type = "interruption_screen"
+    @check_answers_card_number = 0
+    @answer_options = ANSWER_OPTIONS
+    @hidden_in_check_answers = { "depends_on" => [{ "supcharg_value_check" => 0 }, { "supcharg_value_check" => 1 }] }
+    @hint_text = I18n.t("soft_validations.charges.hint_text")
+  end
+
+  ANSWER_OPTIONS = { "0" => { "value" => "Yes" }, "1" => { "value" => "No" } }.freeze
+end

app/models/form/lettings/subsections/household_situation.rb

@@ -24,6 +24,7 @@ class Form::Lettings::Subsections::HouseholdSituation < ::Form::Subsection
       Form::Lettings::Pages::ReferralPrp.new(nil, nil, self),
       Form::Lettings::Pages::ReferralSupportedHousing.new(nil, nil, self),
       Form::Lettings::Pages::ReferralSupportedHousingPrp.new(nil, nil, self),
+      Form::Lettings::Pages::ReferralValueCheck.new(nil, nil, self),
     ].compact
   end
 end

app/models/form/lettings/subsections/income_and_benefits.rb

@@ -26,6 +26,9 @@ class Form::Lettings::Subsections::IncomeAndBenefits < ::Form::Subsection
       Form::Lettings::Pages::RentMonthly.new(nil, nil, self),
       Form::Lettings::Pages::MinRentValueCheck.new("brent_min_rent_value_check", nil, self, check_answers_card_number: 0),
       Form::Lettings::Pages::MaxRentValueCheck.new("brent_max_rent_value_check", nil, self, check_answers_card_number: 0),
+      Form::Lettings::Pages::SchargeValueCheck.new(nil, nil, self),
+      Form::Lettings::Pages::PschargeValueCheck.new(nil, nil, self),
+      Form::Lettings::Pages::SupchargValueCheck.new(nil, nil, self),
       Form::Lettings::Pages::Outstanding.new(nil, nil, self),
       Form::Lettings::Pages::OutstandingAmount.new(nil, nil, self),
     ].compact

app/models/lettings_log.rb

@@ -568,6 +568,10 @@ class LettingsLog < Log
     end
   end
 
+  def la_referral_for_general_needs?
+    is_general_needs? && referral == 4
+  end
+
 private
 
   def reset_invalid_unresolved_log_fields!

app/models/validations/financial_validations.rb

@@ -149,32 +149,32 @@ private
   CHARGE_MAXIMUMS = {
     scharge: {
       private_registered_provider: {
-        general_needs: 155,
-        supported_housing: 480,
+        general_needs: 800,
+        supported_housing: 800,
       },
       local_authority: {
-        general_needs: 155,
-        supported_housing: 365,
+        general_needs: 500,
+        supported_housing: 500,
       },
     },
     pscharge: {
       private_registered_provider: {
-        general_needs: 30,
-        supported_housing: 200,
+        general_needs: 700,
+        supported_housing: 700,
       },
       local_authority: {
-        general_needs: 35,
-        supported_housing: 75,
+        general_needs: 200,
+        supported_housing: 200,
       },
     },
     supcharg: {
       private_registered_provider: {
-        general_needs: 40,
-        supported_housing: 465,
+        general_needs: 800,
+        supported_housing: 800,
       },
       local_authority: {
-        general_needs: 60,
-        supported_housing: 120,
+        general_needs: 200,
+        supported_housing: 200,
       },
     },
   }.freeze

app/models/validations/household_validations.rb

@@ -4,10 +4,6 @@ module Validations::HouseholdValidations
   # Validations methods need to be called 'validate_<page_name>' to run on model save
   # or 'validate_' to run on submit as well
   def validate_reasonable_preference(record)
-    if record.is_not_homeless? && record.rp_homeless == 1
-      record.errors.add :reasonable_preference_reason, I18n.t("validations.household.reasonpref.not_homeless")
-      record.errors.add :homeless, I18n.t("validations.household.homeless.reasonpref.not_homeless")
-    end
     if !record.given_reasonable_preference? && [record.rp_homeless, record.rp_insan_unsat, record.rp_medwel, record.rp_hardship, record.rp_dontknow].any? { |a| a == 1 }
       record.errors.add :reasonable_preference_reason, I18n.t("validations.household.reasonable_preference_reason.reason_not_required")
     end

app/models/validations/sales/financial_validations.rb

@@ -30,9 +30,9 @@ module Validations::Sales::FinancialValidations
     combined_income = record.income1 + record.income2
     relevant_fields = %i[income1 income2 ownershipsch uprn la postcode_full]
     if record.london_property? && combined_income > 90_000
-      relevant_fields.each { |field| record.errors.add field, I18n.t("validations.financial.income.combined_over_hard_max_for_london") }
+      relevant_fields.each { |field| record.errors.add field, :over_combined_hard_max_for_london, message: I18n.t("validations.financial.income.combined_over_hard_max_for_london") }
     elsif record.property_not_in_london? && combined_income > 80_000
-      relevant_fields.each { |field| record.errors.add field, I18n.t("validations.financial.income.combined_over_hard_max_for_outside_london") }
+      relevant_fields.each { |field| record.errors.add field, :over_combined_hard_max_for_outside_london, message: I18n.t("validations.financial.income.combined_over_hard_max_for_outside_london") }
     end
   end
 

app/models/validations/sales/sale_information_validations.rb

@@ -11,19 +11,6 @@ module Validations::Sales::SaleInformationValidations
     end
   end
 
-  def validate_years_living_in_property_before_purchase(record)
-    return unless record.proplen&.nonzero?
-
-    case record.type
-    when 18
-      record.errors.add :type, I18n.t("validations.sale_information.proplen.social_homebuy")
-      record.errors.add :proplen, I18n.t("validations.sale_information.proplen.social_homebuy")
-    when 28, 29
-      record.errors.add :type, I18n.t("validations.sale_information.proplen.rent_to_buy")
-      record.errors.add :proplen, I18n.t("validations.sale_information.proplen.rent_to_buy")
-    end
-  end
-
   def validate_exchange_date(record)
     return unless record.exdate && record.saledate
 

app/models/validations/shared_validations.rb

@@ -49,7 +49,7 @@ module Validations::SharedValidations
       elsif incorrect_accuracy || value.to_d != value.to_i    # if the user enters a value in exponent notation (eg '4e1') the to_i method does not convert this to the correct value
         field = question.check_answer_label || question.id
         case question.step
-        when 1 then record.errors.add question.id.to_sym, I18n.t("validations.numeric.whole_number", field:)
+        when 1 then record.errors.add question.id.to_sym, :not_integer, message: I18n.t("validations.numeric.whole_number", field:)
         when 10 then record.errors.add question.id.to_sym, I18n.t("validations.numeric.nearest_ten", field:)
         end
       end

app/models/validations/soft_validations.rb

@@ -97,6 +97,42 @@ module Validations::SoftValidations
     net_income_in_soft_max_range? ? "higher" : "lower"
   end
 
+  def scharge_over_soft_max?
+    return unless scharge && period && needstype
+    return if weekly_value(scharge).blank?
+
+    max = if needstype == 1
+            owning_organisation.provider_type == "LA" ? 25 : 35
+          else
+            owning_organisation.provider_type == "LA" ? 100 : 200
+          end
+    weekly_value(scharge) > max
+  end
+
+  def pscharge_over_soft_max?
+    return unless pscharge && period && needstype
+    return if weekly_value(pscharge).blank?
+
+    max = if needstype == 1
+            owning_organisation.provider_type == "LA" ? 25 : 35
+          else
+            owning_organisation.provider_type == "LA" ? 75 : 100
+          end
+    weekly_value(pscharge) > max
+  end
+
+  def supcharg_over_soft_max?
+    return unless supcharg && period && needstype
+    return if weekly_value(supcharg).blank?
+
+    max = if needstype == 1
+            owning_organisation.provider_type == "LA" ? 25 : 35
+          else
+            owning_organisation.provider_type == "LA" ? 75 : 85
+          end
+    weekly_value(supcharg) > max
+  end
+
 private
 
   def details_known_or_lead_tenant?(tenant_number)

app/services/bulk_upload/lettings/year2022/row_parser.rb

@@ -689,14 +689,10 @@ private
   end
 
   def validate_reasonable_preference_homeless
-    if field_69 == 1 && homeless == 1 && field_70 == 1
-      errors.add(:field_70, I18n.t("validations.household.reasonpref.not_homeless"))
-    else
-      reason_fields = %i[field_70 field_71 field_72 field_73 field_74]
-      if field_69 == 1 && reason_fields.all? { |field| attributes[field.to_s].blank? }
-        reason_fields.each do |field|
-          errors.add(field, I18n.t("validations.not_answered", question: "reason for reasonable preference"))
-        end
+    reason_fields = %i[field_70 field_71 field_72 field_73 field_74]
+    if field_69 == 1 && reason_fields.all? { |field| attributes[field.to_s].blank? }
+      reason_fields.each do |field|
+        errors.add(field, I18n.t("validations.not_answered", question: "reason for reasonable preference"))
       end
     end
   end

app/services/bulk_upload/lettings/year2023/row_parser.rb

@@ -622,14 +622,10 @@ private
   end
 
   def validate_reasonable_preference_homeless
-    if field_110 == 1 && field_105 == 1 && field_111 == 1
-      errors.add(:field_111, I18n.t("validations.household.reasonpref.not_homeless"))
-    else
-      reason_fields = %i[field_111 field_112 field_113 field_114 field_115]
-      if field_110 == 1 && reason_fields.all? { |field| attributes[field.to_s].blank? }
-        reason_fields.each do |field|
-          errors.add(field, I18n.t("validations.not_answered", question: "reason for reasonable preference"))
-        end
+    reason_fields = %i[field_111 field_112 field_113 field_114 field_115]
+    if field_110 == 1 && reason_fields.all? { |field| attributes[field.to_s].blank? }
+      reason_fields.each do |field|
+        errors.add(field, I18n.t("validations.not_answered", question: "reason for reasonable preference"))
       end
     end
   end

app/services/exports/lettings_log_export_service.rb

@@ -119,7 +119,7 @@ module Exports
     def retrieve_lettings_logs(start_time, recent_export, full_update)
       if !full_update && recent_export
         params = { from: recent_export.started_at, to: start_time }
-        LettingsLog.exportable.where("updated_at >= :from and updated_at <= :to", params)
+        LettingsLog.exportable.where("(updated_at >= :from AND updated_at <= :to) OR (values_updated_at IS NOT NULL AND values_updated_at >= :from AND values_updated_at <= :to)", params)
       else
         params = { to: start_time }
         LettingsLog.exportable.where("updated_at <= :to", params)

app/services/imports/import_report_service.rb

@@ -11,6 +11,7 @@ module Imports
     def create_reports(report_suffix)
       generate_missing_data_coordinators_report(report_suffix)
       generate_logs_report(report_suffix)
+      generate_unassigned_logs_report(report_suffix)
     end
 
     def generate_missing_data_coordinators_report(report_suffix)
@@ -53,5 +54,99 @@ module Imports
 
       @logger.info("Logs report available in s3 import bucket at #{report_name}")
     end
+
+    def generate_unassigned_logs_report(report_suffix)
+      Rails.logger.info("Generating unassigned logs report")
+
+      rep = CSV.generate do |report|
+        headers = ["Owning Organisation ID", "Old Owning Organisation ID", "Managing Organisation ID", "Old Managing Organisation ID", "Log ID", "Old Log ID", "Tenancy code", "Purchaser code"]
+        report << headers
+
+        @institutions_csv.each do |row|
+          name = row[0]
+          organisation = Organisation.find_by(name:)
+          next unless organisation
+
+          unassigned_user = organisation.users.find_by(name: "Unassigned")
+          next unless unassigned_user
+
+          organisation.owned_lettings_logs.where(created_by: unassigned_user).each do |lettings_log|
+            report << [organisation.id, organisation.old_org_id, lettings_log.managing_organisation.id, lettings_log.managing_organisation.old_org_id, lettings_log.id, lettings_log.old_id, lettings_log.tenancycode, nil]
+          end
+          organisation.owned_sales_logs.where(created_by: unassigned_user).each do |sales_log|
+            report << [organisation.id, organisation.old_org_id, nil, nil, sales_log.id, sales_log.old_id, nil, sales_log.purchid]
+          end
+        end
+      end
+
+      report_name = "UnassignedLogsReport_#{report_suffix}"
+      @storage_service.write_file(report_name, BYTE_ORDER_MARK + rep)
+
+      @logger.info("Unassigned logs report available in s3 import bucket at #{report_name}")
+    end
+
+    def generate_missing_answers_report(report_suffix)
+      create_missing_answers_report(report_suffix, LettingsLog)
+      create_missing_answers_report(report_suffix, SalesLog)
+    end
+
+    def create_missing_answers_report(report_suffix, log_class)
+      class_name = log_class.name.underscore.humanize
+      Rails.logger.info("Generating missing imported #{class_name}s answers report")
+
+      unanswered_question_counts, missing_answers_example_sets = process_missing_answers(log_class)
+
+      report_name = "MissingAnswersReport#{log_class.name}_#{report_suffix}.csv"
+      @storage_service.write_file(report_name, BYTE_ORDER_MARK + missing_answers_report(unanswered_question_counts))
+
+      examples_report_name = "MissingAnswersExamples#{log_class.name}_#{report_suffix}.csv"
+      @storage_service.write_file(examples_report_name, BYTE_ORDER_MARK + missing_answers_examples(missing_answers_example_sets))
+
+      @logger.info("Missing #{class_name}s answers report available in s3 import bucket at #{report_name}")
+    end
+
+    def process_missing_answers(log_class)
+      unanswered_question_counts = {}
+      missing_answers_example_sets = {}
+
+      log_class.where.not(old_id: nil).where(status: "in_progress").each do |log|
+        applicable_questions = log.form.subsections.map { |s| s.applicable_questions(log).select { |q| q.enabled?(log) } }.flatten
+        unanswered_questions = (applicable_questions.filter { |q| q.unanswered?(log) }.map(&:id) - log.optional_fields).join(", ")
+
+        if unanswered_question_counts[unanswered_questions].present?
+          unanswered_question_counts[unanswered_questions] += 1
+          missing_answers_example_sets[unanswered_questions] << { id: log.id, old_form_id: log.old_form_id, old_id: log.old_id, owning_organisation_id: log.owning_organisation_id } unless unanswered_question_counts[unanswered_questions] > 10
+        else
+          unanswered_question_counts[unanswered_questions] = 1
+          missing_answers_example_sets[unanswered_questions] = [{ id: log.id, old_form_id: log.old_form_id, old_id: log.old_id, owning_organisation_id: log.owning_organisation_id }]
+        end
+      end
+
+      [unanswered_question_counts, missing_answers_example_sets]
+    end
+
+    def missing_answers_report(unanswered_question_counts)
+      CSV.generate do |report|
+        headers = ["Missing answers", "Total number of affected logs"]
+        report << headers
+
+        unanswered_question_counts.each do |missing_answers, count|
+          report << [missing_answers, count]
+        end
+      end
+    end
+
+    def missing_answers_examples(missing_answers_example_sets)
+      CSV.generate do |report|
+        headers = ["Missing answers", "Organisation ID", "Log ID", "Old Form ID", "Old Log ID"]
+        report << headers
+
+        missing_answers_example_sets.each do |missing_answers, examples|
+          examples.each do |example|
+            report << [missing_answers, example[:owning_organisation_id], example[:id], example[:old_form_id], example[:old_id]]
+          end
+        end
+      end
+    end
   end
 end

app/services/imports/lettings_logs_field_import_service.rb

@@ -12,6 +12,10 @@ module Imports
         import_from(folder, :update_offered)
       when "creation_method"
         import_from(folder, :update_creation_method)
+      when "address"
+        import_from(folder, :update_address)
+      when "reason"
+        import_from(folder, :update_reason)
       else
         raise "Updating #{field} is not supported by the field import service"
       end
@@ -132,5 +136,72 @@ module Imports
         @logger.warn("Could not find record matching legacy ID #{old_id}")
       end
     end
+
+    def update_address(xml_doc)
+      return if meta_field_value(xml_doc, "form-name").include?("Sales")
+
+      old_id = meta_field_value(xml_doc, "document-id")
+      record = LettingsLog.find_by(old_id:)
+      return @logger.info("lettings log #{record.id} is from previous collection year, skipping") if record.collection_start_year < 2023
+
+      if record.present?
+        if string_or_nil(xml_doc, "AddressLine1").present? && string_or_nil(xml_doc, "TownCity").present?
+          record.la = string_or_nil(xml_doc, "Q28ONS")
+          record.postcode_full = compose_postcode(xml_doc, "POSTCODE", "POSTCOD2")
+          record.postcode_known = postcode_known(record)
+          record.address_line1 = string_or_nil(xml_doc, "AddressLine1")
+          record.address_line2 = string_or_nil(xml_doc, "AddressLine2")
+          record.town_or_city = string_or_nil(xml_doc, "TownCity")
+          record.county = string_or_nil(xml_doc, "County")
+          record.uprn = nil
+          record.uprn_known = 0
+          record.uprn_confirmed = 0
+          record.values_updated_at = Time.zone.now
+          record.save!
+          @logger.info("lettings log #{record.id} address_line1 value has been set to #{record.address_line1}")
+          @logger.info("lettings log #{record.id} address_line2 value has been set to #{record.address_line2}")
+          @logger.info("lettings log #{record.id} town_or_city value has been set to #{record.town_or_city}")
+          @logger.info("lettings log #{record.id} county value has been set to #{record.county}")
+          @logger.info("lettings log #{record.id} postcode_full value has been set to #{record.postcode_full}")
+        else
+          @logger.info("lettings log #{record.id} is missing either or both of address_line1 and town or city, skipping")
+        end
+      else
+        @logger.warn("Could not find record matching legacy ID #{old_id}")
+      end
+    end
+
+    def postcode_known(record)
+      if record.postcode_full.nil?
+        record.la.nil? ? nil : 0 # Assumes we selected No in the form since the LA is present
+      else
+        1
+      end
+    end
+
+    def update_reason(xml_doc)
+      return if meta_field_value(xml_doc, "form-name").include?("Sales")
+
+      old_id = meta_field_value(xml_doc, "document-id")
+      record = LettingsLog.find_by(old_id:)
+
+      if record.present?
+        if record.reason.present?
+          @logger.info("lettings log #{record.id} has a value for reason, skipping update")
+        else
+          reason = unsafe_string_as_integer(xml_doc, "Q9a")
+          reasonother = string_or_nil(xml_doc, "Q9aa")
+          if reason == 20 && reasonother.blank?
+            @logger.info("lettings log #{record.id}'s reason is other but other reason is not provided, skipping update")
+          else
+            record.update!(reason:, reasonother:, values_updated_at: Time.zone.now)
+            @logger.info("lettings log #{record.id}'s reason value has been set to #{reason}")
+            @logger.info("lettings log #{record.id}'s reasonother value has been set to #{reasonother}") if record.reasonother.present?
+          end
+        end
+      else
+        @logger.warn("lettings log with old id #{old_id} not found")
+      end
+    end
   end
 end

app/services/imports/lettings_logs_import_service.rb

@@ -61,8 +61,8 @@ module Imports
       # Required fields for status complete or logic to work
       # Note: order matters when we derive from previous values (attributes parameter)
       attributes["startdate"] = compose_date(xml_doc, "DAY", "MONTH", "YEAR")
-      attributes["owning_organisation_id"] = find_organisation_id(xml_doc, "OWNINGORGID")
-      attributes["managing_organisation_id"] = find_organisation_id(xml_doc, "MANINGORGID")
+      attributes["owning_organisation_id"] = find_organisation_id(xml_doc, "owner-institution-id")
+      attributes["managing_organisation_id"] = find_organisation_id(xml_doc, "managing-institution-id")
       attributes["creation_method"] = creation_method(xml_doc)
       attributes["joint"] = unsafe_string_as_integer(xml_doc, "joint")
       attributes["startertenancy"] = unsafe_string_as_integer(xml_doc, "_2a")
@@ -76,14 +76,19 @@ module Imports
       attributes["irproduct_other"] = string_or_nil(xml_doc, "IRProductOther")
       attributes["rent_type"] = rent_type(xml_doc, attributes["lar"], attributes["irproduct"])
       attributes["hhmemb"] = household_members(xml_doc, previous_status)
+
+      people_indexes = people_with_details_ids(xml_doc)
+      available_people_indexes = people_indexes + (people_indexes.max + 1..8).to_a
       (1..8).each do |index|
-        attributes["age#{index}"] = safe_string_as_integer(xml_doc, "P#{index}Age")
-        attributes["age#{index}_known"] = age_known(xml_doc, index, attributes["hhmemb"])
-        attributes["sex#{index}"] = sex(xml_doc, index)
-        attributes["ecstat#{index}"] = unsafe_string_as_integer(xml_doc, "P#{index}Eco")
+        person_index = available_people_indexes[index - 1]
+        attributes["age#{index}"] = safe_string_as_integer(xml_doc, "P#{person_index}Age")
+        attributes["age#{index}_known"] = age_known(xml_doc, index, person_index, attributes["hhmemb"])
+        attributes["sex#{index}"] = sex(xml_doc, person_index)
+        attributes["ecstat#{index}"] = unsafe_string_as_integer(xml_doc, "P#{person_index}Eco")
       end
       (2..8).each do |index|
-        attributes["relat#{index}"] = relat(xml_doc, index)
+        person_index = available_people_indexes[index - 1]
+        attributes["relat#{index}"] = relat(xml_doc, person_index)
         attributes["details_known_#{index}"] = details_known(index, attributes)
 
         # Trips validation
@@ -142,12 +147,6 @@ module Imports
       attributes["rp_hardship"] = unsafe_string_as_integer(xml_doc, "Q14b4").present? ? 1 : nil
       attributes["rp_dontknow"] = unsafe_string_as_integer(xml_doc, "Q14b5").present? ? 1 : nil
 
-      # Trips validation
-      if attributes["homeless"] == 1 && attributes["rp_homeless"] == 1
-        attributes["homeless"] = nil
-        attributes["rp_homeless"] = nil
-      end
-
       attributes["cbl"] = allocation_system(unsafe_string_as_integer(xml_doc, "Q15CBL"))
       attributes["chr"] = allocation_system(unsafe_string_as_integer(xml_doc, "Q15CHR"))
       attributes["cap"] = allocation_system(unsafe_string_as_integer(xml_doc, "Q15CAP"))
@@ -174,7 +173,7 @@ module Imports
                                      0
                                    end
 
-      attributes["offered"] = safe_string_as_integer(xml_doc, "Q20")
+      attributes["offered"] = safe_string_as_decimal(xml_doc, "Q20")
       attributes["propcode"] = string_or_nil(xml_doc, "Q21a")
       attributes["beds"] = safe_string_as_integer(xml_doc, "Q22")
       attributes["unittype_gn"] = unsafe_string_as_integer(xml_doc, "Q23")
@@ -217,9 +216,6 @@ module Imports
 
         schemes = Scheme.where(old_visible_id: scheme_old_visible_id, owning_organisation_id: attributes["owning_organisation_id"])
         location = Location.find_by(old_visible_id: location_old_visible_id, scheme: schemes)
-        if location.nil? && [location_old_visible_id, scheme_old_visible_id].all?(&:present?) && previous_status != "saved"
-          raise "No matching location for scheme #{scheme_old_visible_id} and location #{location_old_visible_id} (visible IDs)"
-        end
 
         if location.present?
           # Set the scheme via location, because the scheme old visible ID can be duplicated at import
@@ -246,14 +242,40 @@ module Imports
       attributes["rent_value_check"] = 0
       attributes["net_income_value_check"] = 0
       attributes["carehome_charges_value_check"] = 0
+      attributes["referral_value_check"] = 0
+      attributes["scharge_value_check"] = 0
+      attributes["pscharge_value_check"] = 0
+      attributes["supcharg_value_check"] = 0
 
       # Sets the log creator
       owner_id = meta_field_value(xml_doc, "owner-user-id").strip
       if owner_id.present?
         user = LegacyUser.find_by(old_user_id: owner_id)&.user
+        if user.blank? || (user.organisation_id != attributes["managing_organisation_id"] && user.organisation_id != attributes["owning_organisation_id"])
+          if user.blank?
+            @logger.error("Lettings log '#{attributes['old_id']}' belongs to legacy user with owner-user-id: '#{owner_id}' which cannot be found. Assigning log to 'Unassigned' user.")
+          else
+            @logger.error("Lettings log '#{attributes['old_id']}' belongs to legacy user with owner-user-id: '#{owner_id}' which belongs to a different organisation. Assigning log to 'Unassigned' user.")
+          end
+          if User.find_by(name: "Unassigned", organisation_id: attributes["managing_organisation_id"])
+            user = User.find_by(name: "Unassigned", organisation_id: attributes["managing_organisation_id"])
+          else
+            user = User.new(
+              name: "Unassigned",
+              organisation_id: attributes["managing_organisation_id"],
+              is_dpo: false,
+              encrypted_password: SecureRandom.hex(10),
+              email: SecureRandom.uuid,
+              confirmed_at: Time.zone.now,
+              active: false,
+            )
+            user.save!(validate: false)
+          end
+        end
 
         attributes["created_by"] = user
       end
+      attributes["values_updated_at"] = Time.zone.now
 
       apply_date_consistency!(attributes)
       apply_household_consistency!(attributes)
@@ -299,11 +321,12 @@ module Imports
         %i[chcharge out_of_range] => %w[chcharge],
         %i[referral internal_transfer_non_social_housing] => %w[referral],
         %i[referral internal_transfer_fixed_or_lifetime] => %w[referral],
-        %i[tenancylength tenancylength_invalid] => %w[tenancylength tenancy],
+        %i[tenancylength tenancylength_invalid] => %w[tenancylength],
         %i[prevten over_25_foster_care] => %w[prevten age1],
         %i[prevten non_temp_accommodation] => %w[prevten rsnvac],
         %i[joint not_joint_tenancy] => %w[joint],
         %i[offered outside_the_range] => %w[offered],
+        %i[offered not_integer] => %w[offered],
         %i[earnings over_hard_max] => %w[ecstat1],
         %i[tshortfall no_outstanding_charges] => %w[tshortfall hbrentshortfall],
         %i[beds outside_the_range] => %w[beds],
@@ -382,7 +405,7 @@ module Imports
     end
 
     def fields_not_present_in_softwire_data
-      %w[majorrepairs illness_type_0 tshortfall_known pregnancy_value_check retirement_value_check rent_value_check net_income_value_check major_repairs_date_value_check void_date_value_check carehome_charges_value_check housingneeds_type housingneeds_other created_by uprn_known uprn_confirmed]
+      %w[majorrepairs illness_type_0 tshortfall_known pregnancy_value_check retirement_value_check rent_value_check net_income_value_check major_repairs_date_value_check void_date_value_check carehome_charges_value_check referral_value_check scharge_value_check pscharge_value_check supcharg_value_check housingneeds_type housingneeds_other created_by uprn_known uprn_confirmed]
     end
 
     def check_status_completed(lettings_log, previous_status)
@@ -435,10 +458,10 @@ module Imports
       end
     end
 
-    def age_known(xml_doc, index, hhmemb)
+    def age_known(xml_doc, index, person_index, hhmemb)
       return nil if hhmemb.present? && index > hhmemb
 
-      age_refused = string_or_nil(xml_doc, "P#{index}AR")
+      age_refused = string_or_nil(xml_doc, "P#{person_index}AR")
       if age_refused.present?
         if age_refused.casecmp?("AGE_REFUSED") || age_refused.casecmp?("No")
           return 1 # No
@@ -563,6 +586,15 @@ module Imports
       ((2..8).map { |x| string_or_nil(xml_doc, "P#{x}Rel") } + [string_or_nil(xml_doc, "P1Sex")]).compact
     end
 
+    def people_with_details_ids(xml_doc)
+      [1] + (2..8).select do |x|
+        string_or_nil(xml_doc, "P#{x}Rel").present? ||
+          string_or_nil(xml_doc, "P#{x}Sex").present? ||
+          string_or_nil(xml_doc, "P#{x}Age").present? ||
+          string_or_nil(xml_doc, "P#{x}Eco").present?
+      end
+    end
+
     def tshortfall_known?(xml_doc, attributes)
       if attributes["tshortfall"].blank? && attributes["hbrentshortfall"] == 1 && overridden?(xml_doc, "xmlns", "Q18dyes")
         1

app/services/imports/logs_import_service.rb

@@ -31,9 +31,9 @@ module Imports
     end
 
     def find_organisation_id(xml_doc, id_field)
-      old_visible_id = string_or_nil(xml_doc, id_field)
-      organisation = Organisation.find_by(old_visible_id:)
-      raise "Organisation not found with legacy ID #{old_visible_id}" if organisation.nil?
+      old_org_id = meta_field_value(xml_doc, id_field)&.strip.presence
+      organisation = Organisation.find_by(old_org_id:)
+      raise "Organisation not found with old org ID #{old_org_id}" if organisation.nil?
 
       organisation.id
     end
@@ -72,7 +72,7 @@ module Imports
       if str.nil?
         nil
       else
-        BigDecimal(str, exception: false)
+        BigDecimal(str, exception: false)&.round(2)
       end
     end
 
@@ -106,7 +106,7 @@ module Imports
         "F"
       when "Other", "Non-binary"
         "X"
-      when "Refused"
+      when "Refused", "Person prefers not to say", "Buyer prefers not to say"
         "R"
       end
     end
@@ -120,7 +120,7 @@ module Imports
         "P"
       when "Other", "Non-binary"
         "X"
-      when "Refused", "Buyer prefers not to say"
+      when "Refused", "Person prefers not to say", "Buyer prefers not to say"
         "R"
       end
     end

app/services/imports/sales_logs_field_import_service.rb

@@ -45,7 +45,7 @@ module Imports
         if record.owning_organisation_id.present?
           @logger.info("sales log #{record.id} has a value for owning_organisation_id, skipping update")
         else
-          owning_organisation_id = find_organisation_id(xml_doc, "OWNINGORGID")
+          owning_organisation_id = find_organisation_id(xml_doc, "owner-institution-id")
           record.update!(owning_organisation_id:)
           @logger.info("sales log #{record.id}'s owning_organisation_id value has been set to #{owning_organisation_id}")
         end

app/services/imports/sales_logs_import_service.rb

@@ -1,5 +1,7 @@
 module Imports
   class SalesLogsImportService < LogsImportService
+    include CollectionTimeHelper
+
     def initialize(storage_service, logger = Rails.logger, allow_updates: false)
       @logs_with_discrepancies = Set.new
       @logs_overridden = Set.new
@@ -15,7 +17,7 @@ module Imports
     def create_log(xml_doc)
       # only import sales logs from 22/23 collection period onwards
       return unless meta_field_value(xml_doc, "form-name").include?("Sales")
-      return unless compose_date(xml_doc, "DAY", "MONTH", "YEAR") >= Time.zone.local(2022, 4, 1)
+      return unless (compose_date(xml_doc, "DAY", "MONTH", "YEAR") || Time.zone.parse(field_value(xml_doc, "xmlns", "CompletionDate"))) >= Time.zone.local(2022, 4, 1)
 
       attributes = {}
 
@@ -24,8 +26,8 @@ module Imports
       # Required fields for status complete or logic to work
       # Note: order matters when we derive from previous values (attributes parameter)
 
-      attributes["saledate"] = compose_date(xml_doc, "DAY", "MONTH", "YEAR")
-      attributes["owning_organisation_id"] = find_organisation_id(xml_doc, "OWNINGORGID")
+      attributes["saledate"] = compose_date(xml_doc, "DAY", "MONTH", "YEAR") || Time.zone.parse(field_value(xml_doc, "xmlns", "CompletionDate"))
+      attributes["owning_organisation_id"] = find_organisation_id(xml_doc, "owner-institution-id")
       attributes["type"] = unsafe_string_as_integer(xml_doc, "DerSaleType")
       attributes["old_id"] = meta_field_value(xml_doc, "document-id")
       attributes["old_form_id"] = safe_string_as_integer(xml_doc, "Form")
@@ -66,11 +68,14 @@ module Imports
       attributes["la"] = string_or_nil(xml_doc, "Q14ONSLACode")
       attributes["income1"] = safe_string_as_integer(xml_doc, "Q2Person1Income")
       attributes["income1nk"] = income_known(unsafe_string_as_integer(xml_doc, "P1IncKnown"))
+      attributes["income1nk"] ||= 0 if attributes["income1"].present?
       attributes["inc1mort"] = unsafe_string_as_integer(xml_doc, "Q2Person1Mortgage")
       attributes["income2"] = safe_string_as_integer(xml_doc, "Q2Person2Income")
       attributes["income2nk"] = income_known(unsafe_string_as_integer(xml_doc, "P2IncKnown"))
+      attributes["income2nk"] ||= 0 if attributes["income2"].present?
       attributes["savings"] = safe_string_as_integer(xml_doc, "Q3Savings")&.round(-1)
       attributes["savingsnk"] = savings_known(xml_doc)
+      attributes["savingsnk"] ||= 0 if attributes["savings"].present?
       attributes["prevown"] = unsafe_string_as_integer(xml_doc, "Q4PrevOwnedProperty")
       attributes["mortgage"] = safe_string_as_decimal(xml_doc, "CALCMORT")
       attributes["inc2mort"] = unsafe_string_as_integer(xml_doc, "Q2Person2MortApplication")
@@ -124,7 +129,7 @@ module Imports
       attributes["mortgagelenderother"] = mortgage_lender_other(xml_doc, attributes)
       attributes["postcode_full"] = parse_postcode(string_or_nil(xml_doc, "Q14Postcode"))
       attributes["pcodenk"] = 0 if attributes["postcode_full"].present? # known if given
-      attributes["soctenant"] = 0 if attributes["ownershipsch"] == 1
+      attributes["soctenant"] = 0 if set_soctenant_fields?(attributes)
 
       attributes["previous_la_known"] = 1 if attributes["prevloc"].present?
       if attributes["la"].present?
@@ -152,6 +157,10 @@ module Imports
       attributes["discounted_sale_value_check"] = 0
       attributes["buyer_livein_value_check"] = 0
       attributes["percentage_discount_value_check"] = 0
+      attributes["hodate_check"] = 0
+      attributes["saledate_check"] = 0
+      attributes["combined_income_value_check"] = 0
+      attributes["stairowned_value_check"] = 0
 
       # 2023/34 attributes
       attributes["address_line1"] = string_or_nil(xml_doc, "AddressLine1")
@@ -179,8 +188,30 @@ module Imports
       if owner_id.present?
         user = LegacyUser.find_by(old_user_id: owner_id)&.user
 
+        if user.blank? || user.organisation_id != attributes["owning_organisation_id"]
+          if user.blank?
+            @logger.error("Sales log '#{attributes['old_id']}' belongs to legacy user with owner-user-id: '#{owner_id}' which cannot be found. Assigning log to 'Unassigned' user.")
+          else
+            @logger.error("Sales log '#{attributes['old_id']}' belongs to legacy user with owner-user-id: '#{owner_id}' which belongs to a different organisation. Assigning log to 'Unassigned' user.")
+          end
+          if User.find_by(name: "Unassigned", organisation_id: attributes["owning_organisation_id"])
+            user = User.find_by(name: "Unassigned", organisation_id: attributes["owning_organisation_id"])
+          else
+            user = User.new(
+              name: "Unassigned",
+              organisation_id: attributes["owning_organisation_id"],
+              is_dpo: false,
+              encrypted_password: SecureRandom.hex(10),
+              email: SecureRandom.uuid,
+              confirmed_at: Time.zone.now,
+              active: false,
+            )
+            user.save!(validate: false)
+          end
+        end
         attributes["created_by"] = user
       end
+      attributes["values_updated_at"] = Time.zone.now
 
       set_default_values(attributes) if previous_status.include?("submitted")
       sales_log = save_sales_log(attributes, previous_status)
@@ -224,15 +255,20 @@ module Imports
         %i[postcode_full postcodes_not_matching] => %w[ppcodenk ppostcode_full],
         %i[exdate over_a_year_from_saledate] => %w[exdate],
         %i[income1 over_hard_max_for_outside_london] => %w[income1],
+        %i[income1 over_combined_hard_max_for_outside_london] => %w[income1 income2],
         %i[income1 over_hard_max_for_london] => %w[income1],
+        %i[income1 over_combined_hard_max_for_london] => %w[income1 income2],
         %i[income2 over_hard_max_for_outside_london] => %w[income2],
+        %i[income2 over_combined_hard_max_for_outside_london] => %w[income1 income2],
         %i[income2 over_hard_max_for_london] => %w[income2],
+        %i[income2 over_combined_hard_max_for_london] => %w[income1 income2],
         %i[equity over_max] => %w[equity],
         %i[equity under_min] => %w[equity],
         %i[mscharge under_min] => %w[mscharge has_mscharge],
         %i[mortgage cannot_be_0] => %w[mortgage],
         %i[frombeds outside_the_range] => %w[frombeds],
         %i[age1 outside_the_range] => %w[age1 age1_known],
+        %i[proplen outside_the_range] => %w[proplen],
       }
 
       errors.each do |(error, fields)|
@@ -308,6 +344,8 @@ module Imports
          discounted_sale_value_check
          buyer_livein_value_check
          percentage_discount_value_check
+         combined_income_value_check
+         stairowned_value_check
          uprn_known
          uprn_confirmed]
     end
@@ -551,6 +589,12 @@ module Imports
       end
     end
 
+    def set_soctenant_fields?(attributes)
+      return false if attributes["ownershipsch"] != 1
+
+      %w[socprevten frombeds fromprop].any? { |field| attributes[field].present? } || collection_start_year_for_date(attributes["saledate"]) < 2023
+    end
+
     def set_default_values(attributes)
       attributes["armedforcesspouse"] ||= 7
       attributes["hhregres"] ||= 8
@@ -567,8 +611,8 @@ module Imports
       attributes["pcodenk"] ||= 1
       attributes["prevten"] ||= 0
       attributes["extrabor"] ||= 3 if attributes["mortgageused"] == 1
-      attributes["socprevten"] ||= 10 if attributes["ownershipsch"] == 1
-      attributes["fromprop"] ||= 0 if attributes["ownershipsch"] == 1
+      attributes["socprevten"] ||= 10 if set_soctenant_fields?(attributes)
+      attributes["fromprop"] ||= 0 if set_soctenant_fields?(attributes)
       attributes["mortgagelender"] ||= 0 if attributes["mortgageused"] == 1
 
       # buyer 1 characteristics

app/services/imports/scheme_import_service.rb

@@ -54,7 +54,7 @@ module Imports
 
     def find_owning_organisation_id(old_org_id)
       organisation = Organisation.find_by(old_org_id:)
-      raise "Organisation not found with legacy ID #{old_org_id}" if organisation.nil?
+      raise "Organisation not found with old org ID #{old_org_id}" if organisation.nil?
 
       organisation.id
     end

app/services/imports/user_import_service.rb

@@ -39,9 +39,15 @@ module Imports
         user.active = user_field_value(xml_document, "active")
 
         user.skip_confirmation_notification!
-        user.save!
-        user.legacy_users.create!(old_user_id:)
-        user
+
+        begin
+          user.save!
+          user.legacy_users.create!(old_user_id:)
+          user
+        rescue ActiveRecord::RecordInvalid => e
+          @logger.error(e.message)
+          @logger.error("Could not save user with email: #{email}")
+        end
       end
     end
 

config/forms/2022_2023.json

@@ -6228,8 +6228,30 @@
                     },
                     "42": {
                       "value": "End of fixed term tenancy (no fault)"
+                    },
+                    "20": {
+                      "value": "Other"
+                    },
+                    "47": {
+                      "value":"Tenant prefers not to say"
+                    },
+                    "divider": {
+                      "value": true
+                    },
+                    "28": {
+                      "value": "Don’t know"
                     }
+                  },
+                  "conditional_for": {
+                    "reasonother": [
+                      20
+                    ]
                   }
+                },
+                "reasonother": {
+                  "header": "What is the reason?",
+                  "hint_text": "",
+                  "type": "text"
                 }
               },
               "depends_on": [
@@ -7035,6 +7057,9 @@
                     "3": {
                       "value": "Nominated by a local housing authority"
                     },
+                    "4" : { 
+                      "value" : "Referred by local authority housing department" 
+                    },
                     "8": {
                       "value": "Re-located through official housing mobility scheme"
                     },
@@ -7059,6 +7084,9 @@
                     "13": {
                       "value": "Youth offending team"
                     },
+                    "17": {
+                      "value": "Children’s Social Care"
+                    },
                     "16": {
                       "value": "Other"
                     }
@@ -7090,7 +7118,7 @@
                       "value": "Tenant applied directly (no referral)"
                     },
                     "3": {
-                      "value": "Referred by local authority housing department"
+                      "value": "Nominated by a local housing authority"
                     },
                     "8": {
                       "value": "Re-located through official housing mobility scheme"
@@ -7116,6 +7144,9 @@
                     "13": {
                       "value": "Youth offending team"
                     },
+                    "17": {
+                      "value": "Children’s Social Care"
+                    },
                     "16": {
                       "value": "Other"
                     }
@@ -7176,6 +7207,9 @@
                     "13": {
                       "value": "Youth offending team"
                     },
+                    "17": {
+                      "value": "Children’s Social Care"
+                    },
                     "16": {
                       "value": "Other"
                     }
@@ -7189,6 +7223,46 @@
                   "renewal": 0
                 }
               ]
+            },
+            "referral_value_check": {
+              "depends_on": [
+                {
+                  "la_referral_for_general_needs?": true
+                }
+              ],
+              "title_text": {
+                "translation": "soft_validations.referral.title_text"
+              },
+              "informative_text": {
+                "translation": "soft_validations.referral.hint_text",
+                "arguments": []
+              },
+              "questions": {
+                "referral_value_check": {
+                  "check_answer_label": "Referral confirmation",
+                  "hidden_in_check_answers": {
+                    "depends_on": [
+                      {
+                        "referral_value_check": 0
+                      },
+                      {
+                        "referral_value_check": 1
+                      }
+                    ]
+                  },
+                  "header": "Are you sure?",
+                  "type": "interruption_screen",
+                  "answer_options": {
+                    "0": {
+                      "value": "Yes"
+                    },
+                    "1": {
+                      "value": "No"
+                    }
+                  }
+                }
+              },
+              "interruption_screen_question_ids": ["needstype", "referral"]
             }
           }
         }
@@ -8290,6 +8364,141 @@
               },
               "interruption_screen_question_ids": ["brent", "startdate", "la", "beds", "rent_type", "needstype"]
             },
+            "scharge_value_check": {
+              "depends_on": [
+                {
+                  "scharge_over_soft_max?": true
+                }
+              ],
+              "title_text": {
+                "translation": "soft_validations.scharge.over_soft_max_title",
+                "arguments": [
+                  {
+                    "key": "scharge",
+                    "label": true,
+                    "i18n_template": "scharge"
+                  }
+                ]
+              },
+              "informative_text": {},
+              "questions": {
+                "scharge_value_check": {
+                  "check_answer_label": "Service charge confirmation",
+                  "hidden_in_check_answers": {
+                    "depends_on": [
+                      {
+                        "scharge_value_check": 0
+                      },
+                      {
+                        "scharge_value_check": 1
+                      }
+                    ]
+                  },
+                  "header": "Are you sure?",
+                  "hint_text": "This is higher than we would expect. Check:<ul class=\"govuk-body-l app-panel--interruption\"><li>the decimal point</li><li>the frequency, for example every week or every calendar month</li><li>the needs type</li></ul>",
+                  "type": "interruption_screen",
+                  "answer_options": {
+                    "0": {
+                      "value": "Yes"
+                    },
+                    "1": {
+                      "value": "No"
+                    }
+                  }
+                }
+              },
+              "interruption_screen_question_ids": ["period", "needstype", "scharge"]
+            },
+            "pscharge_value_check": {
+              "depends_on": [
+                {
+                  "pscharge_over_soft_max?": true
+                }
+              ],
+              "title_text": {
+                "translation": "soft_validations.pscharge.over_soft_max_title",
+                "arguments": [
+                  {
+                    "key": "pscharge",
+                    "label": true,
+                    "i18n_template": "pscharge"
+                  }
+                ]
+              },
+              "informative_text": {},
+              "questions": {
+                "pscharge_value_check": {
+                  "check_answer_label": "Personal service charge confirmation",
+                  "hidden_in_check_answers": {
+                    "depends_on": [
+                      {
+                        "pscharge_value_check": 0
+                      },
+                      {
+                        "pscharge_value_check": 1
+                      }
+                    ]
+                  },
+                  "header": "Are you sure?",
+                  "hint_text": "This is higher than we would expect. Check:<ul class=\"govuk-body-l app-panel--interruption\"><li>the decimal point</li><li>the frequency, for example every week or every calendar month</li><li>the needs type</li></ul>",
+                  "type": "interruption_screen",
+                  "answer_options": {
+                    "0": {
+                      "value": "Yes"
+                    },
+                    "1": {
+                      "value": "No"
+                    }
+                  }
+                }
+              },
+              "interruption_screen_question_ids": ["period", "needstype", "pscharge"]
+            },
+            "supcharg_value_check": {
+              "depends_on": [
+                {
+                  "supcharg_over_soft_max?": true
+                }
+              ],
+              "title_text": {
+                "translation": "soft_validations.supcharg.over_soft_max_title",
+                "arguments": [
+                  {
+                    "key": "supcharg",
+                    "label": true,
+                    "i18n_template": "supcharg"
+                  }
+                ]
+              },
+              "informative_text": {},
+              "questions": {
+                "supcharg_value_check": {
+                  "check_answer_label": "Personal service charge confirmation",
+                  "hidden_in_check_answers": {
+                    "depends_on": [
+                      {
+                        "supcharg_value_check": 0
+                      },
+                      {
+                        "supcharg_value_check": 1
+                      }
+                    ]
+                  },
+                  "header": "Are you sure?",
+                  "hint_text": "This is higher than we would expect. Check:<ul class=\"govuk-body-l app-panel--interruption\"><li>the decimal point</li><li>the frequency, for example every week or every calendar month</li><li>the needs type</li></ul>",
+                  "type": "interruption_screen",
+                  "answer_options": {
+                    "0": {
+                      "value": "Yes"
+                    },
+                    "1": {
+                      "value": "No"
+                    }
+                  }
+                }
+              },
+              "interruption_screen_question_ids": ["period", "needstype", "supcharg"]
+            },
             "outstanding": {
               "header": "",
               "description": "",

config/locales/en.yml

@@ -369,25 +369,25 @@ en:
         less_than_shortfall: "Enter an amount that is more than the shortfall in basic rent"
         scharge:
           private_registered_provider:
-            general_needs: "Enter a value for the service charge between £0 and £155 per week if the landlord is a private registered provider and it is a general needs letting"
-            supported_housing: "Enter a value for the service charge between £0 and £480 per week if the landlord is a private registered provider and it is a supported housing letting"
+            general_needs: "Enter a value for the service charge between £0 and £800 per week if the landlord is a private registered provider and it is a general needs letting"
+            supported_housing: "Enter a value for the service charge between £0 and £800 per week if the landlord is a private registered provider and it is a supported housing letting"
           local_authority:
-            general_needs: "Enter a value for the service charge between £0 and £155 per week if the landlord is a local authority and it is a general needs letting"
-            supported_housing: "Enter a value for the service charge between £0 and £365 per week if the landlord is a local authority and it is a supported housing letting"
+            general_needs: "Enter a value for the service charge between £0 and £500 per week if the landlord is a local authority and it is a general needs letting"
+            supported_housing: "Enter a value for the service charge between £0 and £500 per week if the landlord is a local authority and it is a supported housing letting"
         pscharge:
           private_registered_provider:
-            general_needs: "Enter a value for the personal service charge between £0 and £30 per week if the landlord is a private registered provider and it is a general needs letting"
-            supported_housing: "Enter a value for the personal service charge between £0 and £200 per week if the landlord is a private registered provider and it is a supported housing letting"
+            general_needs: "Enter a value for the personal service charge between £0 and £700 per week if the landlord is a private registered provider and it is a general needs letting"
+            supported_housing: "Enter a value for the personal service charge between £0 and £700 per week if the landlord is a private registered provider and it is a supported housing letting"
           local_authority:
-            general_needs: "Enter a value for the personal service charge between £0 and £35 per week if the landlord is a local authority and it is a general needs letting"
-            supported_housing: "Enter a value for the personal service charge between £0 and £75 per week if the landlord is a local authority and it is a supported housing letting"
+            general_needs: "Enter a value for the personal service charge between £0 and £200 per week if the landlord is a local authority and it is a general needs letting"
+            supported_housing: "Enter a value for the personal service charge between £0 and £200 per week if the landlord is a local authority and it is a supported housing letting"
         supcharg:
           private_registered_provider:
-            general_needs: "Enter a value for the support charge between £0 and £40 per week if the landlord is a private registered provider and it is a general needs letting"
-            supported_housing: "Enter a value for the support charge between £0 and £465 per week if the landlord is a private registered provider and it is a supported housing letting"
+            general_needs: "Enter a value for the support charge between £0 and £800 per week if the landlord is a private registered provider and it is a general needs letting"
+            supported_housing: "Enter a value for the support charge between £0 and £800 per week if the landlord is a private registered provider and it is a supported housing letting"
           local_authority:
-            general_needs: "Enter a value for the support charge between £0 and £60 per week if the landlord is a local authority and it is a general needs letting"
-            supported_housing: "Enter a value for the support charge between £0 and £120 per week if the landlord is a local authority and it is a supported housing letting"
+            general_needs: "Enter a value for the support charge between £0 and £200 per week if the landlord is a local authority and it is a general needs letting"
+            supported_housing: "Enter a value for the support charge between £0 and £200 per week if the landlord is a local authority and it is a supported housing letting"
       ecstat:
         over_hard_max: "Net income of %{hard_max} per week is too high given the tenant’s working situation"
       brent:
@@ -441,8 +441,6 @@ en:
       mortgage: "Mortgage value cannot be £0 if a mortgage was used for the purchase of this property"
 
     household:
-      reasonpref:
-        not_homeless: "Answer cannot be ‘homeless or about to lose their home’ as the tenant was not homeless immediately prior to this letting"
       reasonable_preference_reason:
         reason_required: "Enter a reason if you've answered 'yes' to reasonable preference"
         reason_not_required: "Do not enter a reason if you've answered 'no' to reasonable preference"
@@ -708,6 +706,19 @@ Make sure these answers are correct."
     mortgage:
       title_text: "You told us that the mortgage amount is %{mortgage}"
       hint_text: "This is more than 5 times the income, which is higher than we would expect."
+    referral:
+      title_text: "Are you sure?"
+      hint_text: "This is a general needs log, and this referral type is for supported housing."
+    scharge:
+      over_soft_max_title: "You told us the service charge is %{scharge}"
+    pscharge:
+      over_soft_max_title: "You told us the personal service charge is %{pscharge}"
+    supcharg:
+      over_soft_max_title: "You told us the support charge is %{supcharg}"
+    charges:
+      informative_text: "This is higher than we would expect."
+      hint_text: "Check the following:<ul class=\"govuk-body-l app-panel--interruption\"><li>the decimal point</li><li>the frequency, for example every week or every calendar month</li><li>the needs type</li></ul>"
+  
 
   devise:
     email:

db/migrate/20230828145454_add_migrated_on_fields.rb

@@ -0,0 +1,6 @@
+class AddMigratedOnFields < ActiveRecord::Migration[7.0]
+  def change
+    add_column :lettings_logs, :values_updated_at, :datetime
+    add_column :sales_logs, :values_updated_at, :datetime
+  end
+end

db/migrate/20230912115233_add_referral_value_check.rb

@@ -0,0 +1,5 @@
+class AddReferralValueCheck < ActiveRecord::Migration[7.0]
+  def change
+    add_column :lettings_logs, :referral_value_check, :integer
+  end
+end

db/migrate/20230913093443_add_charges_value_checks.rb

@@ -0,0 +1,9 @@
+class AddChargesValueChecks < ActiveRecord::Migration[7.0]
+  def change
+    change_table :lettings_logs, bulk: true do |t|
+      t.column :supcharg_value_check, :integer
+      t.column :scharge_value_check, :integer
+      t.column :pscharge_value_check, :integer
+    end
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2023_08_16_083950) do
+ActiveRecord::Schema[7.0].define(version: 2023_09_13_093443) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
@@ -297,6 +297,11 @@ ActiveRecord::Schema[7.0].define(version: 2023_08_16_083950) do
     t.integer "status_cache", default: 0, null: false
     t.datetime "discarded_at"
     t.integer "creation_method", default: 1
+    t.datetime "values_updated_at"
+    t.integer "referral_value_check"
+    t.integer "supcharg_value_check"
+    t.integer "scharge_value_check"
+    t.integer "pscharge_value_check"
     t.index ["bulk_upload_id"], name: "index_lettings_logs_on_bulk_upload_id"
     t.index ["created_by_id"], name: "index_lettings_logs_on_created_by_id"
     t.index ["location_id"], name: "index_lettings_logs_on_location_id"
@@ -617,6 +622,7 @@ ActiveRecord::Schema[7.0].define(version: 2023_08_16_083950) do
     t.integer "stairowned_value_check"
     t.integer "creation_method", default: 1
     t.integer "old_form_id"
+    t.datetime "values_updated_at"
     t.index ["bulk_upload_id"], name: "index_sales_logs_on_bulk_upload_id"
     t.index ["created_by_id"], name: "index_sales_logs_on_created_by_id"
     t.index ["old_id"], name: "index_sales_logs_on_old_id", unique: true

lib/tasks/data_import_field.rake

@@ -7,7 +7,7 @@ namespace :core do
 
     # We only allow a reduced list of known fields to be updatable
     case field
-    when "tenancycode", "major_repairs", "lettings_allocation", "offered"
+    when "tenancycode", "major_repairs", "lettings_allocation", "offered", "address", "reason"
       s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
       archive_io = s3_service.get_file_io(path)
       archive_service = Storage::ArchiveService.new(archive_io)

lib/tasks/full_import.rake

@@ -117,6 +117,15 @@ namespace :import do
     Imports::ImportReportService.new(s3_service, institutions_csv).create_reports(institutions_csv_name)
   end
 
+  desc "Generate missing answers report"
+  task :generate_missing_answers_report, %i[file_suffix] => :environment do |_task, args|
+    file_suffix = args[:file_suffix]
+    raise "Usage: rake import:generate_reports['file_suffix']" if file_suffix.blank?
+
+    s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
+    Imports::ImportReportService.new(s3_service, nil).generate_missing_answers_report(file_suffix)
+  end
+
   desc "Run import from logs step to end"
   task :logs_onwards, %i[institutions_csv_name] => %i[environment logs trigger_invites generate_reports]
 

lib/tasks/import_address_from_csv.rake

@@ -0,0 +1,42 @@
+namespace :data_import do
+  desc "Import address data from a csv file"
+  task :import_address_from_csv, %i[file_name] => :environment do |_task, args|
+    file_name = args[:file_name]
+
+    raise "Usage: rake data_import:import_address_from_csv['csv_file_name']" if file_name.blank?
+
+    s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
+    addresses_csv = CSV.parse(s3_service.get_file_io(file_name), headers: true)
+
+    addresses_csv.each do |row|
+      lettings_log_id = row[0]
+
+      if lettings_log_id.blank?
+        Rails.logger.info("Lettings log ID not provided for address: #{[row[1], row[2], row[3], row[4]].join(', ')}")
+        next
+      end
+
+      lettings_log = LettingsLog.find_by(id: lettings_log_id)
+      if lettings_log.blank?
+        Rails.logger.info("Could not find a lettings log with id #{lettings_log_id}")
+        next
+      end
+
+      lettings_log.uprn_known = 0
+      lettings_log.uprn = nil
+      lettings_log.uprn_confirmed = nil
+      lettings_log.address_line1 = row[1]
+      lettings_log.address_line2 = row[2]
+      lettings_log.town_or_city = row[3]
+      lettings_log.postcode_full = row[4]
+      lettings_log.postcode_known = lettings_log.postcode_full.present? ? 1 : nil
+      lettings_log.county = nil
+      lettings_log.is_la_inferred = nil
+      lettings_log.la = nil
+      lettings_log.values_updated_at = Time.zone.now
+
+      lettings_log.save!
+      Rails.logger.info("Updated lettings log #{lettings_log_id}, with address: #{[lettings_log.address_line1, lettings_log.address_line2, lettings_log.town_or_city, lettings_log.postcode_full].join(', ')}")
+    end
+  end
+end

spec/fixtures/files/addresses_reimport.csv

@@ -0,0 +1,6 @@
+lettings_log_id,address_line1,address_line2,town_or_city,postcode_full
+{id},address 1,address 2,town,B1 1BB
+{id2},address 3,address 4,city,B1 1BB
+{id3},,,,C1 1CC
+,address 5,address 6,city,D1 1DD
+fake_id,address 7,address 8,city,D1 1DD

spec/fixtures/files/imported_lettings_logs_missing_answers_examples.csv

@@ -0,0 +1,16 @@
+Missing answers,Organisation ID,Log ID,Old Form ID,Old Log ID
+age1_known,{org_id0},{id0},1000,old_id_age1_known_0
+age1_known,{org_id1},{id1},1001,old_id_age1_known_1
+age1_known,{org_id2},{id2},1002,old_id_age1_known_2
+age1_known,{org_id3},{id3},1003,old_id_age1_known_3
+age1_known,{org_id4},{id4},1004,old_id_age1_known_4
+age1_known,{org_id5},{id5},1005,old_id_age1_known_5
+age1_known,{org_id6},{id6},1006,old_id_age1_known_6
+age1_known,{org_id7},{id7},1007,old_id_age1_known_7
+age1_known,{org_id8},{id8},1008,old_id_age1_known_8
+age1_known,{org_id9},{id9},1009,old_id_age1_known_9
+beds,{org_id2_0},{id2_0},2000,old_id_beds_0
+beds,{org_id2_1},{id2_1},2001,old_id_beds_1
+beds,{org_id2_2},{id2_2},2002,old_id_beds_2
+beds,{org_id2_3},{id2_3},2003,old_id_beds_3
+"beds, age1_known",{org_id},{id},300,beds_and_age

spec/fixtures/files/imported_lettings_logs_missing_answers_report.csv

@@ -0,0 +1,4 @@
+Missing answers,Total number of affected logs
+age1_known,11
+beds,4
+"beds, age1_known",1

spec/fixtures/files/lettings_log_csv_export_codes.csv

@@ -1,2 +1,2 @@
-id,status,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,old_id,old_form_id,collection_start_year,owning_organisation_name,managing_organisation_name,needstype,lettype,renewal,startdate,renttype,rent_type_detail,irproduct,irproduct_other,lar,tenancycode,propcode,uprn_known,uprn,uprn_confirmed,address_line1,address_line2,town_or_city,county,postcode_full,is_la_inferred,la_label,la,first_time_property_let_as_social_housing,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,vacdays,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,hhmemb,pregnancy_value_check,age1_known,refused,hhtype,totchild,totelder,totadult,age1,retirement_value_check,sex1,ethnic_group,ethnic,national,ecstat1,details_known_2,relat2,age2_known,age2,sex2,ecstat2,details_known_3,relat3,age3_known,age3,sex3,ecstat3,details_known_4,relat4,age4_known,age4,sex4,ecstat4,details_known_5,relat5,age5_known,age5,sex5,ecstat5,details_known_6,relat6,age6_known,age6,sex6,ecstat6,details_known_7,relat7,age7_known,age7,sex7,ecstat7,details_known_8,relat8,age8_known,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,new_old,homeless,ppcodenk,ppostcode_full,previous_la_known,is_previous_la_inferred,prevloc_label,prevloc,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,letting_allocation_unknown,referral,net_income_known,incref,earnings,incfreq,net_income_value_check,hb,has_benefits,benefits,household_charge,nocharge,period,is_carehome,chcharge,wchchrg,carehome_charges_value_check,brent,wrent,rent_value_check,scharge,wscharge,pscharge,wpschrge,supcharg,wsupchrg,tcharge,wtcharge,hbrentshortfall,tshortfall_known,tshortfall,wtshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_admin_district,location_startdate
-,completed,s.port@jeemayle.com,false,2023-06-26T00:00:00+01:00,,2023-06-26T00:00:00+01:00,1,,,2023,DLUHC,DLUHC,1,7,0,2023-06-26T00:00:00+01:00,2,1,,,,HIJKLMN,ABCDEFG,0,,,fake address,,London,,NW9 5LL,false,Barnet,E09000003,0,2,6,2,2,7,1,1,3,2023-06-24T00:00:00+01:00,,,1,2023-06-25T00:00:00+01:00,,3,1,4,,2,,1,2,,0,0,4,0,0,2,35,,F,0,2,13,0,0,P,0,32,M,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,4,1,2,1,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,2,7,4,,6,2,1,0,TN23 6LZ,1,false,Ashford,E07000105,1,0,1,0,0,0,0,0,1,,2,0,0,68,1,,6,1,1,,0,2,,,,,200.0,100.0,,50.0,25.0,40.0,20.0,35.0,17.5,325.0,162.5,1,0,12.0,6.0,,,,,,,,,,,,,,,,,,,,
+id,status,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,old_id,old_form_id,collection_start_year,owning_organisation_name,managing_organisation_name,needstype,lettype,renewal,startdate,renttype,rent_type_detail,irproduct,irproduct_other,lar,tenancycode,propcode,uprn_known,uprn,uprn_confirmed,address_line1,address_line2,town_or_city,county,postcode_full,is_la_inferred,la_label,la,first_time_property_let_as_social_housing,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,vacdays,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,hhmemb,pregnancy_value_check,age1_known,refused,hhtype,totchild,totelder,totadult,age1,retirement_value_check,sex1,ethnic_group,ethnic,national,ecstat1,details_known_2,relat2,age2_known,age2,sex2,ecstat2,details_known_3,relat3,age3_known,age3,sex3,ecstat3,details_known_4,relat4,age4_known,age4,sex4,ecstat4,details_known_5,relat5,age5_known,age5,sex5,ecstat5,details_known_6,relat6,age6_known,age6,sex6,ecstat6,details_known_7,relat7,age7_known,age7,sex7,ecstat7,details_known_8,relat8,age8_known,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,new_old,homeless,ppcodenk,ppostcode_full,previous_la_known,is_previous_la_inferred,prevloc_label,prevloc,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,letting_allocation_unknown,referral,referral_value_check,net_income_known,incref,earnings,incfreq,net_income_value_check,hb,has_benefits,benefits,household_charge,nocharge,period,is_carehome,chcharge,wchchrg,carehome_charges_value_check,brent,wrent,rent_value_check,scharge,wscharge,pscharge,wpschrge,supcharg,wsupchrg,tcharge,wtcharge,scharge_value_check,pscharge_value_check,supcharg_value_check,hbrentshortfall,tshortfall_known,tshortfall,wtshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_admin_district,location_startdate
+,completed,s.port@jeemayle.com,false,2023-06-26T00:00:00+01:00,,2023-06-26T00:00:00+01:00,1,,,2023,DLUHC,DLUHC,1,7,0,2023-06-26T00:00:00+01:00,2,1,,,,HIJKLMN,ABCDEFG,0,,,fake address,,London,,NW9 5LL,false,Barnet,E09000003,0,2,6,2,2,7,1,1,3,2023-06-24T00:00:00+01:00,,,1,2023-06-25T00:00:00+01:00,,3,1,4,,2,,1,2,,0,0,4,0,0,2,35,,F,0,2,13,0,0,P,0,32,M,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,4,1,2,1,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,2,7,4,,6,2,1,0,TN23 6LZ,1,false,Ashford,E07000105,1,0,1,0,0,0,0,0,1,,2,,0,0,68,1,,6,1,1,,0,2,,,,,200.0,100.0,,50.0,25.0,40.0,20.0,35.0,17.5,325.0,162.5,,,,1,0,12.0,6.0,,,,,,,,,,,,,,,,,,,,

spec/fixtures/files/lettings_log_csv_export_labels.csv

@@ -1,2 +1,2 @@
-id,status,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,old_id,old_form_id,collection_start_year,owning_organisation_name,managing_organisation_name,needstype,lettype,renewal,startdate,renttype,rent_type_detail,irproduct,irproduct_other,lar,tenancycode,propcode,uprn_known,uprn,uprn_confirmed,address_line1,address_line2,town_or_city,county,postcode_full,is_la_inferred,la_label,la,first_time_property_let_as_social_housing,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,vacdays,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,hhmemb,pregnancy_value_check,age1_known,refused,hhtype,totchild,totelder,totadult,age1,retirement_value_check,sex1,ethnic_group,ethnic,national,ecstat1,details_known_2,relat2,age2_known,age2,sex2,ecstat2,details_known_3,relat3,age3_known,age3,sex3,ecstat3,details_known_4,relat4,age4_known,age4,sex4,ecstat4,details_known_5,relat5,age5_known,age5,sex5,ecstat5,details_known_6,relat6,age6_known,age6,sex6,ecstat6,details_known_7,relat7,age7_known,age7,sex7,ecstat7,details_known_8,relat8,age8_known,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,new_old,homeless,ppcodenk,ppostcode_full,previous_la_known,is_previous_la_inferred,prevloc_label,prevloc,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,letting_allocation_unknown,referral,net_income_known,incref,earnings,incfreq,net_income_value_check,hb,has_benefits,benefits,household_charge,nocharge,period,is_carehome,chcharge,wchchrg,carehome_charges_value_check,brent,wrent,rent_value_check,scharge,wscharge,pscharge,wpschrge,supcharg,wsupchrg,tcharge,wtcharge,hbrentshortfall,tshortfall_known,tshortfall,wtshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_admin_district,location_startdate
-,completed,s.port@jeemayle.com,false,2023-06-26T00:00:00+01:00,,2023-06-26T00:00:00+01:00,single log,,,2023,DLUHC,DLUHC,General needs,7,No,2023-06-26T00:00:00+01:00,2,Affordable Rent,,,,HIJKLMN,ABCDEFG,No,,,fake address,,London,,NW9 5LL,No,Barnet,E09000003,No,Affordable rent basis,Tenant abandoned property,2,2,House,Purpose built,Yes,3,2023-06-24T00:00:00+01:00,,,Yes,2023-06-25T00:00:00+01:00,,Don’t know,Yes,Assured Shorthold Tenancy (AST) – Fixed term,,2,,1,2,,Yes,0,4,0,0,2,35,,Female,White,Irish,Tenant prefers not to say,Other,Yes,Partner,Yes,32,Male,Not seeking work,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,Yes – the person is a current or former regular,No – they left up to and including 5 years ago,Yes,No,Yes,Fully wheelchair accessible housing,1,0,0,0,0,0,No,Yes,0,0,1,0,0,0,0,0,0,0,Less than 1 year,1 year but under 2 years,Loss of tied accommodation,,Other supported housing,2,No,Yes,TN23 6LZ,Yes,No,Ashford,E07000105,Yes,0,1,0,0,0,0,0,1,,Tenant applied directly (no referral or nomination),Yes,0,68,Weekly,,Universal Credit housing element,1,All,,0,Every 2 weeks,,,,,200.0,100.0,,50.0,25.0,40.0,20.0,35.0,17.5,325.0,162.5,Yes,Yes,12.0,6.0,,,,,,,,,,,,,,,,,,,,
+id,status,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,old_id,old_form_id,collection_start_year,owning_organisation_name,managing_organisation_name,needstype,lettype,renewal,startdate,renttype,rent_type_detail,irproduct,irproduct_other,lar,tenancycode,propcode,uprn_known,uprn,uprn_confirmed,address_line1,address_line2,town_or_city,county,postcode_full,is_la_inferred,la_label,la,first_time_property_let_as_social_housing,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,vacdays,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,hhmemb,pregnancy_value_check,age1_known,refused,hhtype,totchild,totelder,totadult,age1,retirement_value_check,sex1,ethnic_group,ethnic,national,ecstat1,details_known_2,relat2,age2_known,age2,sex2,ecstat2,details_known_3,relat3,age3_known,age3,sex3,ecstat3,details_known_4,relat4,age4_known,age4,sex4,ecstat4,details_known_5,relat5,age5_known,age5,sex5,ecstat5,details_known_6,relat6,age6_known,age6,sex6,ecstat6,details_known_7,relat7,age7_known,age7,sex7,ecstat7,details_known_8,relat8,age8_known,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,new_old,homeless,ppcodenk,ppostcode_full,previous_la_known,is_previous_la_inferred,prevloc_label,prevloc,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,letting_allocation_unknown,referral,referral_value_check,net_income_known,incref,earnings,incfreq,net_income_value_check,hb,has_benefits,benefits,household_charge,nocharge,period,is_carehome,chcharge,wchchrg,carehome_charges_value_check,brent,wrent,rent_value_check,scharge,wscharge,pscharge,wpschrge,supcharg,wsupchrg,tcharge,wtcharge,scharge_value_check,pscharge_value_check,supcharg_value_check,hbrentshortfall,tshortfall_known,tshortfall,wtshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_admin_district,location_startdate
+,completed,s.port@jeemayle.com,false,2023-06-26T00:00:00+01:00,,2023-06-26T00:00:00+01:00,single log,,,2023,DLUHC,DLUHC,General needs,7,No,2023-06-26T00:00:00+01:00,2,Affordable Rent,,,,HIJKLMN,ABCDEFG,No,,,fake address,,London,,NW9 5LL,No,Barnet,E09000003,No,Affordable rent basis,Tenant abandoned property,2,2,House,Purpose built,Yes,3,2023-06-24T00:00:00+01:00,,,Yes,2023-06-25T00:00:00+01:00,,Don’t know,Yes,Assured Shorthold Tenancy (AST) – Fixed term,,2,,1,2,,Yes,0,4,0,0,2,35,,Female,White,Irish,Tenant prefers not to say,Other,Yes,Partner,Yes,32,Male,Not seeking work,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,Yes – the person is a current or former regular,No – they left up to and including 5 years ago,Yes,No,Yes,Fully wheelchair accessible housing,1,0,0,0,0,0,No,Yes,0,0,1,0,0,0,0,0,0,0,Less than 1 year,1 year but under 2 years,Loss of tied accommodation,,Other supported housing,2,No,Yes,TN23 6LZ,Yes,No,Ashford,E07000105,Yes,0,1,0,0,0,0,0,1,,Tenant applied directly (no referral or nomination),,Yes,0,68,Weekly,,Universal Credit housing element,1,All,,0,Every 2 weeks,,,,,200.0,100.0,,50.0,25.0,40.0,20.0,35.0,17.5,325.0,162.5,,,,Yes,Yes,12.0,6.0,,,,,,,,,,,,,,,,,,,,

spec/fixtures/files/lettings_log_csv_export_non_support_codes.csv

@@ -1,2 +1,2 @@
-id,status,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,collection_start_year,owning_organisation_name,managing_organisation_name,lettype,renewal,startdate,irproduct,irproduct_other,lar,tenancycode,propcode,uprn_known,uprn,address_line1,address_line2,town_or_city,county,postcode_full,la_label,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,refused,age1,sex1,ethnic_group,ethnic,national,ecstat1,relat2,age2,sex2,ecstat2,relat3,age3,sex3,ecstat3,relat4,age4,sex4,ecstat4,relat5,age5,sex5,ecstat5,relat6,age6,sex6,ecstat6,relat7,age7,sex7,ecstat7,relat8,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,homeless,ppcodenk,ppostcode_full,prevloc_label,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,referral,incref,earnings,incfreq,hb,has_benefits,benefits,household_charge,nocharge,period,chcharge,wchchrg,carehome_charges_value_check,brent,scharge,pscharge,supcharg,tcharge,hbrentshortfall,tshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_admin_district,location_startdate
-,completed,choreographer@owtluk.com,false,2023-06-26T00:00:00+01:00,,2023-06-26T00:00:00+01:00,1,2023,DLUHC,DLUHC,7,0,2023-06-26T00:00:00+01:00,,,,HIJKLMN,ABCDEFG,0,,fake address,,London,,NW9 5LL,Barnet,2,6,2,2,7,1,1,3,2023-06-24T00:00:00+01:00,,1,2023-06-25T00:00:00+01:00,,3,1,4,,2,,1,0,35,F,0,2,13,0,P,32,M,6,,,,,,,,,,,,,,,,,,,,,,,,,1,4,1,2,1,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,2,7,4,,6,1,0,TN23 6LZ,Ashford,1,0,1,0,0,0,0,0,1,2,0,68,1,6,1,1,,0,2,,,,200.0,50.0,40.0,35.0,325.0,1,12.0,,,,,,,,,,,,,,,,,,,,
+id,status,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,collection_start_year,owning_organisation_name,managing_organisation_name,lettype,renewal,startdate,irproduct,irproduct_other,lar,tenancycode,propcode,uprn_known,uprn,address_line1,address_line2,town_or_city,county,postcode_full,la_label,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,refused,age1,sex1,ethnic_group,ethnic,national,ecstat1,relat2,age2,sex2,ecstat2,relat3,age3,sex3,ecstat3,relat4,age4,sex4,ecstat4,relat5,age5,sex5,ecstat5,relat6,age6,sex6,ecstat6,relat7,age7,sex7,ecstat7,relat8,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,homeless,ppcodenk,ppostcode_full,prevloc_label,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,referral,referral_value_check,incref,earnings,incfreq,hb,has_benefits,benefits,household_charge,nocharge,period,chcharge,wchchrg,carehome_charges_value_check,brent,scharge,pscharge,supcharg,tcharge,scharge_value_check,pscharge_value_check,supcharg_value_check,hbrentshortfall,tshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_admin_district,location_startdate
+,completed,choreographer@owtluk.com,false,2023-06-26T00:00:00+01:00,,2023-06-26T00:00:00+01:00,1,2023,DLUHC,DLUHC,7,0,2023-06-26T00:00:00+01:00,,,,HIJKLMN,ABCDEFG,0,,fake address,,London,,NW9 5LL,Barnet,2,6,2,2,7,1,1,3,2023-06-24T00:00:00+01:00,,1,2023-06-25T00:00:00+01:00,,3,1,4,,2,,1,0,35,F,0,2,13,0,P,32,M,6,,,,,,,,,,,,,,,,,,,,,,,,,1,4,1,2,1,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,2,7,4,,6,1,0,TN23 6LZ,Ashford,1,0,1,0,0,0,0,0,1,2,,0,68,1,6,1,1,,0,2,,,,200.0,50.0,40.0,35.0,325.0,,,,1,12.0,,,,,,,,,,,,,,,,,,,,

spec/fixtures/files/lettings_log_csv_export_non_support_labels.csv

@@ -1,2 +1,2 @@
-id,status,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,collection_start_year,owning_organisation_name,managing_organisation_name,lettype,renewal,startdate,irproduct,irproduct_other,lar,tenancycode,propcode,uprn_known,uprn,address_line1,address_line2,town_or_city,county,postcode_full,la_label,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,refused,age1,sex1,ethnic_group,ethnic,national,ecstat1,relat2,age2,sex2,ecstat2,relat3,age3,sex3,ecstat3,relat4,age4,sex4,ecstat4,relat5,age5,sex5,ecstat5,relat6,age6,sex6,ecstat6,relat7,age7,sex7,ecstat7,relat8,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,homeless,ppcodenk,ppostcode_full,prevloc_label,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,referral,incref,earnings,incfreq,hb,has_benefits,benefits,household_charge,nocharge,period,chcharge,wchchrg,carehome_charges_value_check,brent,scharge,pscharge,supcharg,tcharge,hbrentshortfall,tshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_admin_district,location_startdate
-,completed,choreographer@owtluk.com,false,2023-06-26T00:00:00+01:00,,2023-06-26T00:00:00+01:00,single log,2023,DLUHC,DLUHC,7,No,2023-06-26T00:00:00+01:00,,,,HIJKLMN,ABCDEFG,No,,fake address,,London,,NW9 5LL,Barnet,Affordable rent basis,Tenant abandoned property,2,2,House,Purpose built,Yes,3,2023-06-24T00:00:00+01:00,,Yes,2023-06-25T00:00:00+01:00,,Don’t know,Yes,Assured Shorthold Tenancy (AST) – Fixed term,,2,,1,0,35,Female,White,Irish,Tenant prefers not to say,Other,Partner,32,Male,Not seeking work,,,,,,,,,,,,,,,,,,,,,,,,,Yes – the person is a current or former regular,No – they left up to and including 5 years ago,Yes,No,Yes,Fully wheelchair accessible housing,1,0,0,0,0,0,No,Yes,0,0,1,0,0,0,0,0,0,0,Less than 1 year,1 year but under 2 years,Loss of tied accommodation,,Other supported housing,No,Yes,TN23 6LZ,Ashford,Yes,0,1,0,0,0,0,0,1,Tenant applied directly (no referral or nomination),0,68,Weekly,Universal Credit housing element,1,All,,0,Every 2 weeks,,,,200.0,50.0,40.0,35.0,325.0,Yes,12.0,,,,,,,,,,,,,,,,,,,,
+id,status,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,collection_start_year,owning_organisation_name,managing_organisation_name,lettype,renewal,startdate,irproduct,irproduct_other,lar,tenancycode,propcode,uprn_known,uprn,address_line1,address_line2,town_or_city,county,postcode_full,la_label,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,refused,age1,sex1,ethnic_group,ethnic,national,ecstat1,relat2,age2,sex2,ecstat2,relat3,age3,sex3,ecstat3,relat4,age4,sex4,ecstat4,relat5,age5,sex5,ecstat5,relat6,age6,sex6,ecstat6,relat7,age7,sex7,ecstat7,relat8,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,homeless,ppcodenk,ppostcode_full,prevloc_label,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,referral,referral_value_check,incref,earnings,incfreq,hb,has_benefits,benefits,household_charge,nocharge,period,chcharge,wchchrg,carehome_charges_value_check,brent,scharge,pscharge,supcharg,tcharge,scharge_value_check,pscharge_value_check,supcharg_value_check,hbrentshortfall,tshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_admin_district,location_startdate
+,completed,choreographer@owtluk.com,false,2023-06-26T00:00:00+01:00,,2023-06-26T00:00:00+01:00,single log,2023,DLUHC,DLUHC,7,No,2023-06-26T00:00:00+01:00,,,,HIJKLMN,ABCDEFG,No,,fake address,,London,,NW9 5LL,Barnet,Affordable rent basis,Tenant abandoned property,2,2,House,Purpose built,Yes,3,2023-06-24T00:00:00+01:00,,Yes,2023-06-25T00:00:00+01:00,,Don’t know,Yes,Assured Shorthold Tenancy (AST) – Fixed term,,2,,1,0,35,Female,White,Irish,Tenant prefers not to say,Other,Partner,32,Male,Not seeking work,,,,,,,,,,,,,,,,,,,,,,,,,Yes – the person is a current or former regular,No – they left up to and including 5 years ago,Yes,No,Yes,Fully wheelchair accessible housing,1,0,0,0,0,0,No,Yes,0,0,1,0,0,0,0,0,0,0,Less than 1 year,1 year but under 2 years,Loss of tied accommodation,,Other supported housing,No,Yes,TN23 6LZ,Ashford,Yes,0,1,0,0,0,0,0,1,Tenant applied directly (no referral or nomination),,0,68,Weekly,Universal Credit housing element,1,All,,0,Every 2 weeks,,,,200.0,50.0,40.0,35.0,325.0,,,,Yes,12.0,,,,,,,,,,,,,,,,,,,,

spec/fixtures/imports/logs/00d2343e-d5fa-4c89-8400-ec3854b0f2b4.xml

@@ -4,7 +4,7 @@
     <meta:document-id>00d2343e-d5fa-4c89-8400-ec3854b0f2b4</meta:document-id>
     <meta:owner-user-id>c3061a2e6ea0b702e6f6210d5c52d2a92612d2aa</meta:owner-user-id>
     <meta:owner-institution-id>7c5bd5fb549c09a2c55d7cb90d7ba84927e64618</meta:owner-institution-id>
-    <meta:managing-institution-id>7c5bd5fb549c09a2c55d7cb90d7ba84927e64618</meta:managing-institution-id>
+    <meta:managing-institution-id>7c5bd5fb549c09z2c55d9cb90d7ba84927e64618</meta:managing-institution-id>
     <meta:created-date>2022-04-14T16:01:30.369241Z</meta:created-date>
     <meta:modified-date>2022-04-14T16:01:30.369241Z</meta:modified-date>
     <meta:status>submitted-valid</meta:status>

spec/fixtures/imports/logs/0b4a68df-30cc-474a-93c0-a56ce8fdad3b.xml

@@ -3,7 +3,7 @@
     <meta:form-name>2021-CORE-SR-SH</meta:form-name>
     <meta:document-id>0b4a68df-30cc-474a-93c0-a56ce8fdad3b</meta:document-id>
     <meta:owner-user-id>c3061a2e6ea0b702e6f6210d5c52d2a92612d2aa</meta:owner-user-id>
-    <meta:owner-institution-id>7c5bd5fb549c09z2c55d9cb90d7ba84927e64618</meta:owner-institution-id>
+    <meta:owner-institution-id>7c5bd5fb549c09a2c55d7cb90d7ba84927e64618</meta:owner-institution-id>
     <meta:managing-institution-id>7c5bd5fb549c09z2c55d9cb90d7ba84927e64618</meta:managing-institution-id>
     <meta:created-date>2022-01-05T12:50:20.39153Z</meta:created-date>
     <meta:modified-date>2022-01-05T12:50:20.39153Z</meta:modified-date>

spec/fixtures/imports/logs/0ead17cb-1668-442d-898c-0d52879ff592.xml

@@ -19,7 +19,7 @@
     </meta:owner-user-id>
     <meta:owner-institution-id>7c5bd5fb549c09a2c55d7cb90d7ba84927e64618
     </meta:owner-institution-id>
-    <meta:managing-institution-id>7c5bd5fb549c09a2c55d7cb90d7ba84927e64618
+    <meta:managing-institution-id>7c5bd5fb549c09z2c55d9cb90d7ba84927e64618
     </meta:managing-institution-id>
     <meta:created-date>2021-10-08T14:48:17.096123Z</meta:created-date>
     <meta:modified-date>2021-10-08T14:48:17.096123Z</meta:modified-date>

spec/fixtures/imports/sales_logs/discounted_ownership_sales_log.xml

@@ -3,8 +3,8 @@
     <meta:form-name>2022-CORE-Sales</meta:form-name>
     <meta:document-id>discounted_ownership_sales_log</meta:document-id>
     <meta:owner-user-id>c3061a2e6ea0b702e6f6210d5c52d2a92612d2aa</meta:owner-user-id>
-    <meta:owner-institution-id>7c5bd5fb549c09z2c55d9cb90d7ba84927e64618</meta:owner-institution-id>
-    <meta:managing-institution-id>7c5bd5fb549c09z2c55d9cb90d7ba84927e64618</meta:managing-institution-id>
+    <meta:owner-institution-id>7c5bd5fb549c09a2c55d7cb90d7ba84927e64618</meta:owner-institution-id>
+    <meta:managing-institution-id>7c5bd5fb549c09a2c55d7cb90d7ba84927e64618</meta:managing-institution-id>
     <meta:created-date>2023-02-21T11:54:51.786722Z</meta:created-date>
     <meta:modified-date>2023-02-22T10:59:45.88188Z</meta:modified-date>
     <meta:status>submitted-valid</meta:status>

spec/fixtures/imports/sales_logs/lettings_log.xml

@@ -3,8 +3,8 @@
     <meta:form-name>2021-CORE-SR-SH</meta:form-name>
     <meta:document-id>lettings_log</meta:document-id>
     <meta:owner-user-id>c3061a2e6ea0b702e6f6210d5c52d2a92612d2aa</meta:owner-user-id>
-    <meta:owner-institution-id>7c5bd5fb549c09z2c55d9cb90d7ba84927e64618</meta:owner-institution-id>
-    <meta:managing-institution-id>7c5bd5fb549c09z2c55d9cb90d7ba84927e64618</meta:managing-institution-id>
+    <meta:owner-institution-id>7c5bd5fb549c09a2c55d7cb90d7ba84927e64618</meta:owner-institution-id>
+    <meta:managing-institution-id>7c5bd5fb549c09a2c55d7cb90d7ba84927e64618</meta:managing-institution-id>
     <meta:created-date>2022-01-05T12:50:20.39153Z</meta:created-date>
     <meta:modified-date>2022-01-05T12:50:20.39153Z</meta:modified-date>
     <meta:status>submitted-valid</meta:status>

spec/lib/tasks/correct_address_from_csv_spec.rb

@@ -0,0 +1,154 @@
+require "rails_helper"
+require "rake"
+
+RSpec.describe "data_import" do
+  def replace_entity_ids(lettings_log, second_lettings_log, third_lettings_log, export_template)
+    export_template.sub!(/\{id\}/, lettings_log.id.to_s)
+    export_template.sub!(/\{id2\}/, second_lettings_log.id.to_s)
+    export_template.sub!(/\{id3\}/, third_lettings_log.id.to_s)
+  end
+
+  describe ":import_address_from_csv", type: :task do
+    subject(:task) { Rake::Task["data_import:import_address_from_csv"] }
+
+    let(:instance_name) { "paas_import_instance" }
+    let(:storage_service) { instance_double(Storage::S3Service) }
+    let(:env_config_service) { instance_double(Configuration::EnvConfigurationService) }
+    let(:paas_config_service) { instance_double(Configuration::PaasConfigurationService) }
+
+    before do
+      allow(Storage::S3Service).to receive(:new).and_return(storage_service)
+      allow(Configuration::EnvConfigurationService).to receive(:new).and_return(env_config_service)
+      allow(Configuration::PaasConfigurationService).to receive(:new).and_return(paas_config_service)
+      allow(ENV).to receive(:[])
+      allow(ENV).to receive(:[]).with("IMPORT_PAAS_INSTANCE").and_return(instance_name)
+      allow(ENV).to receive(:[]).with("VCAP_SERVICES").and_return("dummy")
+
+      Rake.application.rake_require("tasks/import_address_from_csv")
+      Rake::Task.define_task(:environment)
+      task.reenable
+
+      WebMock.stub_request(:get, /api.postcodes.io/)
+        .to_return(status: 200, body: "{\"status\":404,\"error\":\"Postcode not found\"}", headers: {})
+      WebMock.stub_request(:get, /api.postcodes.io\/postcodes\/B11BB/)
+        .to_return(status: 200, body: '{"status":200,"result":{"admin_district":"Westminster","codes":{"admin_district":"E08000035"}}}', headers: {})
+    end
+
+    context "when the rake task is run" do
+      let(:addresses_csv_path) { "addresses_reimport_123.csv" }
+      let(:wrong_file_path) { "/test/no_csv_here.csv" }
+      let!(:lettings_log) do
+        create(:lettings_log,
+               uprn_known: nil,
+               uprn: nil,
+               uprn_confirmed: nil,
+               address_line1: nil,
+               address_line2: nil,
+               town_or_city: nil,
+               county: nil,
+               postcode_known: 1,
+               postcode_full: "A1 1AA",
+               la: "E06000064",
+               is_la_inferred: true)
+      end
+
+      let!(:second_lettings_log) do
+        create(:lettings_log,
+               uprn_known: 1,
+               uprn: "1",
+               uprn_confirmed: nil,
+               address_line1: "wrong address line1",
+               address_line2: "wrong address 2",
+               town_or_city: "wrong town",
+               county: "wrong city",
+               postcode_known: 1,
+               postcode_full: "A1 1AA",
+               la: "E06000064",
+               is_la_inferred: true)
+      end
+
+      let!(:third_lettings_log) do
+        create(:lettings_log,
+               uprn_known: 1,
+               uprn: "1",
+               uprn_confirmed: nil,
+               address_line1: "wrong address line1",
+               address_line2: "wrong address 2",
+               town_or_city: "wrong town",
+               county: "wrong city",
+               postcode_known: 1,
+               postcode_full: "A1 1AA",
+               la: "E06000064",
+               is_la_inferred: true)
+      end
+
+      before do
+        allow(storage_service).to receive(:get_file_io)
+        .with("addresses_reimport_123.csv")
+        .and_return(replace_entity_ids(lettings_log, second_lettings_log, third_lettings_log, File.open("./spec/fixtures/files/addresses_reimport.csv").read))
+      end
+
+      it "updates the log address when old address was not given" do
+        task.invoke(addresses_csv_path)
+        lettings_log.reload
+        expect(lettings_log.uprn_known).to eq(0)
+        expect(lettings_log.uprn).to eq(nil)
+        expect(lettings_log.uprn_confirmed).to eq(nil)
+        expect(lettings_log.address_line1).to eq("address 1")
+        expect(lettings_log.address_line2).to eq("address 2")
+        expect(lettings_log.town_or_city).to eq("town")
+        expect(lettings_log.county).to eq(nil)
+        expect(lettings_log.postcode_known).to eq(1)
+        expect(lettings_log.postcode_full).to eq("B1 1BB")
+        expect(lettings_log.la).to eq("E08000035")
+        expect(lettings_log.is_la_inferred).to eq(true)
+      end
+
+      it "updates the log address when old address was given" do
+        task.invoke(addresses_csv_path)
+        second_lettings_log.reload
+        expect(second_lettings_log.uprn_known).to eq(0)
+        expect(second_lettings_log.uprn).to eq(nil)
+        expect(second_lettings_log.uprn_confirmed).to eq(nil)
+        expect(second_lettings_log.address_line1).to eq("address 3")
+        expect(second_lettings_log.address_line2).to eq("address 4")
+        expect(second_lettings_log.town_or_city).to eq("city")
+        expect(second_lettings_log.county).to eq(nil)
+        expect(second_lettings_log.postcode_known).to eq(1)
+        expect(second_lettings_log.postcode_full).to eq("B1 1BB")
+        expect(second_lettings_log.la).to eq("E08000035")
+        expect(second_lettings_log.is_la_inferred).to eq(true)
+      end
+
+      it "updates the log address when new address is not given" do
+        task.invoke(addresses_csv_path)
+        third_lettings_log.reload
+        expect(third_lettings_log.uprn_known).to eq(0)
+        expect(third_lettings_log.uprn).to eq(nil)
+        expect(third_lettings_log.uprn_confirmed).to eq(nil)
+        expect(third_lettings_log.address_line1).to eq(nil)
+        expect(third_lettings_log.address_line2).to eq(nil)
+        expect(third_lettings_log.town_or_city).to eq(nil)
+        expect(third_lettings_log.county).to eq(nil)
+        expect(third_lettings_log.postcode_known).to eq(1)
+        expect(third_lettings_log.postcode_full).to eq("C11CC")
+        expect(third_lettings_log.la).to eq(nil)
+        expect(third_lettings_log.is_la_inferred).to eq(false)
+      end
+
+      it "logs the progress of the update" do
+        expect(Rails.logger).to receive(:info).with("Updated lettings log #{lettings_log.id}, with address: address 1, address 2, town, B1 1BB")
+        expect(Rails.logger).to receive(:info).with("Updated lettings log #{second_lettings_log.id}, with address: address 3, address 4, city, B1 1BB")
+        expect(Rails.logger).to receive(:info).with("Updated lettings log #{third_lettings_log.id}, with address: , , , C11CC")
+        expect(Rails.logger).to receive(:info).with("Lettings log ID not provided for address: address 5, address 6, city, D1 1DD")
+        expect(Rails.logger).to receive(:info).with("Could not find a lettings log with id fake_id")
+
+        task.invoke(addresses_csv_path)
+      end
+
+      it "raises an error when no path is given" do
+        expect { task.invoke(nil) }.to raise_error(RuntimeError, "Usage: rake data_import:import_address_from_csv['csv_file_name']")
+      end
+    end
+  end
+end

spec/lib/tasks/data_import_field_spec.rb

@@ -92,6 +92,30 @@ describe "data_import_field imports" do
         end
       end
 
+      context "and we update the address fields" do
+        let(:field) { "address" }
+
+        it "updates the 2023 logs from the given XML file" do
+          expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name)
+          expect(storage_service).to receive(:get_file_io).with("spec/fixtures/imports/logs")
+          expect(Imports::LettingsLogsFieldImportService).to receive(:new).with(archive_service)
+          expect(import_service).to receive(:update_field).with(field, "logs")
+          task.invoke(field, fixture_path)
+        end
+      end
+
+      context "and we update the reason field" do
+        let(:field) { "reason" }
+
+        it "updates the 2023 logs from the given XML file" do
+          expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name)
+          expect(storage_service).to receive(:get_file_io).with("spec/fixtures/imports/logs")
+          expect(Imports::LettingsLogsFieldImportService).to receive(:new).with(archive_service)
+          expect(import_service).to receive(:update_field).with(field, "logs")
+          task.invoke(field, fixture_path)
+        end
+      end
+
       it "raises an exception if no parameters are provided" do
         expect { task.invoke }.to raise_error(/Usage/)
       end

spec/lib/tasks/full_import_spec.rb

@@ -54,6 +54,32 @@ describe "full import", type: :task do
     end
   end
 
+  describe "import:generate_missing_answers_report" do
+    subject(:task) { Rake::Task["import:generate_missing_answers_report"] }
+
+    before do
+      Rake.application.rake_require("tasks/full_import")
+      Rake::Task.define_task(:environment)
+      task.reenable
+    end
+
+    context "when generating a missing answers report" do
+      let(:import_report_service) { instance_double(Imports::ImportReportService) }
+
+      before do
+        allow(Imports::ImportReportService).to receive(:new).and_return(import_report_service)
+        allow(ENV).to receive(:[]).with("VCAP_SERVICES").and_return("dummy")
+      end
+
+      it "creates a missing answers report" do
+        expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name)
+        expect(Imports::ImportReportService).to receive(:new).with(storage_service, nil)
+        expect(import_report_service).to receive(:generate_missing_answers_report).with("some_name")
+        task.invoke("some_name")
+      end
+    end
+  end
+
   describe "import:initial" do
     subject(:task) { Rake::Task["import:initial"] }
 

spec/models/form/lettings/subsections/household_situation_spec.rb

@@ -30,6 +30,7 @@ RSpec.describe Form::Lettings::Subsections::HouseholdSituation, type: :model do
         referral_prp
         referral_supported_housing
         referral_supported_housing_prp
+        referral_value_check
       ],
     )
   end

spec/models/form/lettings/subsections/income_and_benefits_spec.rb

@@ -32,6 +32,9 @@ RSpec.describe Form::Lettings::Subsections::IncomeAndBenefits, type: :model do
         rent_monthly
         brent_min_rent_value_check
         brent_max_rent_value_check
+        scharge_value_check
+        pscharge_value_check
+        supcharg_value_check
         outstanding
         outstanding_amount
       ],

spec/models/validations/financial_validations_spec.rb

@@ -263,39 +263,39 @@ RSpec.describe Validations::FinancialValidations do
 
         [{
           period: { label: "weekly", value: 1 },
-          charge: { field: "scharge", value: 156 },
+          charge: { field: "scharge", value: 801 },
         },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "scharge", value: 672 },
+           charge: { field: "scharge", value: 3471 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "scharge", value: 311 },
+           charge: { field: "scharge", value: 1601 },
          },
          {
            period: { label: "weekly", value: 1 },
-           charge: { field: "pscharge", value: 31 },
+           charge: { field: "pscharge", value: 701 },
          },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "pscharge", value: 150 },
+           charge: { field: "pscharge", value: 3200 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "pscharge", value: 61 },
+           charge: { field: "pscharge", value: 1401 },
          },
          {
            period: { label: "weekly", value: 1 },
-           charge: { field: "supcharg", value: 41 },
+           charge: { field: "supcharg", value: 801 },
          },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "supcharg", value: 200 },
+           charge: { field: "supcharg", value: 3471 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "supcharg", value: 81 },
+           charge: { field: "supcharg", value: 1601 },
          }].each do |test_case|
           it "does not allow charges outside the range when period is #{test_case[:period][:label]}" do
             record.period = test_case[:period][:value]
@@ -308,39 +308,39 @@ RSpec.describe Validations::FinancialValidations do
 
         [{
           period: { label: "weekly", value: 1 },
-          charge: { field: "scharge", value: 154 },
+          charge: { field: "scharge", value: 799 },
         },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "scharge", value: 670 },
+           charge: { field: "scharge", value: 3400 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "scharge", value: 309 },
+           charge: { field: "scharge", value: 1599 },
          },
          {
            period: { label: "weekly", value: 1 },
-           charge: { field: "pscharge", value: 30 },
+           charge: { field: "pscharge", value: 699 },
          },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "pscharge", value: 120 },
+           charge: { field: "pscharge", value: 2500 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "pscharge", value: 59 },
+           charge: { field: "pscharge", value: 1399 },
          },
          {
            period: { label: "weekly", value: 1 },
-           charge: { field: "supcharg", value: 39 },
+           charge: { field: "supcharg", value: 799 },
          },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "supcharg", value: 120 },
+           charge: { field: "supcharg", value: 3000 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "supcharg", value: 79 },
+           charge: { field: "supcharg", value: 1599 },
          }].each do |test_case|
           it "does allow charges inside the range when period is #{test_case[:period][:label]}" do
             record.period = test_case[:period][:value]
@@ -357,39 +357,39 @@ RSpec.describe Validations::FinancialValidations do
 
         [{
           period: { label: "weekly", value: 1 },
-          charge: { field: "scharge", value: 481 },
+          charge: { field: "scharge", value: 801 },
         },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "scharge", value: 2081 },
+           charge: { field: "scharge", value: 3471 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "scharge", value: 961 },
+           charge: { field: "scharge", value: 1601 },
          },
          {
            period: { label: "weekly", value: 1 },
-           charge: { field: "pscharge", value: 201 },
+           charge: { field: "pscharge", value: 701 },
          },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "pscharge", value: 1000 },
+           charge: { field: "pscharge", value: 3200 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "pscharge", value: 400.80 },
+           charge: { field: "pscharge", value: 1401 },
          },
          {
            period: { label: "weekly", value: 1 },
-           charge: { field: "supcharg", value: 466 },
+           charge: { field: "supcharg", value: 801 },
          },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "supcharg", value: 3100 },
+           charge: { field: "supcharg", value: 3471 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "supcharg", value: 990 },
+           charge: { field: "supcharg", value: 1601 },
          }].each do |test_case|
           it "does not allow charges outside the range when period is #{test_case[:period][:label]}" do
             record.period = test_case[:period][:value]
@@ -402,39 +402,39 @@ RSpec.describe Validations::FinancialValidations do
 
         [{
           period: { label: "weekly", value: 1 },
-          charge: { field: "scharge", value: 366 },
+          charge: { field: "scharge", value: 799 },
         },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "scharge", value: 1582 },
+           charge: { field: "scharge", value: 3400 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "scharge", value: 731 },
+           charge: { field: "scharge", value: 1599 },
          },
          {
            period: { label: "weekly", value: 1 },
-           charge: { field: "pscharge", value: 199.99 },
+           charge: { field: "pscharge", value: 699 },
          },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "pscharge", value: 800 },
+           charge: { field: "pscharge", value: 2500 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "pscharge", value: 400 },
+           charge: { field: "pscharge", value: 1399 },
          },
          {
            period: { label: "weekly", value: 1 },
-           charge: { field: "supcharg", value: 464 },
+           charge: { field: "supcharg", value: 799 },
          },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "supcharg", value: 2000 },
+           charge: { field: "supcharg", value: 3400 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "supcharg", value: 880 },
+           charge: { field: "supcharg", value: 1599 },
          }].each do |test_case|
           it "does allow charges inside the range when period is #{test_case[:period][:label]}" do
             record.period = test_case[:period][:value]
@@ -455,39 +455,39 @@ RSpec.describe Validations::FinancialValidations do
 
         [{
           period: { label: "weekly", value: 1 },
-          charge: { field: "scharge", value: 156 },
+          charge: { field: "scharge", value: 501 },
         },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "scharge", value: 672 },
+           charge: { field: "scharge", value: 2300 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "scharge", value: 311 },
+           charge: { field: "scharge", value: 1001 },
          },
          {
            period: { label: "weekly", value: 1 },
-           charge: { field: "pscharge", value: 36 },
+           charge: { field: "pscharge", value: 201 },
          },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "pscharge", value: 190 },
+           charge: { field: "pscharge", value: 1000 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "pscharge", value: 71 },
+           charge: { field: "pscharge", value: 401 },
          },
          {
            period: { label: "weekly", value: 1 },
-           charge: { field: "supcharg", value: 61 },
+           charge: { field: "supcharg", value: 201 },
          },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "supcharg", value: 300 },
+           charge: { field: "supcharg", value: 1000 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "supcharg", value: 122 },
+           charge: { field: "supcharg", value: 401 },
          }].each do |test_case|
           it "does not allow charges outside the range when period is #{test_case[:period][:label]}" do
             record.period = test_case[:period][:value]
@@ -500,39 +500,39 @@ RSpec.describe Validations::FinancialValidations do
 
         [{
           period: { label: "weekly", value: 1 },
-          charge: { field: "scharge", value: 44 },
+          charge: { field: "scharge", value: 499 },
         },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "scharge", value: 160 },
+           charge: { field: "scharge", value: 2000 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "scharge", value: 89 },
+           charge: { field: "scharge", value: 999 },
          },
          {
            period: { label: "weekly", value: 1 },
-           charge: { field: "pscharge", value: 34 },
+           charge: { field: "pscharge", value: 199 },
          },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "pscharge", value: 140 },
+           charge: { field: "pscharge", value: 800 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "pscharge", value: 69 },
+           charge: { field: "pscharge", value: 399 },
          },
          {
            period: { label: "weekly", value: 1 },
-           charge: { field: "supcharg", value: 59.99 },
+           charge: { field: "supcharg", value: 199.99 },
          },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "supcharg", value: 240 },
+           charge: { field: "supcharg", value: 800 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "supcharg", value: 119 },
+           charge: { field: "supcharg", value: 399 },
          }].each do |test_case|
           it "does allow charges inside the range when period is #{test_case[:period][:label]}" do
             record.period = test_case[:period][:value]
@@ -549,39 +549,39 @@ RSpec.describe Validations::FinancialValidations do
 
         [{
           period: { label: "weekly", value: 1 },
-          charge: { field: "scharge", value: 365.90 },
+          charge: { field: "scharge", value: 501 },
         },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "scharge", value: 2081 },
+           charge: { field: "scharge", value: 2300 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "scharge", value: 961 },
+           charge: { field: "scharge", value: 1001 },
          },
          {
            period: { label: "weekly", value: 1 },
-           charge: { field: "pscharge", value: 76 },
+           charge: { field: "pscharge", value: 201 },
          },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "pscharge", value: 400 },
+           charge: { field: "pscharge", value: 1000 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "pscharge", value: 151 },
+           charge: { field: "pscharge", value: 401 },
          },
          {
            period: { label: "weekly", value: 1 },
-           charge: { field: "supcharg", value: 121 },
+           charge: { field: "supcharg", value: 201 },
          },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "supcharg", value: 620 },
+           charge: { field: "supcharg", value: 1000 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "supcharg", value: 241 },
+           charge: { field: "supcharg", value: 401 },
          }].each do |test_case|
           it "does not allow charges outside the range when period is #{test_case[:period][:label]}" do
             record.period = test_case[:period][:value]
@@ -617,39 +617,39 @@ RSpec.describe Validations::FinancialValidations do
 
         [{
           period: { label: "weekly", value: 1 },
-          charge: { field: "scharge", value: 364.88 },
+          charge: { field: "scharge", value: 499 },
         },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "scharge", value: 1200 },
+           charge: { field: "scharge", value: 2000 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "scharge", value: 700.99 },
+           charge: { field: "scharge", value: 999 },
          },
          {
            period: { label: "weekly", value: 1 },
-           charge: { field: "pscharge", value: 74 },
+           charge: { field: "pscharge", value: 199 },
          },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "pscharge", value: 210 },
+           charge: { field: "pscharge", value: 800 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "pscharge", value: 149 },
+           charge: { field: "pscharge", value: 399 },
          },
          {
            period: { label: "weekly", value: 1 },
-           charge: { field: "supcharg", value: 119 },
+           charge: { field: "supcharg", value: 199.99 },
          },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "supcharg", value: 480 },
+           charge: { field: "supcharg", value: 800 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "supcharg", value: 239 },
+           charge: { field: "supcharg", value: 399 },
          }].each do |test_case|
           it "does allow charges inside the range when period is #{test_case[:period][:label]}" do
             record.period = test_case[:period][:value]

spec/models/validations/household_validations_spec.rb

@@ -12,36 +12,6 @@ RSpec.describe Validations::HouseholdValidations do
   end
 
   describe "reasonable preference validations" do
-    context "when reasonable preference is homeless" do
-      context "when the tenant was not previously homeless" do
-        it "adds an error" do
-          record.homeless = 1
-          record.rp_homeless = 1
-          household_validator.validate_reasonable_preference(record)
-          expect(record.errors["reasonable_preference_reason"])
-            .to include(match I18n.t("validations.household.reasonpref.not_homeless"))
-          expect(record.errors["homeless"])
-            .to include(match I18n.t("validations.household.homeless.reasonpref.not_homeless"))
-        end
-      end
-
-      context "when reasonable preference is given" do
-        context "when the tenant was previously homeless" do
-          it "does not add an error" do
-            record.homeless = 1
-            record.reasonpref = 1
-            household_validator.validate_reasonable_preference(record)
-            expect(record.errors["reasonpref"]).to be_empty
-            expect(record.errors["homeless"]).to be_empty
-            record.homeless = 0
-            household_validator.validate_reasonable_preference(record)
-            expect(record.errors["reasonpref"]).to be_empty
-            expect(record.errors["homeless"]).to be_empty
-          end
-        end
-      end
-    end
-
     context "when reasonable preference is not given" do
       it "validates that no reason is needed" do
         record.reasonpref = 1

spec/models/validations/sales/sale_information_validations_spec.rb

@@ -171,60 +171,6 @@ RSpec.describe Validations::Sales::SaleInformationValidations do
     end
   end
 
-  describe "#validate_years_living_in_property_before_purchase" do
-    context "when proplen blank" do
-      let(:record) { build(:sales_log, proplen: nil) }
-
-      it "does not add an error" do
-        sale_information_validator.validate_years_living_in_property_before_purchase(record)
-
-        expect(record.errors).not_to be_present
-      end
-    end
-
-    context "when type blank" do
-      let(:record) { build(:sales_log, type: nil) }
-
-      it "does not add an error" do
-        sale_information_validator.validate_years_living_in_property_before_purchase(record)
-
-        expect(record.errors).not_to be_present
-      end
-    end
-
-    context "when proplen 0" do
-      let(:record) { build(:sales_log, proplen: 0) }
-
-      it "does not add an error" do
-        sale_information_validator.validate_years_living_in_property_before_purchase(record)
-
-        expect(record.errors).not_to be_present
-      end
-    end
-
-    context "when type Rent to Buy and proplen > 0" do
-      let(:record) { build(:sales_log, proplen: 1, type: 28) }
-
-      it "adds an error" do
-        sale_information_validator.validate_years_living_in_property_before_purchase(record)
-
-        expect(record.errors[:type]).to include(I18n.t("validations.sale_information.proplen.rent_to_buy"))
-        expect(record.errors[:proplen]).to include(I18n.t("validations.sale_information.proplen.rent_to_buy"))
-      end
-    end
-
-    context "when type Social HomeBuy and proplen > 0" do
-      let(:record) { build(:sales_log, proplen: 1, type: 18) }
-
-      it "adds an error" do
-        sale_information_validator.validate_years_living_in_property_before_purchase(record)
-
-        expect(record.errors[:type]).to include(I18n.t("validations.sale_information.proplen.social_homebuy"))
-        expect(record.errors[:proplen]).to include(I18n.t("validations.sale_information.proplen.social_homebuy"))
-      end
-    end
-  end
-
   describe "#validate_discounted_ownership_value" do
     context "when sale is on or after 24/25 collection window" do
       context "when grant is routed to" do

spec/models/validations/soft_validations_spec.rb

@@ -345,4 +345,676 @@ RSpec.describe Validations::SoftValidations do
       expect(record).to be_care_home_charge_expected_not_provided
     end
   end
+
+  describe "#la_referral_for_general_needs?" do
+    it "returns false if needstype is 'Supported Housing'" do
+      record.needstype = 2
+      record.referral = 4
+
+      expect(record).not_to be_la_referral_for_general_needs
+    end
+
+    it "returns false if needstype is not given" do
+      record.needstype = nil
+      record.referral = 4
+
+      expect(record).not_to be_la_referral_for_general_needs
+    end
+
+    it "returns false if referral is not given" do
+      record.needstype = 1
+      record.referral = nil
+
+      expect(record).not_to be_la_referral_for_general_needs
+    end
+
+    it "returns true if needstype is 'General needs' and referral is 4" do
+      record.needstype = 1
+      record.referral = 4
+
+      expect(record).to be_la_referral_for_general_needs
+    end
+  end
+
+  describe "scharge_over_soft_max?" do
+    context "and organisation is PRP" do
+      before do
+        record.owning_organisation.update(provider_type: "PRP")
+      end
+
+      it "returns false if scharge is not given" do
+        record.scharge = nil
+        record.needstype = 1
+        record.period = 1
+
+        expect(record).not_to be_scharge_over_soft_max
+      end
+
+      it "returns false if period is not given" do
+        record.scharge = 201
+        record.needstype = 1
+        record.period = nil
+
+        expect(record).not_to be_scharge_over_soft_max
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        scharge: 34,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         scharge: 100,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         scharge: 69,
+       }].each do |test_case|
+        it "returns false if scharge is under soft max for general needs #{test_case[:period][:label]}(35)" do
+          record.scharge = test_case[:scharge]
+          record.needstype = 1
+          record.period = test_case[:period][:value]
+
+          expect(record).not_to be_scharge_over_soft_max
+        end
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        scharge: 199,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         scharge: 400,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         scharge: 399,
+       }].each do |test_case|
+        it "returns false if scharge is under soft max for supported housing #{test_case[:period][:label]} (200)" do
+          record.scharge = test_case[:scharge]
+          record.needstype = 2
+          record.period = test_case[:period][:value]
+
+          expect(record).not_to be_scharge_over_soft_max
+        end
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        scharge: 36,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         scharge: 180,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         scharge: 71,
+       }].each do |test_case|
+        it "returns true if scharge is over soft max for general needs #{test_case[:period][:label]} (35)" do
+          record.scharge = test_case[:scharge]
+          record.needstype = 1
+          record.period = test_case[:period][:value]
+
+          expect(record).to be_scharge_over_soft_max
+        end
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        scharge: 201,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         scharge: 1000,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         scharge: 401,
+       }].each do |test_case|
+        it "returns true if scharge is over soft max for supported housing #{test_case[:period][:label]} (200)" do
+          record.scharge = test_case[:scharge]
+          record.needstype = 2
+          record.period = test_case[:period][:value]
+
+          expect(record).to be_scharge_over_soft_max
+        end
+      end
+    end
+
+    context "and organisation is LA" do
+      before do
+        record.owning_organisation.update(provider_type: "LA")
+      end
+
+      it "returns false if scharge is not given" do
+        record.scharge = nil
+        record.needstype = 1
+        record.period = 1
+
+        expect(record).not_to be_scharge_over_soft_max
+      end
+
+      it "returns false if period is not given" do
+        record.scharge = 201
+        record.needstype = 1
+        record.period = nil
+
+        expect(record).not_to be_scharge_over_soft_max
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        scharge: 24,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         scharge: 88,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         scharge: 49,
+       }].each do |test_case|
+        it "returns false if scharge is under soft max for general needs #{test_case[:period][:label]}(25)" do
+          record.scharge = test_case[:scharge]
+          record.needstype = 1
+          record.period = test_case[:period][:value]
+
+          expect(record).not_to be_scharge_over_soft_max
+        end
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        scharge: 99,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         scharge: 400,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         scharge: 199,
+       }].each do |test_case|
+        it "returns false if scharge is under soft max for supported housing #{test_case[:period][:label]} (100)" do
+          record.scharge = test_case[:scharge]
+          record.needstype = 2
+          record.period = test_case[:period][:value]
+
+          expect(record).not_to be_scharge_over_soft_max
+        end
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        scharge: 26,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         scharge: 120,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         scharge: 51,
+       }].each do |test_case|
+        it "returns true if scharge is over soft max for general needs #{test_case[:period][:label]} (25)" do
+          record.scharge = test_case[:scharge]
+          record.needstype = 1
+          record.period = test_case[:period][:value]
+
+          expect(record).to be_scharge_over_soft_max
+        end
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        scharge: 101,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         scharge: 450,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         scharge: 201,
+       }].each do |test_case|
+        it "returns true if scharge is over soft max for supported housing #{test_case[:period][:label]} (100)" do
+          record.scharge = test_case[:scharge]
+          record.needstype = 2
+          record.period = test_case[:period][:value]
+
+          expect(record).to be_scharge_over_soft_max
+        end
+      end
+    end
+  end
+
+  describe "pscharge_over_soft_max?" do
+    context "and organisation is PRP" do
+      before do
+        record.owning_organisation.update(provider_type: "PRP")
+      end
+
+      it "returns false if pscharge is not given" do
+        record.pscharge = nil
+        record.needstype = 1
+        record.period = 1
+
+        expect(record).not_to be_pscharge_over_soft_max
+      end
+
+      it "returns false if period is not given" do
+        record.pscharge = 201
+        record.needstype = 1
+        record.period = nil
+
+        expect(record).not_to be_pscharge_over_soft_max
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        pscharge: 34,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         pscharge: 100,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         pscharge: 69,
+       }].each do |test_case|
+        it "returns false if pscharge is under soft max for general needs #{test_case[:period][:label]}(35)" do
+          record.pscharge = test_case[:pscharge]
+          record.needstype = 1
+          record.period = test_case[:period][:value]
+
+          expect(record).not_to be_pscharge_over_soft_max
+        end
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        pscharge: 99,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         pscharge: 400,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         pscharge: 199,
+       }].each do |test_case|
+        it "returns false if pscharge is under soft max for supported housing #{test_case[:period][:label]} (100)" do
+          record.pscharge = test_case[:pscharge]
+          record.needstype = 2
+          record.period = test_case[:period][:value]
+
+          expect(record).not_to be_pscharge_over_soft_max
+        end
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        pscharge: 36,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         pscharge: 180,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         pscharge: 71,
+       }].each do |test_case|
+        it "returns true if pscharge is over soft max for general needs #{test_case[:period][:label]} (35)" do
+          record.pscharge = test_case[:pscharge]
+          record.needstype = 1
+          record.period = test_case[:period][:value]
+
+          expect(record).to be_pscharge_over_soft_max
+        end
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        pscharge: 101,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         pscharge: 450,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         pscharge: 201,
+       }].each do |test_case|
+        it "returns true if pscharge is over soft max for supported housing #{test_case[:period][:label]} (100)" do
+          record.pscharge = test_case[:pscharge]
+          record.needstype = 2
+          record.period = test_case[:period][:value]
+
+          expect(record).to be_pscharge_over_soft_max
+        end
+      end
+    end
+
+    context "and organisation is LA" do
+      before do
+        record.owning_organisation.update(provider_type: "LA")
+      end
+
+      it "returns false if pscharge is not given" do
+        record.pscharge = nil
+        record.needstype = 1
+        record.period = 1
+
+        expect(record).not_to be_pscharge_over_soft_max
+      end
+
+      it "returns false if period is not given" do
+        record.pscharge = 201
+        record.needstype = 1
+        record.period = nil
+
+        expect(record).not_to be_pscharge_over_soft_max
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        pscharge: 24,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         pscharge: 88,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         pscharge: 49,
+       }].each do |test_case|
+        it "returns false if pscharge is under soft max for general needs #{test_case[:period][:label]}(25)" do
+          record.pscharge = test_case[:pscharge]
+          record.needstype = 1
+          record.period = test_case[:period][:value]
+
+          expect(record).not_to be_pscharge_over_soft_max
+        end
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        pscharge: 74,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         pscharge: 250,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         pscharge: 149,
+       }].each do |test_case|
+        it "returns false if pscharge is under soft max for supported housing #{test_case[:period][:label]} (75)" do
+          record.pscharge = test_case[:pscharge]
+          record.needstype = 2
+          record.period = test_case[:period][:value]
+
+          expect(record).not_to be_pscharge_over_soft_max
+        end
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        pscharge: 26,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         pscharge: 120,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         pscharge: 51,
+       }].each do |test_case|
+        it "returns true if pscharge is over soft max for general needs #{test_case[:period][:label]} (25)" do
+          record.pscharge = test_case[:pscharge]
+          record.needstype = 1
+          record.period = test_case[:period][:value]
+
+          expect(record).to be_pscharge_over_soft_max
+        end
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        pscharge: 76,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         pscharge: 350,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         pscharge: 151,
+       }].each do |test_case|
+        it "returns true if pscharge is over soft max for supported housing #{test_case[:period][:label]} (75)" do
+          record.pscharge = test_case[:pscharge]
+          record.needstype = 2
+          record.period = test_case[:period][:value]
+
+          expect(record).to be_pscharge_over_soft_max
+        end
+      end
+    end
+  end
+
+  describe "supcharg_over_soft_max?" do
+    context "and organisation is PRP" do
+      before do
+        record.owning_organisation.update(provider_type: "PRP")
+      end
+
+      it "returns false if supcharg is not given" do
+        record.supcharg = nil
+        record.needstype = 1
+        record.period = 1
+
+        expect(record).not_to be_supcharg_over_soft_max
+      end
+
+      it "returns false if period is not given" do
+        record.supcharg = 201
+        record.needstype = 1
+        record.period = nil
+
+        expect(record).not_to be_supcharg_over_soft_max
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        supcharg: 34,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         supcharg: 100,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         supcharg: 69,
+       }].each do |test_case|
+        it "returns false if supcharg is under soft max for general needs #{test_case[:period][:label]}(35)" do
+          record.supcharg = test_case[:supcharg]
+          record.needstype = 1
+          record.period = test_case[:period][:value]
+
+          expect(record).not_to be_supcharg_over_soft_max
+        end
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        supcharg: 84,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         supcharg: 320,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         supcharg: 169,
+       }].each do |test_case|
+        it "returns false if supcharg is under soft max for supported housing #{test_case[:period][:label]} (85)" do
+          record.supcharg = test_case[:supcharg]
+          record.needstype = 2
+          record.period = test_case[:period][:value]
+
+          expect(record).not_to be_supcharg_over_soft_max
+        end
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        supcharg: 36,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         supcharg: 180,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         supcharg: 71,
+       }].each do |test_case|
+        it "returns true if supcharg is over soft max for general needs #{test_case[:period][:label]} (35)" do
+          record.supcharg = test_case[:supcharg]
+          record.needstype = 1
+          record.period = test_case[:period][:value]
+
+          expect(record).to be_supcharg_over_soft_max
+        end
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        supcharg: 86,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         supcharg: 400,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         supcharg: 171,
+       }].each do |test_case|
+        it "returns true if supcharg is over soft max for supported housing #{test_case[:period][:label]} (85)" do
+          record.supcharg = test_case[:supcharg]
+          record.needstype = 2
+          record.period = test_case[:period][:value]
+
+          expect(record).to be_supcharg_over_soft_max
+        end
+      end
+    end
+
+    context "and organisation is LA" do
+      before do
+        record.owning_organisation.update(provider_type: "LA")
+      end
+
+      it "returns false if supcharg is not given" do
+        record.supcharg = nil
+        record.needstype = 1
+        record.period = 1
+
+        expect(record).not_to be_supcharg_over_soft_max
+      end
+
+      it "returns false if period is not given" do
+        record.supcharg = 201
+        record.needstype = 1
+        record.period = nil
+
+        expect(record).not_to be_supcharg_over_soft_max
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        supcharg: 24,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         supcharg: 88,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         supcharg: 49,
+       }].each do |test_case|
+        it "returns false if supcharg is under soft max for general needs #{test_case[:period][:label]}(25)" do
+          record.supcharg = test_case[:supcharg]
+          record.needstype = 1
+          record.period = test_case[:period][:value]
+
+          expect(record).not_to be_supcharg_over_soft_max
+        end
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        supcharg: 74,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         supcharg: 250,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         supcharg: 149,
+       }].each do |test_case|
+        it "returns false if supcharg is under soft max for supported housing #{test_case[:period][:label]} (75)" do
+          record.supcharg = test_case[:supcharg]
+          record.needstype = 2
+          record.period = test_case[:period][:value]
+
+          expect(record).not_to be_supcharg_over_soft_max
+        end
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        supcharg: 26,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         supcharg: 120,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         supcharg: 51,
+       }].each do |test_case|
+        it "returns true if supcharg is over soft max for general needs #{test_case[:period][:label]} (25)" do
+          record.supcharg = test_case[:supcharg]
+          record.needstype = 1
+          record.period = test_case[:period][:value]
+
+          expect(record).to be_supcharg_over_soft_max
+        end
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        supcharg: 76,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         supcharg: 350,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         supcharg: 151,
+       }].each do |test_case|
+        it "returns true if supcharg is over soft max for supported housing #{test_case[:period][:label]} (75)" do
+          record.supcharg = test_case[:supcharg]
+          record.needstype = 2
+          record.period = test_case[:period][:value]
+
+          expect(record).to be_supcharg_over_soft_max
+        end
+      end
+    end
+  end
 end

spec/services/bulk_upload/lettings/year2022/row_parser_spec.rb

@@ -711,14 +711,6 @@ RSpec.describe BulkUpload::Lettings::Year2022::RowParser do
     end
 
     describe "#field_68 - 74" do
-      context "when not homeless but reasonable preference for homelessness" do
-        let(:attributes) { { bulk_upload:, field_68: "1", field_69: "1", field_70: "1" } }
-
-        it "is not permitted" do
-          expect(parser.errors[:field_70]).to be_present
-        end
-      end
-
       context "when there is a reasonable preference but none is given" do
         let(:attributes) { { bulk_upload:, field_69: "1", field_70: nil, field_71: nil, field_72: nil, field_73: nil, field_74: nil } }
 

spec/services/bulk_upload/lettings/year2023/row_parser_spec.rb

@@ -803,14 +803,6 @@ RSpec.describe BulkUpload::Lettings::Year2023::RowParser do
     end
 
     describe "#field_105, field_110 - 15" do
-      context "when not homeless but reasonable preference for homelessness" do
-        let(:attributes) { { bulk_upload:, field_105: "1", field_110: "1", field_111: "1" } }
-
-        it "is not permitted" do
-          expect(parser.errors[:field_111]).to be_present
-        end
-      end
-
       context "when there is a reasonable preference but none is given" do
         let(:attributes) { { bulk_upload:, field_110: "1", field_111: nil, field_112: nil, field_113: nil, field_114: nil, field_115: nil } }
 

spec/services/exports/lettings_log_export_service_spec.rb

@@ -328,6 +328,26 @@ RSpec.describe Exports::LettingsLogExportService do
         expect(LogsExport.last.increment_number).to eq(1)
       end
     end
+
+    context "and a log has been migrated since the previous partial export" do
+      before do
+        FactoryBot.create(:lettings_log, startdate: Time.zone.local(2022, 2, 1), updated_at: Time.zone.local(2022, 4, 27), values_updated_at: Time.zone.local(2022, 4, 29))
+        FactoryBot.create(:lettings_log, startdate: Time.zone.local(2022, 2, 1), updated_at: Time.zone.local(2022, 4, 27), values_updated_at: Time.zone.local(2022, 4, 29))
+        LogsExport.create!(started_at: Time.zone.local(2022, 4, 28), base_number: 1, increment_number: 1)
+      end
+
+      it "generates an XML manifest file with the expected content within the ZIP file" do
+        expected_content = replace_record_number(local_manifest_file.read, 2)
+        expect(storage_service).to receive(:write_file).with(expected_master_manifest_rerun, any_args)
+        expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) do |_, content|
+          entry = Zip::File.open_buffer(content).find_entry(expected_manifest_filename)
+          expect(entry).not_to be_nil
+          expect(entry.get_input_stream.read).to eq(expected_content)
+        end
+
+        export_service.export_xml_lettings_logs
+      end
+    end
   end
 
   context "when exporting a supported housing lettings logs in XML" do

spec/services/imports/import_report_service_spec.rb

@@ -67,4 +67,113 @@ RSpec.describe Imports::ImportReportService do
       report_service.generate_logs_report("report_suffix.csv")
     end
   end
+
+  describe "#generate_unassigned_logs_report" do
+    context "when there is no unassigned user (all the logs have ben assigned)" do
+      let(:institutions_csv) { CSV.parse("Institution name,Id,Old Completed lettings logs,Old In progress lettings logs,Old Completed sales logs,Old In progress sales logs\norg1,1,2,1,4,3", headers: true) }
+
+      it "writes an empty unassigned logs report" do
+        expect(storage_service).to receive(:write_file).with("UnassignedLogsReport_report_suffix.csv", "\uFEFFOwning Organisation ID,Old Owning Organisation ID,Managing Organisation ID,Old Managing Organisation ID,Log ID,Old Log ID,Tenancy code,Purchaser code\n")
+
+        report_service.generate_unassigned_logs_report("report_suffix.csv")
+      end
+    end
+
+    context "when some logs have been added to Unassigned user" do
+      let(:organisation) { create(:organisation, old_org_id: "1", name: "org1") }
+      let(:organisation2) { create(:organisation, old_org_id: "2", name: "org2") }
+      let(:unassigned_user) { create(:user, name: "Unassigned", organisation:) }
+      let(:institutions_csv) { CSV.parse("Institution name,Id,Old Completed lettings logs,Old In progress lettings logs,Old Completed sales logs,Old In progress sales logs\norg1,1,2,1,4,3", headers: true) }
+      let!(:lettings_log) { create(:lettings_log, owning_organisation: organisation, managing_organisation: organisation2, created_by: unassigned_user, tenancycode: "tenancycode", old_id: "12") }
+      let!(:sales_log) { create(:sales_log, owning_organisation: organisation, created_by: unassigned_user, purchid: "purchid", old_id: "23") }
+
+      before do
+        create(:organisation_relationship, parent_organisation: organisation, child_organisation: organisation2)
+      end
+
+      it "writes a report with all unassigned logs" do
+        expect(storage_service).to receive(:write_file).with("UnassignedLogsReport_report_suffix.csv", "\uFEFFOwning Organisation ID,Old Owning Organisation ID,Managing Organisation ID,Old Managing Organisation ID,Log ID,Old Log ID,Tenancy code,Purchaser code\n#{organisation.id},1,#{organisation2.id},2,#{lettings_log.id},12,tenancycode,\n#{organisation.id},1,,,#{sales_log.id},23,,purchid\n")
+
+        report_service.generate_unassigned_logs_report("report_suffix.csv")
+      end
+    end
+  end
+
+  describe "#generate_missing_answers_report" do
+    context "when there are in progress imported lettings logs" do
+      let(:institutions_csv) { nil }
+      let(:expected_content) { File.read("spec/fixtures/files/imported_lettings_logs_missing_answers_report.csv") }
+      let(:expected_answers_examples_content) { File.read("spec/fixtures/files/imported_lettings_logs_missing_answers_examples.csv") }
+
+      before do
+        create_list(:lettings_log, 11, :completed, age1_known: nil) do |log, i|
+          log.old_form_id = "100#{i}"
+          log.old_id = "old_id_age1_known_#{i}"
+          log.save!
+          expected_answers_examples_content.sub!("{id#{i}}", log.id.to_s)
+          expected_answers_examples_content.sub!("{org_id#{i}}", log.owning_organisation_id.to_s)
+        end
+        create_list(:lettings_log, 4, :completed, beds: nil) do |log, i|
+          log.old_form_id = "200#{i}"
+          log.old_id = "old_id_beds_#{i}"
+          expected_answers_examples_content.sub!("{id2_#{i}}", log.id.to_s)
+          expected_answers_examples_content.sub!("{org_id2_#{i}}", log.owning_organisation_id.to_s)
+          log.save!
+        end
+        create(:lettings_log, :completed, age1_known: nil, beds: nil, old_form_id: "300", old_id: "beds_and_age") do |log|
+          expected_answers_examples_content.sub!("{id}", log.id.to_s)
+          expected_answers_examples_content.sub!("{org_id}", log.owning_organisation_id.to_s)
+        end
+
+        create_list(:lettings_log, 2, :completed, age1_known: nil)
+      end
+
+      it "generates a csv with expected missing fields" do
+        expect(storage_service).to receive(:write_file).with("MissingAnswersReportLettingsLog_report_suffix.csv", "#{expected_content}")
+        expect(storage_service).to receive(:write_file).with("MissingAnswersExamplesLettingsLog_report_suffix.csv", "#{expected_answers_examples_content}")
+        expect(storage_service).to receive(:write_file).with("MissingAnswersReportSalesLog_report_suffix.csv", "\uFEFFMissing answers,Total number of affected logs\n")
+        expect(storage_service).to receive(:write_file).with("MissingAnswersExamplesSalesLog_report_suffix.csv", "\uFEFFMissing answers,Organisation ID,Log ID,Old Form ID,Old Log ID\n")
+
+        report_service.generate_missing_answers_report("report_suffix")
+      end
+    end
+
+    context "when there are in progress imported sales logs" do
+      let(:institutions_csv) { nil }
+      let(:expected_content) { File.read("spec/fixtures/files/imported_lettings_logs_missing_answers_report.csv") }
+      let(:expected_answers_examples_content) { File.read("spec/fixtures/files/imported_lettings_logs_missing_answers_examples.csv") }
+
+      before do
+        create_list(:sales_log, 11, :completed, age1_known: nil) do |log, i|
+          log.old_id = "old_id_age1_known_#{i}"
+          log.old_form_id = "100#{i}"
+          log.save!
+          expected_answers_examples_content.sub!("{id#{i}}", log.id.to_s)
+          expected_answers_examples_content.sub!("{org_id#{i}}", log.owning_organisation_id.to_s)
+        end
+        create_list(:sales_log, 4, :completed, beds: nil) do |log, i|
+          log.old_id = "old_id_beds_#{i}"
+          log.old_form_id = "200#{i}"
+          expected_answers_examples_content.sub!("{id2_#{i}}", log.id.to_s)
+          expected_answers_examples_content.sub!("{org_id2_#{i}}", log.owning_organisation_id.to_s)
+          log.save!
+        end
+        create(:sales_log, :completed, age1_known: nil, beds: nil, old_id: "beds_and_age", old_form_id: "300") do |log|
+          expected_answers_examples_content.sub!("{id}", log.id.to_s)
+          expected_answers_examples_content.sub!("{org_id}", log.owning_organisation_id.to_s)
+        end
+
+        create_list(:sales_log, 2, :completed, age1_known: nil)
+      end
+
+      it "generates a csv with expected missing fields" do
+        expect(storage_service).to receive(:write_file).with("MissingAnswersReportLettingsLog_report_suffix.csv", "\uFEFFMissing answers,Total number of affected logs\n")
+        expect(storage_service).to receive(:write_file).with("MissingAnswersExamplesLettingsLog_report_suffix.csv", "\uFEFFMissing answers,Organisation ID,Log ID,Old Form ID,Old Log ID\n")
+        expect(storage_service).to receive(:write_file).with("MissingAnswersReportSalesLog_report_suffix.csv", "#{expected_content}")
+        expect(storage_service).to receive(:write_file).with("MissingAnswersExamplesSalesLog_report_suffix.csv", "#{expected_answers_examples_content}")
+
+        report_service.generate_missing_answers_report("report_suffix")
+      end
+    end
+  end
 end

spec/services/imports/lettings_logs_field_import_service_spec.rb

@@ -323,4 +323,226 @@ RSpec.describe Imports::LettingsLogsFieldImportService do
       end
     end
   end
+
+  context "when updating address" do
+    let(:field) { "address" }
+
+    before do
+      WebMock.stub_request(:get, /api.postcodes.io\/postcodes\/B11BB/)
+      .to_return(status: 200, body: '{"status":200,"result":{"admin_district":"Westminster","codes":{"admin_district":"E08000035"}}}', headers: {})
+
+      stub_request(:get, "https://api.os.uk/search/places/v1/uprn?key=OS_DATA_KEY&uprn=123")
+        .to_return(status: 500, body: "{}", headers: {})
+
+      Timecop.freeze(2023, 5, 5)
+      Singleton.__init__(FormHandler)
+      Imports::LettingsLogsImportService.new(storage_service, logger).create_logs(fixture_directory)
+      lettings_log_file.rewind
+    end
+
+    after do
+      Timecop.unfreeze
+      Singleton.__init__(FormHandler)
+    end
+
+    context "when the lettings log has no address values" do
+      let(:lettings_log_id) { "00d2343e-d5fa-4c89-8400-ec3854b0f2b4" }
+      let(:lettings_log) { LettingsLog.find_by(old_id: lettings_log_id) }
+
+      before do
+        lettings_log.update!(uprn_known: nil,
+                             startdate: Time.zone.local(2023, 5, 5),
+                             uprn: nil,
+                             uprn_confirmed: nil,
+                             address_line1: nil,
+                             address_line2: nil,
+                             town_or_city: nil,
+                             county: nil,
+                             postcode_known: nil,
+                             postcode_full: nil,
+                             la: nil,
+                             is_la_inferred: nil)
+      end
+
+      context "and new address values include address" do
+        before do
+          lettings_log_xml.at_xpath("//xmlns:UPRN").content = "123456781234"
+          lettings_log_xml.at_xpath("//xmlns:AddressLine1").content = "address 1"
+          lettings_log_xml.at_xpath("//xmlns:AddressLine2").content = "address 2"
+          lettings_log_xml.at_xpath("//xmlns:TownCity").content = "towncity"
+          lettings_log_xml.at_xpath("//xmlns:County").content = "county"
+          lettings_log_xml.at_xpath("//xmlns:POSTCODE").content = "B1"
+          lettings_log_xml.at_xpath("//xmlns:POSTCOD2").content = "1BB"
+          lettings_log_xml.at_xpath("//xmlns:Q28ONS").content = nil
+        end
+
+        it "updates the lettings_log prioritising address values" do
+          expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} address_line1 value has been set to address 1/)
+          expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} address_line2 value has been set to address 2/)
+          expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} town_or_city value has been set to towncity/)
+          expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} county value has been set to county/)
+          expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} postcode_full value has been set to B1 1BB/)
+          import_service.send(:update_address, lettings_log_xml)
+          lettings_log.reload
+
+          expect(lettings_log.uprn_known).to eq(0)
+          expect(lettings_log.uprn).to eq(nil)
+          expect(lettings_log.uprn_confirmed).to eq(nil)
+          expect(lettings_log.address_line1).to eq("address 1")
+          expect(lettings_log.address_line2).to eq("address 2")
+          expect(lettings_log.town_or_city).to eq("towncity")
+          expect(lettings_log.county).to eq("county")
+          expect(lettings_log.postcode_known).to eq(1)
+          expect(lettings_log.postcode_full).to eq("B1 1BB")
+          expect(lettings_log.la).to eq("E08000035")
+          expect(lettings_log.is_la_inferred).to eq(true)
+        end
+      end
+
+      context "and new address values don't include address" do
+        it "skips the update" do
+          expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} is missing either or both of address_line1 and town or city, skipping/)
+          import_service.send(:update_address, lettings_log_xml)
+        end
+      end
+    end
+
+    context "when the lettings log has address values" do
+      let(:lettings_log_id) { "00d2343e-d5fa-4c89-8400-ec3854b0f2b4" }
+      let(:lettings_log) { LettingsLog.find_by(old_id: lettings_log_id) }
+
+      before do
+        lettings_log_xml.at_xpath("//xmlns:UPRN").content = "123456781234"
+        lettings_log_xml.at_xpath("//xmlns:AddressLine1").content = "address 1"
+        lettings_log_xml.at_xpath("//xmlns:AddressLine2").content = "address 2"
+        lettings_log_xml.at_xpath("//xmlns:TownCity").content = "towncity"
+        lettings_log_xml.at_xpath("//xmlns:County").content = "county"
+        lettings_log_xml.at_xpath("//xmlns:POSTCODE").content = "B1"
+        lettings_log_xml.at_xpath("//xmlns:POSTCOD2").content = "1BC"
+        lettings_log_xml.at_xpath("//xmlns:Q28ONS").content = nil
+        lettings_log.update!(uprn_known: 1,
+                             startdate: Time.zone.local(2023, 5, 5),
+                             uprn: "123",
+                             uprn_confirmed: 0,
+                             address_line1: "wrong address line1",
+                             address_line2: "wrong address 2",
+                             town_or_city: "wrong town",
+                             county: "wrong city",
+                             postcode_known: 1,
+                             postcode_full: "A11AA",
+                             la: "E06000064",
+                             is_la_inferred: true)
+      end
+
+      it "replaces the lettings_log address values" do
+        expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} address_line1 value has been set to address 1/)
+        expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} address_line2 value has been set to address 2/)
+        expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} town_or_city value has been set to towncity/)
+        expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} county value has been set to county/)
+        expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} postcode_full value has been set to B11BC/)
+        import_service.send(:update_address, lettings_log_xml)
+        lettings_log.reload
+
+        expect(lettings_log.uprn_known).to eq(0)
+        expect(lettings_log.uprn).to eq(nil)
+        expect(lettings_log.uprn_confirmed).to eq(nil)
+        expect(lettings_log.address_line1).to eq("address 1")
+        expect(lettings_log.address_line2).to eq("address 2")
+        expect(lettings_log.town_or_city).to eq("towncity")
+        expect(lettings_log.county).to eq("county")
+        expect(lettings_log.postcode_known).to eq(1)
+        expect(lettings_log.postcode_full).to eq("B11BC")
+        expect(lettings_log.la).to eq(nil)
+        expect(lettings_log.is_la_inferred).to eq(false)
+      end
+    end
+
+    context "when the lettings log is from before collection 23/24" do
+      let(:lettings_log_id) { "00d2343e-d5fa-4c89-8400-ec3854b0f2b4" }
+      let(:lettings_log) { LettingsLog.find_by(old_id: lettings_log_id) }
+
+      before do
+        lettings_log.update!(startdate: Time.zone.local(2022, 5, 5))
+      end
+
+      it "skips the update" do
+        expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} is from previous collection year, skipping/)
+        import_service.send(:update_address, lettings_log_xml)
+      end
+    end
+  end
+
+  context "when updating reason" do
+    let(:field) { "reason" }
+
+    context "when the lettings log has no reason value" do
+      let(:lettings_log) { LettingsLog.find_by(old_id: lettings_log_id) }
+
+      before do
+        Imports::LettingsLogsImportService.new(storage_service, logger).create_logs(fixture_directory)
+        lettings_log_file.rewind
+        lettings_log.update!(reason: nil, values_updated_at: nil)
+        lettings_log_xml.at_xpath("//xmlns:Q9a").content = "47"
+      end
+
+      it "updates the lettings_log reason value" do
+        expect(logger).to receive(:info).with(/lettings log \d+'s reason value has been set to 47/)
+        expect { import_service.send(:update_reason, lettings_log_xml) }
+          .to(change { lettings_log.reload.reason }.from(nil).to(47))
+        expect(lettings_log.values_updated_at).not_to be_nil
+      end
+    end
+
+    context "when the lettings log has a different reason value" do
+      let(:lettings_log) { LettingsLog.find_by(old_id: lettings_log_id) }
+
+      before do
+        Imports::LettingsLogsImportService.new(storage_service, logger).create_logs(fixture_directory)
+        lettings_log_file.rewind
+        lettings_log.update!(reason: 18, values_updated_at: nil)
+        lettings_log_xml.at_xpath("//xmlns:Q9a").content = "47"
+      end
+
+      it "does not update the lettings_log reason value" do
+        expect(logger).to receive(:info).with(/lettings log \d+ has a value for reason, skipping update/)
+        expect { import_service.send(:update_reason, lettings_log_xml) }
+          .not_to(change { lettings_log.reload.reason })
+        expect(lettings_log.values_updated_at).to be_nil
+      end
+    end
+
+    context "when the new value is 'other'" do
+      let(:lettings_log) { LettingsLog.find_by(old_id: lettings_log_id) }
+
+      before do
+        Imports::LettingsLogsImportService.new(storage_service, logger).create_logs(fixture_directory)
+        lettings_log_file.rewind
+        lettings_log.update!(reason: nil, values_updated_at: nil)
+        lettings_log_xml.at_xpath("//xmlns:Q9a").content = "20"
+      end
+
+      context "and other value is given" do
+        before do
+          lettings_log_xml.at_xpath("//xmlns:Q9aa").content = "other"
+        end
+
+        it "updates the lettings_log reason value" do
+          expect(logger).to receive(:info).with(/lettings log \d+'s reason value has been set to 20/)
+          expect(logger).to receive(:info).with(/lettings log \d+'s reasonother value has been set to other/)
+          expect { import_service.send(:update_reason, lettings_log_xml) }
+            .to(change { lettings_log.reload.reason }.from(nil).to(20))
+          expect(lettings_log.values_updated_at).not_to be_nil
+        end
+      end
+
+      context "and other value is not given" do
+        it "does not update the lettings_log reason value" do
+          expect(logger).to receive(:info).with(/lettings log \d+'s reason is other but other reason is not provided, skipping update/)
+          expect { import_service.send(:update_reason, lettings_log_xml) }
+            .not_to(change { lettings_log.reload.reason })
+          expect(lettings_log.values_updated_at).to be_nil
+        end
+      end
+    end
+  end
 end

spec/services/imports/lettings_logs_import_service_spec.rb

@@ -22,8 +22,8 @@ RSpec.describe Imports::LettingsLogsImportService do
     let(:real_2022_2023_form) { Form.new("config/forms/2022_2023.json") }
     let(:fixture_directory) { "spec/fixtures/imports/logs" }
 
-    let(:organisation) { FactoryBot.create(:organisation, old_visible_id: "1", provider_type: "PRP") }
-    let(:managing_organisation) { FactoryBot.create(:organisation, old_visible_id: "2", provider_type: "PRP") }
+    let(:organisation) { FactoryBot.create(:organisation, old_org_id: "7c5bd5fb549c09a2c55d7cb90d7ba84927e64618", provider_type: "PRP") }
+    let(:managing_organisation) { FactoryBot.create(:organisation, old_org_id: "7c5bd5fb549c09z2c55d9cb90d7ba84927e64618", provider_type: "PRP") }
     let(:scheme1) { FactoryBot.create(:scheme, old_visible_id: "0123", owning_organisation: organisation) }
     let(:scheme2) { FactoryBot.create(:scheme, old_visible_id: "456", owning_organisation: organisation) }
 
@@ -36,8 +36,8 @@ RSpec.describe Imports::LettingsLogsImportService do
              .to_return(status: 200, body: '{"status":200,"result":{"admin_district":"Westminster","codes":{"admin_district":"E08000035"}}}', headers: {})
 
       allow(Organisation).to receive(:find_by).and_return(nil)
-      allow(Organisation).to receive(:find_by).with(old_visible_id: organisation.old_visible_id).and_return(organisation)
-      allow(Organisation).to receive(:find_by).with(old_visible_id: managing_organisation.old_visible_id).and_return(managing_organisation)
+      allow(Organisation).to receive(:find_by).with(old_org_id: organisation.old_org_id).and_return(organisation)
+      allow(Organisation).to receive(:find_by).with(old_org_id: managing_organisation.old_org_id).and_return(managing_organisation)
 
       # Created by users
       FactoryBot.create(:user, old_user_id: "c3061a2e6ea0b702e6f6210d5c52d2a92612d2aa", organisation:)
@@ -135,6 +135,90 @@ RSpec.describe Imports::LettingsLogsImportService do
       let(:lettings_log_file) { open_file(fixture_directory, lettings_log_id) }
       let(:lettings_log_xml) { Nokogiri::XML(lettings_log_file) }
 
+      context "and the user does not exist" do
+        before { lettings_log_xml.at_xpath("//meta:owner-user-id").content = "fake_id" }
+
+        it "creates a new unassigned user" do
+          expect(logger).to receive(:error).with("Lettings log '0ead17cb-1668-442d-898c-0d52879ff592' belongs to legacy user with owner-user-id: 'fake_id' which cannot be found. Assigning log to 'Unassigned' user.")
+          lettings_log_service.send(:create_log, lettings_log_xml)
+
+          lettings_log = LettingsLog.where(old_id: lettings_log_id).first
+          expect(lettings_log&.created_by&.name).to eq("Unassigned")
+        end
+
+        it "only creates one unassigned user" do
+          expect(logger).to receive(:error).with("Lettings log '0ead17cb-1668-442d-898c-0d52879ff592' belongs to legacy user with owner-user-id: 'fake_id' which cannot be found. Assigning log to 'Unassigned' user.")
+          expect(logger).to receive(:error).with("Lettings log 'fake_id' belongs to legacy user with owner-user-id: 'fake_id' which cannot be found. Assigning log to 'Unassigned' user.")
+          lettings_log_service.send(:create_log, lettings_log_xml)
+          lettings_log_xml.at_xpath("//meta:document-id").content = "fake_id"
+          lettings_log_service.send(:create_log, lettings_log_xml)
+
+          lettings_log = LettingsLog.where(old_id: lettings_log_id).first
+          second_lettings_log = LettingsLog.where(old_id: "fake_id").first
+          expect(lettings_log&.created_by).to eq(second_lettings_log&.created_by)
+        end
+
+        context "when unassigned user exist for a different organisation" do
+          let!(:other_unassigned_user) { create(:user, name: "Unassigned") }
+
+          it "creates a new unassigned user for current organisation" do
+            expect(logger).to receive(:error).with("Lettings log '0ead17cb-1668-442d-898c-0d52879ff592' belongs to legacy user with owner-user-id: 'fake_id' which cannot be found. Assigning log to 'Unassigned' user.")
+            lettings_log_service.send(:create_log, lettings_log_xml)
+
+            lettings_log = LettingsLog.where(old_id: lettings_log_id).first
+            expect(lettings_log&.created_by&.name).to eq("Unassigned")
+            expect(lettings_log&.created_by).not_to eq(other_unassigned_user)
+          end
+        end
+      end
+
+      context "and the user exists on a different organisation" do
+        before do
+          create(:legacy_user, old_user_id: "fake_id")
+          lettings_log_xml.at_xpath("//meta:owner-user-id").content = "fake_id"
+        end
+
+        it "creates a new unassigned user" do
+          expect(logger).to receive(:error).with("Lettings log '0ead17cb-1668-442d-898c-0d52879ff592' belongs to legacy user with owner-user-id: 'fake_id' which belongs to a different organisation. Assigning log to 'Unassigned' user.")
+          lettings_log_service.send(:create_log, lettings_log_xml)
+
+          lettings_log = LettingsLog.where(old_id: lettings_log_id).first
+          expect(lettings_log&.created_by&.name).to eq("Unassigned")
+        end
+
+        it "only creates one unassigned user" do
+          expect(logger).to receive(:error).with("Lettings log '0ead17cb-1668-442d-898c-0d52879ff592' belongs to legacy user with owner-user-id: 'fake_id' which belongs to a different organisation. Assigning log to 'Unassigned' user.")
+          expect(logger).to receive(:error).with("Lettings log 'fake_id' belongs to legacy user with owner-user-id: 'fake_id' which belongs to a different organisation. Assigning log to 'Unassigned' user.")
+          lettings_log_service.send(:create_log, lettings_log_xml)
+          lettings_log_xml.at_xpath("//meta:document-id").content = "fake_id"
+          lettings_log_service.send(:create_log, lettings_log_xml)
+
+          lettings_log = LettingsLog.where(old_id: lettings_log_id).first
+          second_lettings_log = LettingsLog.where(old_id: "fake_id").first
+          expect(lettings_log&.created_by).to eq(second_lettings_log&.created_by)
+        end
+
+        context "when unassigned user exist for a different organisation" do
+          let!(:other_unassigned_user) { create(:user, name: "Unassigned") }
+
+          it "creates a new unassigned user for current organisation" do
+            expect(logger).to receive(:error).with("Lettings log '0ead17cb-1668-442d-898c-0d52879ff592' belongs to legacy user with owner-user-id: 'fake_id' which belongs to a different organisation. Assigning log to 'Unassigned' user.")
+            lettings_log_service.send(:create_log, lettings_log_xml)
+
+            lettings_log = LettingsLog.where(old_id: lettings_log_id).first
+            expect(lettings_log&.created_by&.name).to eq("Unassigned")
+            expect(lettings_log&.created_by).not_to eq(other_unassigned_user)
+          end
+        end
+      end
+
+      it "correctly sets imported at date" do
+        lettings_log_service.send(:create_log, lettings_log_xml)
+
+        lettings_log = LettingsLog.where(old_id: lettings_log_id).first
+        expect(lettings_log&.values_updated_at).to eq(Time.zone.local(2022, 1, 1))
+      end
+
       context "and the void date is after the start date" do
         before { lettings_log_xml.at_xpath("//xmlns:VYEAR").content = 2023 }
 
@@ -147,11 +231,11 @@ RSpec.describe Imports::LettingsLogsImportService do
       end
 
       context "and the organisation legacy ID does not exist" do
-        before { lettings_log_xml.at_xpath("//xmlns:OWNINGORGID").content = 99_999 }
+        before { lettings_log_xml.at_xpath("//meta:owner-institution-id").content = 99_999 }
 
         it "raises an exception" do
           expect { lettings_log_service.send(:create_log, lettings_log_xml) }
-            .to raise_error(RuntimeError, "Organisation not found with legacy ID 99999")
+            .to raise_error(RuntimeError, "Organisation not found with old org ID 99999")
         end
       end
 
@@ -241,7 +325,7 @@ RSpec.describe Imports::LettingsLogsImportService do
             .not_to raise_error
         end
 
-        it "clears out the invalid answers" do
+        it "clears out the invalid tenancy length" do
           allow(logger).to receive(:warn)
 
           lettings_log_service.send(:create_log, lettings_log_xml)
@@ -249,7 +333,7 @@ RSpec.describe Imports::LettingsLogsImportService do
 
           expect(lettings_log).not_to be_nil
           expect(lettings_log.tenancylength).to be_nil
-          expect(lettings_log.tenancy).to be_nil
+          expect(lettings_log.tenancy).to eq(4)
         end
       end
 
@@ -442,6 +526,105 @@ RSpec.describe Imports::LettingsLogsImportService do
         end
       end
 
+      context "and the number of times the property was relet is 0.00" do
+        before do
+          lettings_log_xml.at_xpath("//xmlns:Q20").content = "0.00"
+        end
+
+        it "does not raise an error" do
+          expect { lettings_log_service.send(:create_log, lettings_log_xml) }
+            .not_to raise_error
+        end
+
+        it "does not clear offered answer" do
+          lettings_log_service.send(:create_log, lettings_log_xml)
+          lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
+
+          expect(lettings_log).not_to be_nil
+          expect(lettings_log.offered).to equal(0)
+        end
+      end
+
+      context "and the number of times the property was relet is 0.01" do
+        before do
+          lettings_log_xml.at_xpath("//xmlns:Q20").content = "0.01"
+        end
+
+        it "intercepts the relevant validation error" do
+          expect { lettings_log_service.send(:create_log, lettings_log_xml) }
+            .not_to raise_error
+        end
+
+        it "clears out the number offered answer" do
+          allow(logger).to receive(:warn)
+
+          lettings_log_service.send(:create_log, lettings_log_xml)
+          lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
+
+          expect(lettings_log).not_to be_nil
+          expect(lettings_log.offered).to be_nil
+        end
+      end
+
+      context "and the number of times the property was relet is a non nil string that is nil as a BigDecimal" do
+        before do
+          lettings_log_xml.at_xpath("//xmlns:Q20").content = "a"
+        end
+
+        it "doesn't throw an error" do
+          expect { lettings_log_service.send(:create_log, lettings_log_xml) }
+            .not_to raise_error
+        end
+
+        it "clears out the number offered answer" do
+          allow(logger).to receive(:warn)
+
+          lettings_log_service.send(:create_log, lettings_log_xml)
+          lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
+
+          expect(lettings_log).not_to be_nil
+          expect(lettings_log.offered).to be_nil
+        end
+      end
+
+      context "and the gender identity is refused" do
+        before do
+          lettings_log_xml.at_xpath("//xmlns:P1Sex").content = "Person prefers not to say"
+        end
+
+        it "does not raise an error" do
+          expect { lettings_log_service.send(:create_log, lettings_log_xml) }
+            .not_to raise_error
+        end
+
+        it "saves the correct answer" do
+          lettings_log_service.send(:create_log, lettings_log_xml)
+          lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
+
+          expect(lettings_log).not_to be_nil
+          expect(lettings_log.sex1).to eq("R")
+        end
+      end
+
+      context "and the relationship is refused" do
+        before do
+          lettings_log_xml.at_xpath("//xmlns:P2Rel").content = "Person prefers not to say"
+        end
+
+        it "does not raise an error" do
+          expect { lettings_log_service.send(:create_log, lettings_log_xml) }
+            .not_to raise_error
+        end
+
+        it "saves the correct answer" do
+          lettings_log_service.send(:create_log, lettings_log_xml)
+          lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
+
+          expect(lettings_log).not_to be_nil
+          expect(lettings_log.relat2).to eq("R")
+        end
+      end
+
       context "when the log being imported was manually entered" do
         it "sets the creation method correctly" do
           lettings_log_service.send(:create_log, lettings_log_xml)
@@ -634,6 +817,29 @@ RSpec.describe Imports::LettingsLogsImportService do
         end
       end
 
+      context "and scharge is ever so slightly positive" do
+        let(:lettings_log_id) { "0b4a68df-30cc-474a-93c0-a56ce8fdad3b" }
+
+        before do
+          lettings_log_xml.at_xpath("//xmlns:Q18aii").content = "1.66533E-16"
+        end
+
+        it "does not raise an error" do
+          expect { lettings_log_service.send(:create_log, lettings_log_xml) }
+            .not_to raise_error
+        end
+
+        it "sets scharge to 0" do
+          allow(logger).to receive(:warn)
+
+          lettings_log_service.send(:create_log, lettings_log_xml)
+          lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
+
+          expect(lettings_log).not_to be_nil
+          expect(lettings_log.scharge).to eq(0)
+        end
+      end
+
       context "and tshortfall is not positive" do
         let(:lettings_log_id) { "0b4a68df-30cc-474a-93c0-a56ce8fdad3b" }
 
@@ -685,7 +891,7 @@ RSpec.describe Imports::LettingsLogsImportService do
       context "and pscharge is out of range" do
         before do
           lettings_log_xml.at_xpath("//xmlns:Q17").content = "1"
-          lettings_log_xml.at_xpath("//xmlns:Q18aiii").content = "36"
+          lettings_log_xml.at_xpath("//xmlns:Q18aiii").content = "701"
         end
 
         it "intercepts the relevant validation error" do
@@ -708,7 +914,7 @@ RSpec.describe Imports::LettingsLogsImportService do
       context "and supcharg is out of range" do
         before do
           lettings_log_xml.at_xpath("//xmlns:Q17").content = "1"
-          lettings_log_xml.at_xpath("//xmlns:Q18aiv").content = "46"
+          lettings_log_xml.at_xpath("//xmlns:Q18aiv").content = "801"
         end
 
         it "intercepts the relevant validation error" do
@@ -731,7 +937,7 @@ RSpec.describe Imports::LettingsLogsImportService do
       context "and scharge is out of range" do
         before do
           lettings_log_xml.at_xpath("//xmlns:Q17").content = "1"
-          lettings_log_xml.at_xpath("//xmlns:Q18aii").content = "156"
+          lettings_log_xml.at_xpath("//xmlns:Q18aii").content = "801"
         end
 
         it "intercepts the relevant validation error" do
@@ -1063,6 +1269,25 @@ RSpec.describe Imports::LettingsLogsImportService do
         end
       end
 
+      context "and the scheme and location are not valid" do
+        let(:lettings_log_id) { "0b4a68df-30cc-474a-93c0-a56ce8fdad3b" }
+
+        before do
+          lettings_log_xml.at_xpath("//xmlns:_1cmangroupcode").content = "999"
+          lettings_log_xml.at_xpath("//xmlns:_1cschemecode").content = "999"
+        end
+
+        it "saves log without location and scheme" do
+          expect(logger).not_to receive(:warn)
+          lettings_log_service.send(:create_log, lettings_log_xml)
+          lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
+
+          expect(lettings_log.scheme_id).to be_nil
+          expect(lettings_log.location_id).to be_nil
+          expect(lettings_log.status).to eq("in_progress")
+        end
+      end
+
       context "and this is a supported housing log with a single location under a scheme" do
         let(:lettings_log_id) { "0b4a68df-30cc-474a-93c0-a56ce8fdad3b" }
 
@@ -1078,6 +1303,226 @@ RSpec.describe Imports::LettingsLogsImportService do
           expect(lettings_log.status).to eq("completed")
         end
       end
+
+      context "and there are several household members" do
+        context "and one person details are skipped" do
+          before do
+            lettings_log_xml.at_xpath("//xmlns:HHMEMB").content = 3
+            lettings_log_xml.at_xpath("//xmlns:P2AR").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Age").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Sex").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Rel").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Eco").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P3Age").content = 7
+            lettings_log_xml.at_xpath("//xmlns:P3Sex").content = "Male"
+            lettings_log_xml.at_xpath("//xmlns:P3Rel").content = "Child"
+            lettings_log_xml.at_xpath("//xmlns:P3Eco").content = "9) Child under 16"
+          end
+
+          it "correctly moves person details" do
+            lettings_log_service.send(:create_log, lettings_log_xml)
+
+            lettings_log = LettingsLog.where(old_id: lettings_log_id).first
+            expect(lettings_log&.hhmemb).to eq(3)
+            expect(lettings_log&.details_known_2).to eq(0)
+            expect(lettings_log&.age2_known).to eq(0)
+            expect(lettings_log&.age2).to eq(7)
+            expect(lettings_log&.sex2).to eq("M")
+            expect(lettings_log&.relat2).to eq("C")
+
+            expect(lettings_log&.details_known_3).to eq(0)
+            expect(lettings_log&.age3_known).to eq(0)
+            expect(lettings_log&.age3).to eq(nil)
+            expect(lettings_log&.sex3).to eq(nil)
+            expect(lettings_log&.relat3).to eq(nil)
+
+            [4, 5].each do |i|
+              expect(lettings_log&.send("details_known_#{i}")).to eq(nil)
+              expect(lettings_log&.send("age#{i}_known")).to eq(nil)
+              expect(lettings_log&.send("age#{i}")).to eq(nil)
+              expect(lettings_log&.send("sex#{i}")).to eq(nil)
+              expect(lettings_log&.send("relat#{i}")).to eq(nil)
+              expect(lettings_log&.send("ecstat#{i}")).to eq(nil)
+            end
+          end
+        end
+
+        context "and several consecutive person details are skipped" do
+          before do
+            lettings_log_xml.at_xpath("//xmlns:HHMEMB").content = 4
+
+            lettings_log_xml.at_xpath("//xmlns:P2AR").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Age").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Sex").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Rel").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Eco").content = nil
+
+            lettings_log_xml.at_xpath("//xmlns:P3AR").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P3Age").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P3Sex").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P3Rel").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P3Eco").content = nil
+
+            lettings_log_xml.at_xpath("//xmlns:P4Age").content = 7
+            lettings_log_xml.at_xpath("//xmlns:P4Sex").content = "Male"
+            lettings_log_xml.at_xpath("//xmlns:P4Rel").content = "Child"
+            lettings_log_xml.at_xpath("//xmlns:P4Eco").content = "9) Child under 16"
+
+            lettings_log_xml.at_xpath("//xmlns:P5Age").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P5Sex").content = "Male"
+            lettings_log_xml.at_xpath("//xmlns:P5Rel").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P5Eco").content = nil
+
+            lettings_log_xml.at_xpath("//xmlns:P6Age").content = 12
+            lettings_log_xml.at_xpath("//xmlns:P6Sex").content = "Female"
+            lettings_log_xml.at_xpath("//xmlns:P6Rel").content = "Child"
+            lettings_log_xml.at_xpath("//xmlns:P6Eco").content = "9) Child under 16"
+          end
+
+          it "correctly moves person details" do
+            lettings_log_service.send(:create_log, lettings_log_xml)
+
+            lettings_log = LettingsLog.where(old_id: lettings_log_id).first
+            expect(lettings_log&.hhmemb).to eq(4)
+            expect(lettings_log&.details_known_2).to eq(0)
+            expect(lettings_log&.age2_known).to eq(0)
+            expect(lettings_log&.age2).to eq(7)
+            expect(lettings_log&.sex2).to eq("M")
+            expect(lettings_log&.relat2).to eq("C")
+
+            expect(lettings_log&.details_known_3).to eq(0)
+            expect(lettings_log&.age3_known).to eq(0)
+            expect(lettings_log&.age3).to eq(nil)
+            expect(lettings_log&.sex3).to eq("M")
+            expect(lettings_log&.relat3).to eq(nil)
+
+            expect(lettings_log&.details_known_4).to eq(0)
+            expect(lettings_log&.age4_known).to eq(0)
+            expect(lettings_log&.age4).to eq(12)
+            expect(lettings_log&.sex4).to eq("F")
+            expect(lettings_log&.relat4).to eq("C")
+
+            [5, 6, 7, 8].each do |i|
+              expect(lettings_log&.send("details_known_#{i}")).to eq(nil)
+              expect(lettings_log&.send("age#{i}_known")).to eq(nil)
+              expect(lettings_log&.send("age#{i}")).to eq(nil)
+              expect(lettings_log&.send("sex#{i}")).to eq(nil)
+              expect(lettings_log&.send("relat#{i}")).to eq(nil)
+              expect(lettings_log&.send("ecstat#{i}")).to eq(nil)
+            end
+          end
+        end
+
+        context "and several non consecutive person details are skipped" do
+          before do
+            lettings_log_xml.at_xpath("//xmlns:HHMEMB").content = 4
+            lettings_log_xml.at_xpath("//xmlns:P2AR").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Age").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Sex").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Rel").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Eco").content = nil
+
+            lettings_log_xml.at_xpath("//xmlns:P3Age").content = 7
+            lettings_log_xml.at_xpath("//xmlns:P3Sex").content = "Male"
+            lettings_log_xml.at_xpath("//xmlns:P3Rel").content = "Child"
+            lettings_log_xml.at_xpath("//xmlns:P3Eco").content = "9) Child under 16"
+
+            lettings_log_xml.at_xpath("//xmlns:P4AR").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P4Age").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P4Sex").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P4Rel").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P4Eco").content = nil
+
+            lettings_log_xml.at_xpath("//xmlns:P5Age").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P5Sex").content = "Male"
+            lettings_log_xml.at_xpath("//xmlns:P5Rel").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P5Eco").content = nil
+
+            lettings_log_xml.at_xpath("//xmlns:P6Age").content = 12
+            lettings_log_xml.at_xpath("//xmlns:P6Sex").content = "Female"
+            lettings_log_xml.at_xpath("//xmlns:P6Rel").content = "Child"
+            lettings_log_xml.at_xpath("//xmlns:P6Eco").content = "9) Child under 16"
+          end
+
+          it "correctly moves person details" do
+            lettings_log_service.send(:create_log, lettings_log_xml)
+
+            lettings_log = LettingsLog.where(old_id: lettings_log_id).first
+            expect(lettings_log&.hhmemb).to eq(4)
+            expect(lettings_log&.details_known_2).to eq(0)
+            expect(lettings_log&.age2_known).to eq(0)
+            expect(lettings_log&.age2).to eq(7)
+            expect(lettings_log&.sex2).to eq("M")
+            expect(lettings_log&.relat2).to eq("C")
+
+            expect(lettings_log&.details_known_3).to eq(0)
+            expect(lettings_log&.age3_known).to eq(0)
+            expect(lettings_log&.age3).to eq(nil)
+            expect(lettings_log&.sex3).to eq("M")
+            expect(lettings_log&.relat3).to eq(nil)
+
+            expect(lettings_log&.details_known_4).to eq(0)
+            expect(lettings_log&.age4_known).to eq(0)
+            expect(lettings_log&.age4).to eq(12)
+            expect(lettings_log&.sex4).to eq("F")
+            expect(lettings_log&.relat4).to eq("C")
+
+            [5, 6, 7, 8].each do |i|
+              expect(lettings_log&.send("details_known_#{i}")).to eq(nil)
+              expect(lettings_log&.send("age#{i}_known")).to eq(nil)
+              expect(lettings_log&.send("age#{i}")).to eq(nil)
+              expect(lettings_log&.send("sex#{i}")).to eq(nil)
+              expect(lettings_log&.send("relat#{i}")).to eq(nil)
+              expect(lettings_log&.send("ecstat#{i}")).to eq(nil)
+            end
+          end
+        end
+
+        context "with 3 houusehold members without any person data" do
+          before do
+            lettings_log_xml.at_xpath("//xmlns:HHMEMB").content = 3
+            lettings_log_xml.at_xpath("//xmlns:P2AR").content = "No"
+            lettings_log_xml.at_xpath("//xmlns:P2Age").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Sex").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Rel").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Eco").content = nil
+
+            lettings_log_xml.at_xpath("//xmlns:P3AR").content = "No"
+            lettings_log_xml.at_xpath("//xmlns:P3Age").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P3Sex").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P3Rel").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P3Eco").content = nil
+          end
+
+          it "correctly sets person details" do
+            lettings_log_service.send(:create_log, lettings_log_xml)
+
+            lettings_log = LettingsLog.where(old_id: lettings_log_id).first
+            expect(lettings_log&.hhmemb).to eq(3)
+
+            expect(lettings_log&.details_known_2).to eq(0)
+            expect(lettings_log&.age2_known).to eq(1)
+            expect(lettings_log&.age2).to eq(nil)
+            expect(lettings_log&.sex2).to eq(nil)
+            expect(lettings_log&.relat2).to eq(nil)
+
+            expect(lettings_log&.details_known_3).to eq(0)
+            expect(lettings_log&.age3_known).to eq(1)
+            expect(lettings_log&.age3).to eq(nil)
+            expect(lettings_log&.sex3).to eq(nil)
+            expect(lettings_log&.relat3).to eq(nil)
+
+            [4, 5, 6, 7, 8].each do |i|
+              expect(lettings_log&.send("details_known_#{i}")).to eq(nil)
+              expect(lettings_log&.send("age#{i}_known")).to eq(nil)
+              expect(lettings_log&.send("age#{i}")).to eq(nil)
+              expect(lettings_log&.send("sex#{i}")).to eq(nil)
+              expect(lettings_log&.send("relat#{i}")).to eq(nil)
+              expect(lettings_log&.send("ecstat#{i}")).to eq(nil)
+            end
+          end
+        end
+      end
     end
   end
 
@@ -1102,10 +1547,8 @@ RSpec.describe Imports::LettingsLogsImportService do
     let(:real_2022_2023_form) { Form.new("config/forms/2022_2023.json") }
     let(:fixture_directory) { "spec/fixtures/imports/logs" }
 
-    let(:organisation) { FactoryBot.create(:organisation, old_visible_id: "1", provider_type: "PRP") }
-    let(:managing_organisation) { FactoryBot.create(:organisation, old_visible_id: "2", provider_type: "PRP") }
-    let(:local_authority_organisation) { FactoryBot.create(:organisation, old_visible_id: "79740", provider_type: "LA") }
-    let(:local_authority_organisation_managing_organisation) { FactoryBot.create(:organisation, old_visible_id: "79741", provider_type: "PRP") }
+    let(:organisation) { FactoryBot.create(:organisation, old_org_id: "7c5bd5fb549c09a2c55d7cb90d7ba84927e64618", provider_type: "PRP") }
+    let(:managing_organisation) { FactoryBot.create(:organisation, old_org_id: "7c5bd5fb549c09z2c55d9cb90d7ba84927e64618", provider_type: "PRP") }
     let(:scheme1) { FactoryBot.create(:scheme, old_visible_id: "0123", owning_organisation: organisation) }
     let(:scheme2) { FactoryBot.create(:scheme, old_visible_id: "456", owning_organisation: organisation) }
 
@@ -1118,9 +1561,8 @@ RSpec.describe Imports::LettingsLogsImportService do
              .to_return(status: 200, body: '{"status":200,"result":{"admin_district":"Westminster","codes":{"admin_district":"E08000035"}}}', headers: {})
 
       allow(Organisation).to receive(:find_by).and_return(nil)
-      allow(Organisation).to receive(:find_by).with(old_visible_id: organisation.old_visible_id).and_return(organisation)
-      allow(Organisation).to receive(:find_by).with(old_visible_id: managing_organisation.old_visible_id).and_return(managing_organisation)
-      allow(Organisation).to receive(:find_by).with(old_visible_id: local_authority_organisation.old_visible_id).and_return(local_authority_organisation)
+      allow(Organisation).to receive(:find_by).with(old_org_id: organisation.old_org_id).and_return(organisation)
+      allow(Organisation).to receive(:find_by).with(old_org_id: managing_organisation.old_org_id).and_return(managing_organisation)
 
       # Created by users
       FactoryBot.create(:user, old_user_id: "c3061a2e6ea0b702e6f6210d5c52d2a92612d2aa", organisation:)
@@ -1351,7 +1793,7 @@ RSpec.describe Imports::LettingsLogsImportService do
         .not_to raise_error
       end
 
-      it "clears out the referral answer" do
+      it "clears out the period answer" do
         allow(logger).to receive(:warn)
         lettings_log_service.send(:create_log, lettings_log_xml)
         lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
@@ -1629,5 +2071,59 @@ RSpec.describe Imports::LettingsLogsImportService do
         expect(lettings_log&.la).to eq("E06000047")
       end
     end
+
+    context "and the referral soft validation is triggered (referral_value_check)" do
+      let(:lettings_log_id) { "00d2343e-d5fa-4c89-8400-ec3854b0f2b4" }
+      let(:lettings_log_file) { open_file(fixture_directory, lettings_log_id) }
+      let(:lettings_log_xml) { Nokogiri::XML(lettings_log_file) }
+
+      before do
+        lettings_log_xml.at_xpath("//xmlns:DAY").content = "2"
+        lettings_log_xml.at_xpath("//xmlns:MONTH").content = "10"
+        lettings_log_xml.at_xpath("//xmlns:YEAR").content = "2022"
+        lettings_log_xml.at_xpath("//xmlns:Q16").content = 4
+      end
+
+      it "completes the log" do
+        lettings_log_service.send(:create_log, lettings_log_xml)
+        lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
+        expect(lettings_log.status).to eq("completed")
+      end
+    end
+
+    context "and the scharge/pscharge/supcharg soft validations are triggered" do
+      let(:lettings_log_id) { "00d2343e-d5fa-4c89-8400-ec3854b0f2b4" }
+      let(:lettings_log_file) { open_file(fixture_directory, lettings_log_id) }
+      let(:lettings_log_xml) { Nokogiri::XML(lettings_log_file) }
+
+      around do |example|
+        Timecop.freeze(Time.zone.local(2023, 4, 1)) do
+          Singleton.__init__(FormHandler)
+          example.run
+        end
+        Timecop.return
+        Singleton.__init__(FormHandler)
+      end
+
+      before do
+        lettings_log_xml.at_xpath("//xmlns:AddressLine1").content = "address 1"
+        lettings_log_xml.at_xpath("//xmlns:TownCity").content = "towncity"
+        lettings_log_xml.at_xpath("//xmlns:DAY").content = "10"
+        lettings_log_xml.at_xpath("//xmlns:MONTH").content = "4"
+        lettings_log_xml.at_xpath("//xmlns:YEAR").content = "2023"
+        lettings_log_xml.at_xpath("//xmlns:P1Nat").content = "18"
+
+        lettings_log_xml.at_xpath("//xmlns:Q17").content = "1"
+        lettings_log_xml.at_xpath("//xmlns:Q18aii").content = "800"
+        lettings_log_xml.at_xpath("//xmlns:Q18aiii").content = "300"
+        lettings_log_xml.at_xpath("//xmlns:Q18aiv").content = "300"
+      end
+
+      it "completes the log" do
+        lettings_log_service.send(:create_log, lettings_log_xml)
+        lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
+        expect(lettings_log.status).to eq("completed")
+      end
+    end
   end
 end

spec/services/imports/sales_logs_field_import_service_spec.rb

@@ -9,7 +9,7 @@ RSpec.describe Imports::SalesLogsFieldImportService do
   let(:fixture_directory) { "spec/fixtures/imports/sales_logs" }
   let(:sales_log_filename) { "shared_ownership_sales_log" }
   let(:sales_log_file) { File.open("#{fixture_directory}/#{sales_log_filename}.xml") }
-  let(:organisation) { create(:organisation, old_visible_id: "1") }
+  let(:organisation) { create(:organisation, old_visible_id: "1", old_org_id: "7c5bd5fb549c09a2c55d7cb90d7ba84927e64618") }
   let(:old_user_id) { "c3061a2e6ea0b702e6f6210d5c52d2a92612d2aa" }
 
   let(:remote_folder) { "sales_logs" }

spec/services/imports/sales_logs_import_service_spec.rb

@@ -8,8 +8,8 @@ RSpec.describe Imports::SalesLogsImportService do
 
   let(:fixture_directory) { "spec/fixtures/imports/sales_logs" }
 
-  let(:organisation) { FactoryBot.create(:organisation, old_visible_id: "1", provider_type: "PRP") }
-  let(:managing_organisation) { FactoryBot.create(:organisation, old_visible_id: "2", provider_type: "PRP") }
+  let(:organisation) { FactoryBot.create(:organisation, old_org_id: "7c5bd5fb549c09a2c55d7cb90d7ba84927e64618", provider_type: "PRP") }
+  let(:managing_organisation) { FactoryBot.create(:organisation, old_org_id: "7c5bd5fb549c09z2c55d9cb90d7ba84927e64618", provider_type: "PRP") }
   let(:remote_folder) { "sales_logs" }
 
   def open_file(directory, filename)
@@ -27,8 +27,8 @@ RSpec.describe Imports::SalesLogsImportService do
 
   before do
     allow(Organisation).to receive(:find_by).and_return(nil)
-    allow(Organisation).to receive(:find_by).with(old_visible_id: organisation.old_visible_id).and_return(organisation)
-    allow(Organisation).to receive(:find_by).with(old_visible_id: managing_organisation.old_visible_id).and_return(managing_organisation)
+    allow(Organisation).to receive(:find_by).with(old_org_id: organisation.old_org_id).and_return(organisation)
+    allow(Organisation).to receive(:find_by).with(old_org_id: managing_organisation.old_org_id).and_return(managing_organisation)
 
     # Created by users
     FactoryBot.create(:user, old_user_id: "c3061a2e6ea0b702e6f6210d5c52d2a92612d2aa", organisation:)
@@ -102,17 +102,106 @@ RSpec.describe Imports::SalesLogsImportService do
   end
 
   context "when importing a specific log" do
+    let(:sales_log_id) { "shared_ownership_sales_log" }
     let(:sales_log_file) { open_file(fixture_directory, sales_log_id) }
     let(:sales_log_xml) { Nokogiri::XML(sales_log_file) }
 
+    it "correctly sets values updated at date" do
+      sales_log_service.send(:create_log, sales_log_xml)
+
+      sales_log = SalesLog.where(old_id: sales_log_id).first
+      expect(sales_log&.values_updated_at).to eq(Time.zone.local(2023, 2, 1))
+    end
+
     context "and the organisation legacy ID does not exist" do
       let(:sales_log_id) { "shared_ownership_sales_log" }
 
-      before { sales_log_xml.at_xpath("//xmlns:OWNINGORGID").content = 99_999 }
+      before { sales_log_xml.at_xpath("//meta:owner-institution-id").content = 99_999 }
 
       it "raises an exception" do
         expect { sales_log_service.send(:create_log, sales_log_xml) }
-          .to raise_error(RuntimeError, "Organisation not found with legacy ID 99999")
+          .to raise_error(RuntimeError, "Organisation not found with old org ID 99999")
+      end
+    end
+
+    context "and the user does not exist" do
+      let(:sales_log_id) { "shared_ownership_sales_log" }
+
+      before { sales_log_xml.at_xpath("//meta:owner-user-id").content = "fake_id" }
+
+      it "creates a new unassigned user" do
+        expect(logger).to receive(:error).with("Sales log 'shared_ownership_sales_log' belongs to legacy user with owner-user-id: 'fake_id' which cannot be found. Assigning log to 'Unassigned' user.")
+        sales_log_service.send(:create_log, sales_log_xml)
+
+        sales_log = SalesLog.where(old_id: sales_log_id).first
+        expect(sales_log&.created_by&.name).to eq("Unassigned")
+      end
+
+      it "only creates one unassigned user" do
+        expect(logger).to receive(:error).with("Sales log 'shared_ownership_sales_log' belongs to legacy user with owner-user-id: 'fake_id' which cannot be found. Assigning log to 'Unassigned' user.")
+        expect(logger).to receive(:error).with("Sales log 'fake_id' belongs to legacy user with owner-user-id: 'fake_id' which cannot be found. Assigning log to 'Unassigned' user.")
+        sales_log_service.send(:create_log, sales_log_xml)
+        sales_log_xml.at_xpath("//meta:document-id").content = "fake_id"
+        sales_log_service.send(:create_log, sales_log_xml)
+
+        sales_log = SalesLog.where(old_id: sales_log_id).first
+        second_sales_log = SalesLog.where(old_id: "fake_id").first
+        expect(sales_log&.created_by).to eq(second_sales_log&.created_by)
+      end
+
+      context "when unassigned user exist for a different organisation" do
+        let!(:other_unassigned_user) { create(:user, name: "Unassigned") }
+
+        it "creates a new unassigned user for current organisation" do
+          expect(logger).to receive(:error).with("Sales log 'shared_ownership_sales_log' belongs to legacy user with owner-user-id: 'fake_id' which cannot be found. Assigning log to 'Unassigned' user.")
+          sales_log_service.send(:create_log, sales_log_xml)
+
+          sales_log = SalesLog.where(old_id: sales_log_id).first
+          expect(sales_log&.created_by&.name).to eq("Unassigned")
+          expect(sales_log&.created_by).not_to eq(other_unassigned_user)
+        end
+      end
+    end
+
+    context "and the user exists on a different organisation" do
+      let(:sales_log_id) { "shared_ownership_sales_log" }
+
+      before do
+        create(:legacy_user, old_user_id: "fake_id")
+        sales_log_xml.at_xpath("//meta:owner-user-id").content = "fake_id"
+      end
+
+      it "creates a new unassigned user" do
+        expect(logger).to receive(:error).with("Sales log 'shared_ownership_sales_log' belongs to legacy user with owner-user-id: 'fake_id' which belongs to a different organisation. Assigning log to 'Unassigned' user.")
+        sales_log_service.send(:create_log, sales_log_xml)
+
+        sales_log = SalesLog.where(old_id: sales_log_id).first
+        expect(sales_log&.created_by&.name).to eq("Unassigned")
+      end
+
+      it "only creates one unassigned user" do
+        expect(logger).to receive(:error).with("Sales log 'shared_ownership_sales_log' belongs to legacy user with owner-user-id: 'fake_id' which belongs to a different organisation. Assigning log to 'Unassigned' user.")
+        expect(logger).to receive(:error).with("Sales log 'fake_id' belongs to legacy user with owner-user-id: 'fake_id' which belongs to a different organisation. Assigning log to 'Unassigned' user.")
+        sales_log_service.send(:create_log, sales_log_xml)
+        sales_log_xml.at_xpath("//meta:document-id").content = "fake_id"
+        sales_log_service.send(:create_log, sales_log_xml)
+
+        sales_log = SalesLog.where(old_id: sales_log_id).first
+        second_sales_log = SalesLog.where(old_id: "fake_id").first
+        expect(sales_log&.created_by).to eq(second_sales_log&.created_by)
+      end
+
+      context "when unassigned user exist for a different organisation" do
+        let!(:other_unassigned_user) { create(:user, name: "Unassigned") }
+
+        it "creates a new unassigned user for current organisation" do
+          expect(logger).to receive(:error).with("Sales log 'shared_ownership_sales_log' belongs to legacy user with owner-user-id: 'fake_id' which belongs to a different organisation. Assigning log to 'Unassigned' user.")
+          sales_log_service.send(:create_log, sales_log_xml)
+
+          sales_log = SalesLog.where(old_id: sales_log_id).first
+          expect(sales_log&.created_by&.name).to eq("Unassigned")
+          expect(sales_log&.created_by).not_to eq(other_unassigned_user)
+        end
       end
     end
 
@@ -139,6 +228,31 @@ RSpec.describe Imports::SalesLogsImportService do
       end
     end
 
+    context "and the log startdate is only present in the CompletionDate field" do
+      let(:sales_log_id) { "shared_ownership_sales_log" }
+
+      before do
+        sales_log_xml.at_xpath("//xmlns:DAY").content = nil
+        sales_log_xml.at_xpath("//xmlns:MONTH").content = nil
+        sales_log_xml.at_xpath("//xmlns:YEAR").content = nil
+        sales_log_xml.at_xpath("//xmlns:CompletionDate").content = "2022-10-9"
+        sales_log_xml.at_xpath("//xmlns:HODAY").content = 9
+        sales_log_xml.at_xpath("//xmlns:HOMONTH").content = 10
+        sales_log_xml.at_xpath("//xmlns:HOYEAR").content = 2022
+        sales_log_xml.at_xpath("//xmlns:EXDAY").content = 9
+        sales_log_xml.at_xpath("//xmlns:EXMONTH").content = 10
+        sales_log_xml.at_xpath("//xmlns:EXYEAR").content = 2022
+      end
+
+      it "creates the log with the correct saledate" do
+        expect(logger).not_to receive(:error)
+        expect(logger).not_to receive(:warn)
+        expect { sales_log_service.send(:create_log, sales_log_xml) }
+          .to change(SalesLog, :count).by(1)
+        expect(SalesLog.last.saledate).to eq(Time.zone.local(2022, 10, 9))
+      end
+    end
+
     context "when the log is valid" do
       let(:sales_log_id) { "shared_ownership_sales_log" }
 
@@ -295,6 +409,42 @@ RSpec.describe Imports::SalesLogsImportService do
           sales_log = SalesLog.find_by(old_id: sales_log_id)
           expect(sales_log.proplen_asked).to eq(1)
         end
+
+        context "when setting soctenant fields" do
+          it "does not set soctenant value if none of the soctenant questions are answered" do
+            sales_log_xml.at_xpath("//xmlns:Q20Bedrooms").content = nil
+            sales_log_xml.at_xpath("//xmlns:PrevRentType").content = nil
+            sales_log_xml.at_xpath("//xmlns:Q21PropertyType").content = nil
+
+            expect(logger).not_to receive(:error)
+            expect(logger).not_to receive(:warn)
+            expect(logger).not_to receive(:info)
+            expect { sales_log_service.send(:create_log, sales_log_xml) }
+            .to change(SalesLog, :count).by(1)
+            sales_log = SalesLog.find_by(old_id: sales_log_id)
+            expect(sales_log.soctenant).to eq(nil)
+            expect(sales_log.frombeds).to eq(nil)
+            expect(sales_log.fromprop).to eq(nil)
+            expect(sales_log.socprevten).to eq(nil)
+          end
+
+          it "sets soctenant to don't know if any of the soctenant questions are answered" do
+            sales_log_xml.at_xpath("//xmlns:Q20Bedrooms").content = "2"
+            sales_log_xml.at_xpath("//xmlns:PrevRentType").content = nil
+            sales_log_xml.at_xpath("//xmlns:Q21PropertyType").content = nil
+
+            expect(logger).not_to receive(:error)
+            expect(logger).not_to receive(:warn)
+            expect(logger).not_to receive(:info)
+            expect { sales_log_service.send(:create_log, sales_log_xml) }
+            .to change(SalesLog, :count).by(1)
+            sales_log = SalesLog.find_by(old_id: sales_log_id)
+            expect(sales_log.soctenant).to eq(0)
+            expect(sales_log.frombeds).to eq(2)
+            expect(sales_log.fromprop).to eq(0)
+            expect(sales_log.socprevten).to eq(10)
+          end
+        end
       end
 
       context "with discounted sale type" do
@@ -506,6 +656,56 @@ RSpec.describe Imports::SalesLogsImportService do
       end
     end
 
+    context "and the hodate soft validation is triggered (hodate_value_check)" do
+      let(:sales_log_id) { "shared_ownership_sales_log" }
+
+      before do
+        sales_log_xml.at_xpath("//xmlns:HODAY").content = "1"
+        sales_log_xml.at_xpath("//xmlns:HOMONTH").content = "1"
+        sales_log_xml.at_xpath("//xmlns:HOYEAR").content = "2018"
+      end
+
+      it "completes the log" do
+        sales_log_service.send(:create_log, sales_log_xml)
+        sales_log = SalesLog.find_by(old_id: sales_log_id)
+        expect(sales_log.status).to eq("completed")
+      end
+    end
+
+    context "and the combined income soft validation is triggered (combined_income_value_check)" do
+      let(:sales_log_id) { "shared_ownership_sales_log" }
+
+      before do
+        sales_log_xml.at_xpath("//xmlns:joint").content = "1 Yes"
+        sales_log_xml.at_xpath("//xmlns:Q2Person1Income").content = "45000"
+        sales_log_xml.at_xpath("//xmlns:P2IncKnown").content = "1 Yes"
+        sales_log_xml.at_xpath("//xmlns:Q2Person2Income").content = "45000"
+      end
+
+      it "completes the log" do
+        sales_log_service.send(:create_log, sales_log_xml)
+        sales_log = SalesLog.find_by(old_id: sales_log_id)
+        expect(sales_log.status).to eq("completed")
+      end
+    end
+
+    context "and the stairowned soft validation is triggered (stairowned_value_check)" do
+      let(:sales_log_id) { "shared_ownership_sales_log" }
+
+      before do
+        sales_log_xml.at_xpath("//xmlns:DerSaleType").content = "24"
+        sales_log_xml.at_xpath("//xmlns:PercentOwns").content = "77"
+        sales_log_xml.at_xpath("//xmlns:Q17aStaircase").content = "1"
+        sales_log_xml.at_xpath("//xmlns:PercentBought").content = "10"
+      end
+
+      it "completes the log" do
+        sales_log_service.send(:create_log, sales_log_xml)
+        sales_log = SalesLog.find_by(old_id: sales_log_id)
+        expect(sales_log.status).to eq("completed")
+      end
+    end
+
     context "and it has an invalid record with invalid child age" do
       let(:sales_log_id) { "discounted_ownership_sales_log" }
 
@@ -761,7 +961,7 @@ RSpec.describe Imports::SalesLogsImportService do
           .not_to raise_error
       end
 
-      it "clears out the referral answer" do
+      it "clears out the mortgage answer" do
         allow(logger).to receive(:warn)
 
         sales_log_service.send(:create_log, sales_log_xml)
@@ -772,6 +972,29 @@ RSpec.describe Imports::SalesLogsImportService do
       end
     end
 
+    context "when proplen is out of range" do
+      let(:sales_log_id) { "shared_ownership_sales_log" }
+
+      before do
+        sales_log_xml.at_xpath("//xmlns:Q16aProplen2").content = "2000"
+      end
+
+      it "intercepts the relevant validation error" do
+        expect { sales_log_service.send(:create_log, sales_log_xml) }
+          .not_to raise_error
+      end
+
+      it "clears out the proplen answer" do
+        allow(logger).to receive(:warn)
+
+        sales_log_service.send(:create_log, sales_log_xml)
+        sales_log = SalesLog.find_by(old_id: sales_log_id)
+
+        expect(sales_log).not_to be_nil
+        expect(sales_log.proplen).to be_nil
+      end
+    end
+
     context "and it has an invalid income 1" do
       let(:sales_log_id) { "shared_ownership_sales_log" }
 
@@ -822,6 +1045,34 @@ RSpec.describe Imports::SalesLogsImportService do
       end
     end
 
+    context "and it has an invalid combined income outside london" do
+      let(:sales_log_id) { "shared_ownership_sales_log" }
+
+      before do
+        sales_log_xml.at_xpath("//xmlns:joint").content = "1 Yes"
+        sales_log_xml.at_xpath("//xmlns:Q2Person1Income").content = "45000"
+        sales_log_xml.at_xpath("//xmlns:P2IncKnown").content = "1 Yes"
+        sales_log_xml.at_xpath("//xmlns:Q2Person2Income").content = "40000"
+        sales_log_xml.at_xpath("//xmlns:Q14ONSLACode").content = "E07000223"
+      end
+
+      it "intercepts the relevant validation error" do
+        expect { sales_log_service.send(:create_log, sales_log_xml) }
+          .not_to raise_error
+      end
+
+      it "clears out the invalid answers" do
+        allow(logger).to receive(:warn)
+
+        sales_log_service.send(:create_log, sales_log_xml)
+        sales_log = SalesLog.find_by(old_id: sales_log_id)
+
+        expect(sales_log).not_to be_nil
+        expect(sales_log.income1).to be_nil
+        expect(sales_log.income2).to be_nil
+      end
+    end
+
     context "and it has an invalid income 1 for london" do
       let(:sales_log_id) { "shared_ownership_sales_log" }
 
@@ -872,6 +1123,34 @@ RSpec.describe Imports::SalesLogsImportService do
       end
     end
 
+    context "and it has an invalid combined income for london" do
+      let(:sales_log_id) { "shared_ownership_sales_log" }
+
+      before do
+        sales_log_xml.at_xpath("//xmlns:joint").content = "1 Yes"
+        sales_log_xml.at_xpath("//xmlns:Q2Person1Income").content = "50000"
+        sales_log_xml.at_xpath("//xmlns:P2IncKnown").content = "1 Yes"
+        sales_log_xml.at_xpath("//xmlns:Q2Person2Income").content = "45000"
+        sales_log_xml.at_xpath("//xmlns:Q14ONSLACode").content = "E09000012"
+      end
+
+      it "intercepts the relevant validation error" do
+        expect { sales_log_service.send(:create_log, sales_log_xml) }
+          .not_to raise_error
+      end
+
+      it "clears out the invalid answers" do
+        allow(logger).to receive(:warn)
+
+        sales_log_service.send(:create_log, sales_log_xml)
+        sales_log = SalesLog.find_by(old_id: sales_log_id)
+
+        expect(sales_log).not_to be_nil
+        expect(sales_log.income1).to be_nil
+        expect(sales_log.income2).to be_nil
+      end
+    end
+
     context "and it has an invalid mscharge" do
       let(:sales_log_id) { "shared_ownership_sales_log" }
 
@@ -1848,6 +2127,7 @@ RSpec.describe Imports::SalesLogsImportService do
 
           sales_log = SalesLog.find_by(old_id: sales_log_id)
           expect(sales_log&.fromprop).to be(0)
+          expect(sales_log&.soctenant).to be(0)
         end
       end
 
@@ -1864,6 +2144,7 @@ RSpec.describe Imports::SalesLogsImportService do
 
           sales_log = SalesLog.find_by(old_id: sales_log_id)
           expect(sales_log&.socprevten).to be(10)
+          expect(sales_log&.soctenant).to be(0)
         end
       end
 
@@ -1919,5 +2200,72 @@ RSpec.describe Imports::SalesLogsImportService do
         end
       end
     end
+
+    context "when setting income for in progress logs" do
+      let(:sales_log_id) { "shared_ownership_sales_log" }
+
+      before do
+        sales_log_xml.at_xpath("//meta:status").content = "saved"
+      end
+
+      it "sets income to known if not answered but income is given for person 1" do
+        sales_log_xml.at_xpath("//xmlns:joint").content = "2 No"
+        sales_log_xml.at_xpath("//xmlns:P1IncKnown").content = ""
+        sales_log_xml.at_xpath("//xmlns:Q2Person1Income").content = "20000"
+        sales_log_xml.at_xpath("//xmlns:P2IncKnown").content = ""
+        sales_log_xml.at_xpath("//xmlns:Q2Person2Income").content = "30000"
+        sales_log_service.send(:create_log, sales_log_xml)
+
+        sales_log = SalesLog.find_by(old_id: sales_log_id)
+        expect(sales_log&.income1nk).to eq(0)
+        expect(sales_log&.income1).to eq(20_000)
+        expect(sales_log&.income2nk).to eq(nil)
+        expect(sales_log&.income2).to eq(nil)
+      end
+
+      it "sets income to known if not answered but income is given for person 2" do
+        sales_log_xml.at_xpath("//xmlns:joint").content = "1 Yes"
+        sales_log_xml.at_xpath("//xmlns:JointMore").content = "2 No"
+        sales_log_xml.at_xpath("//xmlns:P1IncKnown").content = ""
+        sales_log_xml.at_xpath("//xmlns:Q2Person1Income").content = "20000"
+        sales_log_xml.at_xpath("//xmlns:P2IncKnown").content = ""
+        sales_log_xml.at_xpath("//xmlns:Q2Person2Income").content = "30000"
+        sales_log_service.send(:create_log, sales_log_xml)
+
+        sales_log = SalesLog.find_by(old_id: sales_log_id)
+        expect(sales_log&.income1nk).to eq(0)
+        expect(sales_log&.income1).to eq(20_000)
+        expect(sales_log&.income2nk).to eq(0)
+        expect(sales_log&.income2).to eq(30_000)
+      end
+
+      it "sets savings to known if savings are given" do
+        sales_log_xml.at_xpath("//xmlns:Q3Savings").content = "10000"
+        sales_log_xml.at_xpath("//xmlns:savingsKnown").content = ""
+        sales_log_service.send(:create_log, sales_log_xml)
+
+        sales_log = SalesLog.find_by(old_id: sales_log_id)
+        expect(sales_log&.savingsnk).to eq(0)
+        expect(sales_log&.savings).to eq(10_000)
+      end
+
+      it "does not set savings or income fields when values aren't given" do
+        sales_log_xml.at_xpath("//xmlns:P1IncKnown").content = ""
+        sales_log_xml.at_xpath("//xmlns:Q2Person1Income").content = ""
+        sales_log_xml.at_xpath("//xmlns:P2IncKnown").content = ""
+        sales_log_xml.at_xpath("//xmlns:Q2Person2Income").content = ""
+        sales_log_xml.at_xpath("//xmlns:Q3Savings").content = ""
+        sales_log_xml.at_xpath("//xmlns:savingsKnown").content = ""
+        sales_log_service.send(:create_log, sales_log_xml)
+
+        sales_log = SalesLog.find_by(old_id: sales_log_id)
+        expect(sales_log&.income1nk).to eq(nil)
+        expect(sales_log&.income1).to eq(nil)
+        expect(sales_log&.income2nk).to eq(nil)
+        expect(sales_log&.income2).to eq(nil)
+        expect(sales_log&.savingsnk).to eq(nil)
+        expect(sales_log&.savings).to eq(nil)
+      end
+    end
   end
 end

spec/services/imports/user_import_service_spec.rb

@@ -44,10 +44,24 @@ RSpec.describe Imports::UserImportService do
     end
 
     it "refuses to create a user belonging to a non existing organisation" do
-      expect(logger).to receive(:error).with(/ActiveRecord::RecordInvalid/)
+      expect(logger).to receive(:error).with(/Could not save user with email: john.doe@gov.uk/)
+      expect(logger).to receive(:error).with(/Validation failed: Organisation Select the user’s organisation/)
       import_service.create_users("user_directory")
     end
 
+    context "when the user with the same email already exists" do
+      before do
+        create(:organisation, old_org_id:)
+        create(:user, email: "john.doe@gov.uk")
+      end
+
+      it "logs an error and user email" do
+        expect(logger).to receive(:error).with(/Could not save user with email: john.doe@gov.uk/)
+        expect(logger).to receive(:error).with(/Validation failed: email Enter an email address that hasn’t already been used to sign up/)
+        import_service.create_users("user_directory")
+      end
+    end
+
     context "when the user is a data coordinator" do
       let(:old_user_id) { "d4729b1a5dfb68bb1e01c08445830c0add40907c" }