Merge branch 'main' into CLDC-2672-duplicate-logs-question-page-button

app/models/form/lettings/questions/referral_prp.rb

@@ -45,6 +45,9 @@ class Form::Lettings::Questions::ReferralPrp < ::Form::Question
     "13" => {
       "value" => "Youth offending team",
     },
+    "17" => {
+      "value" => "Children’s Social Care",
+    },
     "16" => {
       "value" => "Other",
     },

app/models/form/lettings/questions/referral_supported_housing.rb

@@ -45,6 +45,9 @@ class Form::Lettings::Questions::ReferralSupportedHousing < ::Form::Question
     "13" => {
       "value" => "Youth offending team",
     },
+    "17" => {
+      "value" => "Children’s Social Care",
+    },
     "16" => {
       "value" => "Other",
     },

app/models/form/lettings/questions/referral_supported_housing_prp.rb

@@ -24,6 +24,7 @@ class Form::Lettings::Questions::ReferralSupportedHousingPrp < ::Form::Question
     "12" => { "value" => "Police, probation or prison" },
     "7" => { "value" => "Voluntary agency" },
     "13" => { "value" => "Youth offending team" },
+    "17" => { "value" => "Children’s Social Care" },
     "16" => { "value" => "Other" },
   }.freeze
 end

app/models/validations/sales/financial_validations.rb

@@ -30,9 +30,9 @@ module Validations::Sales::FinancialValidations
     combined_income = record.income1 + record.income2
     relevant_fields = %i[income1 income2 ownershipsch uprn la postcode_full]
     if record.london_property? && combined_income > 90_000
-      relevant_fields.each { |field| record.errors.add field, I18n.t("validations.financial.income.combined_over_hard_max_for_london") }
+      relevant_fields.each { |field| record.errors.add field, :over_combined_hard_max_for_london, message: I18n.t("validations.financial.income.combined_over_hard_max_for_london") }
     elsif record.property_not_in_london? && combined_income > 80_000
-      relevant_fields.each { |field| record.errors.add field, I18n.t("validations.financial.income.combined_over_hard_max_for_outside_london") }
+      relevant_fields.each { |field| record.errors.add field, :over_combined_hard_max_for_outside_london, message: I18n.t("validations.financial.income.combined_over_hard_max_for_outside_london") }
     end
   end
 

app/models/validations/shared_validations.rb

@@ -49,7 +49,7 @@ module Validations::SharedValidations
       elsif incorrect_accuracy || value.to_d != value.to_i    # if the user enters a value in exponent notation (eg '4e1') the to_i method does not convert this to the correct value
         field = question.check_answer_label || question.id
         case question.step
-        when 1 then record.errors.add question.id.to_sym, I18n.t("validations.numeric.whole_number", field:)
+        when 1 then record.errors.add question.id.to_sym, :not_integer, message: I18n.t("validations.numeric.whole_number", field:)
         when 10 then record.errors.add question.id.to_sym, I18n.t("validations.numeric.nearest_ten", field:)
         end
       end

app/services/imports/import_report_service.rb

@@ -84,5 +84,69 @@ module Imports
 
       @logger.info("Unassigned logs report available in s3 import bucket at #{report_name}")
     end
+
+    def generate_missing_answers_report(report_suffix)
+      create_missing_answers_report(report_suffix, LettingsLog)
+      create_missing_answers_report(report_suffix, SalesLog)
+    end
+
+    def create_missing_answers_report(report_suffix, log_class)
+      class_name = log_class.name.underscore.humanize
+      Rails.logger.info("Generating missing imported #{class_name}s answers report")
+
+      unanswered_question_counts, missing_answers_example_sets = process_missing_answers(log_class)
+
+      report_name = "MissingAnswersReport#{log_class.name}_#{report_suffix}.csv"
+      @storage_service.write_file(report_name, BYTE_ORDER_MARK + missing_answers_report(unanswered_question_counts))
+
+      examples_report_name = "MissingAnswersExamples#{log_class.name}_#{report_suffix}.csv"
+      @storage_service.write_file(examples_report_name, BYTE_ORDER_MARK + missing_answers_examples(missing_answers_example_sets))
+
+      @logger.info("Missing #{class_name}s answers report available in s3 import bucket at #{report_name}")
+    end
+
+    def process_missing_answers(log_class)
+      unanswered_question_counts = {}
+      missing_answers_example_sets = {}
+
+      log_class.where.not(old_id: nil).where(status: "in_progress").each do |log|
+        applicable_questions = log.form.subsections.map { |s| s.applicable_questions(log).select { |q| q.enabled?(log) } }.flatten
+        unanswered_questions = (applicable_questions.filter { |q| q.unanswered?(log) }.map(&:id) - log.optional_fields).join(", ")
+
+        if unanswered_question_counts[unanswered_questions].present?
+          unanswered_question_counts[unanswered_questions] += 1
+          missing_answers_example_sets[unanswered_questions] << { id: log.id, old_form_id: log.old_form_id, owning_organisation_id: log.owning_organisation_id } unless unanswered_question_counts[unanswered_questions] > 10
+        else
+          unanswered_question_counts[unanswered_questions] = 1
+          missing_answers_example_sets[unanswered_questions] = [{ id: log.id, old_form_id: log.old_form_id, owning_organisation_id: log.owning_organisation_id }]
+        end
+      end
+
+      [unanswered_question_counts, missing_answers_example_sets]
+    end
+
+    def missing_answers_report(unanswered_question_counts)
+      CSV.generate do |report|
+        headers = ["Missing answers", "Total number of affected logs"]
+        report << headers
+
+        unanswered_question_counts.each do |missing_answers, count|
+          report << [missing_answers, count]
+        end
+      end
+    end
+
+    def missing_answers_examples(missing_answers_example_sets)
+      CSV.generate do |report|
+        headers = ["Missing answers", "Organisation ID", "Log ID", "Old Form ID"]
+        report << headers
+
+        missing_answers_example_sets.each do |missing_answers, examples|
+          examples.each do |example|
+            report << [missing_answers, example[:owning_organisation_id], example[:id], example[:old_form_id]]
+          end
+        end
+      end
+    end
   end
 end

app/services/imports/lettings_logs_field_import_service.rb

@@ -12,6 +12,10 @@ module Imports
         import_from(folder, :update_offered)
       when "creation_method"
         import_from(folder, :update_creation_method)
+      when "address"
+        import_from(folder, :update_address)
+      when "reason"
+        import_from(folder, :update_reason)
       else
         raise "Updating #{field} is not supported by the field import service"
       end
@@ -132,5 +136,72 @@ module Imports
         @logger.warn("Could not find record matching legacy ID #{old_id}")
       end
     end
+
+    def update_address(xml_doc)
+      return if meta_field_value(xml_doc, "form-name").include?("Sales")
+
+      old_id = meta_field_value(xml_doc, "document-id")
+      record = LettingsLog.find_by(old_id:)
+      return @logger.info("lettings log #{record.id} is from previous collection year, skipping") if record.collection_start_year < 2023
+
+      if record.present?
+        if string_or_nil(xml_doc, "AddressLine1").present? && string_or_nil(xml_doc, "TownCity").present?
+          record.la = string_or_nil(xml_doc, "Q28ONS")
+          record.postcode_full = compose_postcode(xml_doc, "POSTCODE", "POSTCOD2")
+          record.postcode_known = postcode_known(record)
+          record.address_line1 = string_or_nil(xml_doc, "AddressLine1")
+          record.address_line2 = string_or_nil(xml_doc, "AddressLine2")
+          record.town_or_city = string_or_nil(xml_doc, "TownCity")
+          record.county = string_or_nil(xml_doc, "County")
+          record.uprn = nil
+          record.uprn_known = 0
+          record.uprn_confirmed = 0
+          record.values_updated_at = Time.zone.now
+          record.save!
+          @logger.info("lettings log #{record.id} address_line1 value has been set to #{record.address_line1}")
+          @logger.info("lettings log #{record.id} address_line2 value has been set to #{record.address_line2}")
+          @logger.info("lettings log #{record.id} town_or_city value has been set to #{record.town_or_city}")
+          @logger.info("lettings log #{record.id} county value has been set to #{record.county}")
+          @logger.info("lettings log #{record.id} postcode_full value has been set to #{record.postcode_full}")
+        else
+          @logger.info("lettings log #{record.id} is missing either or both of address_line1 and town or city, skipping")
+        end
+      else
+        @logger.warn("Could not find record matching legacy ID #{old_id}")
+      end
+    end
+
+    def postcode_known(record)
+      if record.postcode_full.nil?
+        record.la.nil? ? nil : 0 # Assumes we selected No in the form since the LA is present
+      else
+        1
+      end
+    end
+
+    def update_reason(xml_doc)
+      return if meta_field_value(xml_doc, "form-name").include?("Sales")
+
+      old_id = meta_field_value(xml_doc, "document-id")
+      record = LettingsLog.find_by(old_id:)
+
+      if record.present?
+        if record.reason.present?
+          @logger.info("lettings log #{record.id} has a value for reason, skipping update")
+        else
+          reason = unsafe_string_as_integer(xml_doc, "Q9a")
+          reasonother = string_or_nil(xml_doc, "Q9aa")
+          if reason == 20 && reasonother.blank?
+            @logger.info("lettings log #{record.id}'s reason is other but other reason is not provided, skipping update")
+          else
+            record.update!(reason:, reasonother:, values_updated_at: Time.zone.now)
+            @logger.info("lettings log #{record.id}'s reason value has been set to #{reason}")
+            @logger.info("lettings log #{record.id}'s reasonother value has been set to #{reasonother}") if record.reasonother.present?
+          end
+        end
+      else
+        @logger.warn("lettings log with old id #{old_id} not found")
+      end
+    end
   end
 end

app/services/imports/lettings_logs_import_service.rb

@@ -76,14 +76,19 @@ module Imports
       attributes["irproduct_other"] = string_or_nil(xml_doc, "IRProductOther")
       attributes["rent_type"] = rent_type(xml_doc, attributes["lar"], attributes["irproduct"])
       attributes["hhmemb"] = household_members(xml_doc, previous_status)
+
+      people_indexes = people_with_details_ids(xml_doc)
+      available_people_indexes = people_indexes + (people_indexes.max + 1..8).to_a
       (1..8).each do |index|
-        attributes["age#{index}"] = safe_string_as_integer(xml_doc, "P#{index}Age")
-        attributes["age#{index}_known"] = age_known(xml_doc, index, attributes["hhmemb"])
-        attributes["sex#{index}"] = sex(xml_doc, index)
-        attributes["ecstat#{index}"] = unsafe_string_as_integer(xml_doc, "P#{index}Eco")
+        person_index = available_people_indexes[index - 1]
+        attributes["age#{index}"] = safe_string_as_integer(xml_doc, "P#{person_index}Age")
+        attributes["age#{index}_known"] = age_known(xml_doc, index, person_index, attributes["hhmemb"])
+        attributes["sex#{index}"] = sex(xml_doc, person_index)
+        attributes["ecstat#{index}"] = unsafe_string_as_integer(xml_doc, "P#{person_index}Eco")
       end
       (2..8).each do |index|
-        attributes["relat#{index}"] = relat(xml_doc, index)
+        person_index = available_people_indexes[index - 1]
+        attributes["relat#{index}"] = relat(xml_doc, person_index)
         attributes["details_known_#{index}"] = details_known(index, attributes)
 
         # Trips validation
@@ -142,12 +147,6 @@ module Imports
       attributes["rp_hardship"] = unsafe_string_as_integer(xml_doc, "Q14b4").present? ? 1 : nil
       attributes["rp_dontknow"] = unsafe_string_as_integer(xml_doc, "Q14b5").present? ? 1 : nil
 
-      # Trips validation
-      if attributes["homeless"] == 1 && attributes["rp_homeless"] == 1
-        attributes["homeless"] = nil
-        attributes["rp_homeless"] = nil
-      end
-
       attributes["cbl"] = allocation_system(unsafe_string_as_integer(xml_doc, "Q15CBL"))
       attributes["chr"] = allocation_system(unsafe_string_as_integer(xml_doc, "Q15CHR"))
       attributes["cap"] = allocation_system(unsafe_string_as_integer(xml_doc, "Q15CAP"))
@@ -217,9 +216,6 @@ module Imports
 
         schemes = Scheme.where(old_visible_id: scheme_old_visible_id, owning_organisation_id: attributes["owning_organisation_id"])
         location = Location.find_by(old_visible_id: location_old_visible_id, scheme: schemes)
-        if location.nil? && [location_old_visible_id, scheme_old_visible_id].all?(&:present?) && previous_status != "saved"
-          raise "No matching location for scheme #{scheme_old_visible_id} and location #{location_old_visible_id} (visible IDs)"
-        end
 
         if location.present?
           # Set the scheme via location, because the scheme old visible ID can be duplicated at import
@@ -251,8 +247,12 @@ module Imports
       owner_id = meta_field_value(xml_doc, "owner-user-id").strip
       if owner_id.present?
         user = LegacyUser.find_by(old_user_id: owner_id)&.user
-        if user.blank?
-          @logger.error("Lettings log '#{attributes['old_id']}' belongs to legacy user with owner-user-id: '#{owner_id}' which cannot be found. Assigning log to 'Unassigned' user.")
+        if user.blank? || (user.organisation_id != attributes["managing_organisation_id"] && user.organisation_id != attributes["owning_organisation_id"])
+          if user.blank?
+            @logger.error("Lettings log '#{attributes['old_id']}' belongs to legacy user with owner-user-id: '#{owner_id}' which cannot be found. Assigning log to 'Unassigned' user.")
+          else
+            @logger.error("Lettings log '#{attributes['old_id']}' belongs to legacy user with owner-user-id: '#{owner_id}' which belongs to a different organisation. Assigning log to 'Unassigned' user.")
+          end
           if User.find_by(name: "Unassigned", organisation_id: attributes["managing_organisation_id"])
             user = User.find_by(name: "Unassigned", organisation_id: attributes["managing_organisation_id"])
           else
@@ -317,11 +317,12 @@ module Imports
         %i[chcharge out_of_range] => %w[chcharge],
         %i[referral internal_transfer_non_social_housing] => %w[referral],
         %i[referral internal_transfer_fixed_or_lifetime] => %w[referral],
-        %i[tenancylength tenancylength_invalid] => %w[tenancylength tenancy],
+        %i[tenancylength tenancylength_invalid] => %w[tenancylength],
         %i[prevten over_25_foster_care] => %w[prevten age1],
         %i[prevten non_temp_accommodation] => %w[prevten rsnvac],
         %i[joint not_joint_tenancy] => %w[joint],
         %i[offered outside_the_range] => %w[offered],
+        %i[offered not_integer] => %w[offered],
         %i[earnings over_hard_max] => %w[ecstat1],
         %i[tshortfall no_outstanding_charges] => %w[tshortfall hbrentshortfall],
         %i[beds outside_the_range] => %w[beds],
@@ -453,10 +454,10 @@ module Imports
       end
     end
 
-    def age_known(xml_doc, index, hhmemb)
+    def age_known(xml_doc, index, person_index, hhmemb)
       return nil if hhmemb.present? && index > hhmemb
 
-      age_refused = string_or_nil(xml_doc, "P#{index}AR")
+      age_refused = string_or_nil(xml_doc, "P#{person_index}AR")
       if age_refused.present?
         if age_refused.casecmp?("AGE_REFUSED") || age_refused.casecmp?("No")
           return 1 # No
@@ -581,6 +582,15 @@ module Imports
       ((2..8).map { |x| string_or_nil(xml_doc, "P#{x}Rel") } + [string_or_nil(xml_doc, "P1Sex")]).compact
     end
 
+    def people_with_details_ids(xml_doc)
+      [1] + (2..8).select do |x|
+        string_or_nil(xml_doc, "P#{x}Rel").present? ||
+          string_or_nil(xml_doc, "P#{x}Sex").present? ||
+          string_or_nil(xml_doc, "P#{x}Age").present? ||
+          string_or_nil(xml_doc, "P#{x}Eco").present?
+      end
+    end
+
     def tshortfall_known?(xml_doc, attributes)
       if attributes["tshortfall"].blank? && attributes["hbrentshortfall"] == 1 && overridden?(xml_doc, "xmlns", "Q18dyes")
         1

app/services/imports/logs_import_service.rb

@@ -72,7 +72,7 @@ module Imports
       if str.nil?
         nil
       else
-        BigDecimal(str, exception: false)
+        BigDecimal(str, exception: false)&.round(2)
       end
     end
 

app/services/imports/sales_logs_import_service.rb

@@ -1,5 +1,7 @@
 module Imports
   class SalesLogsImportService < LogsImportService
+    include CollectionTimeHelper
+
     def initialize(storage_service, logger = Rails.logger, allow_updates: false)
       @logs_with_discrepancies = Set.new
       @logs_overridden = Set.new
@@ -15,7 +17,7 @@ module Imports
     def create_log(xml_doc)
       # only import sales logs from 22/23 collection period onwards
       return unless meta_field_value(xml_doc, "form-name").include?("Sales")
-      return unless compose_date(xml_doc, "DAY", "MONTH", "YEAR") >= Time.zone.local(2022, 4, 1)
+      return unless (compose_date(xml_doc, "DAY", "MONTH", "YEAR") || Time.zone.parse(field_value(xml_doc, "xmlns", "CompletionDate"))) >= Time.zone.local(2022, 4, 1)
 
       attributes = {}
 
@@ -24,7 +26,7 @@ module Imports
       # Required fields for status complete or logic to work
       # Note: order matters when we derive from previous values (attributes parameter)
 
-      attributes["saledate"] = compose_date(xml_doc, "DAY", "MONTH", "YEAR")
+      attributes["saledate"] = compose_date(xml_doc, "DAY", "MONTH", "YEAR") || Time.zone.parse(field_value(xml_doc, "xmlns", "CompletionDate"))
       attributes["owning_organisation_id"] = find_organisation_id(xml_doc, "OWNINGORGID")
       attributes["type"] = unsafe_string_as_integer(xml_doc, "DerSaleType")
       attributes["old_id"] = meta_field_value(xml_doc, "document-id")
@@ -124,7 +126,7 @@ module Imports
       attributes["mortgagelenderother"] = mortgage_lender_other(xml_doc, attributes)
       attributes["postcode_full"] = parse_postcode(string_or_nil(xml_doc, "Q14Postcode"))
       attributes["pcodenk"] = 0 if attributes["postcode_full"].present? # known if given
-      attributes["soctenant"] = 0 if attributes["ownershipsch"] == 1
+      attributes["soctenant"] = 0 if set_soctenant_fields?(attributes)
 
       attributes["previous_la_known"] = 1 if attributes["prevloc"].present?
       if attributes["la"].present?
@@ -152,6 +154,10 @@ module Imports
       attributes["discounted_sale_value_check"] = 0
       attributes["buyer_livein_value_check"] = 0
       attributes["percentage_discount_value_check"] = 0
+      attributes["hodate_check"] = 0
+      attributes["saledate_check"] = 0
+      attributes["combined_income_value_check"] = 0
+      attributes["stairowned_value_check"] = 0
 
       # 2023/34 attributes
       attributes["address_line1"] = string_or_nil(xml_doc, "AddressLine1")
@@ -179,8 +185,12 @@ module Imports
       if owner_id.present?
         user = LegacyUser.find_by(old_user_id: owner_id)&.user
 
-        if user.blank?
-          @logger.error("Sales log '#{attributes['old_id']}' belongs to legacy user with owner-user-id: '#{owner_id}' which cannot be found. Assigning log to 'Unassigned' user.")
+        if user.blank? || user.organisation_id != attributes["owning_organisation_id"]
+          if user.blank?
+            @logger.error("Sales log '#{attributes['old_id']}' belongs to legacy user with owner-user-id: '#{owner_id}' which cannot be found. Assigning log to 'Unassigned' user.")
+          else
+            @logger.error("Sales log '#{attributes['old_id']}' belongs to legacy user with owner-user-id: '#{owner_id}' which belongs to a different organisation. Assigning log to 'Unassigned' user.")
+          end
           if User.find_by(name: "Unassigned", organisation_id: attributes["owning_organisation_id"])
             user = User.find_by(name: "Unassigned", organisation_id: attributes["owning_organisation_id"])
           else
@@ -242,15 +252,20 @@ module Imports
         %i[postcode_full postcodes_not_matching] => %w[ppcodenk ppostcode_full],
         %i[exdate over_a_year_from_saledate] => %w[exdate],
         %i[income1 over_hard_max_for_outside_london] => %w[income1],
+        %i[income1 over_combined_hard_max_for_outside_london] => %w[income1 income2],
         %i[income1 over_hard_max_for_london] => %w[income1],
+        %i[income1 over_combined_hard_max_for_london] => %w[income1 income2],
         %i[income2 over_hard_max_for_outside_london] => %w[income2],
+        %i[income2 over_combined_hard_max_for_outside_london] => %w[income1 income2],
         %i[income2 over_hard_max_for_london] => %w[income2],
+        %i[income2 over_combined_hard_max_for_london] => %w[income1 income2],
         %i[equity over_max] => %w[equity],
         %i[equity under_min] => %w[equity],
         %i[mscharge under_min] => %w[mscharge has_mscharge],
         %i[mortgage cannot_be_0] => %w[mortgage],
         %i[frombeds outside_the_range] => %w[frombeds],
         %i[age1 outside_the_range] => %w[age1 age1_known],
+        %i[proplen outside_the_range] => %w[proplen],
       }
 
       errors.each do |(error, fields)|
@@ -326,6 +341,8 @@ module Imports
          discounted_sale_value_check
          buyer_livein_value_check
          percentage_discount_value_check
+         combined_income_value_check
+         stairowned_value_check
          uprn_known
          uprn_confirmed]
     end
@@ -569,6 +586,12 @@ module Imports
       end
     end
 
+    def set_soctenant_fields?(attributes)
+      return false if attributes["ownershipsch"] != 1
+
+      %w[socprevten frombeds fromprop].any? { |field| attributes[field].present? } || collection_start_year_for_date(attributes["saledate"]) < 2023
+    end
+
     def set_default_values(attributes)
       attributes["armedforcesspouse"] ||= 7
       attributes["hhregres"] ||= 8
@@ -585,8 +608,8 @@ module Imports
       attributes["pcodenk"] ||= 1
       attributes["prevten"] ||= 0
       attributes["extrabor"] ||= 3 if attributes["mortgageused"] == 1
-      attributes["socprevten"] ||= 10 if attributes["ownershipsch"] == 1
-      attributes["fromprop"] ||= 0 if attributes["ownershipsch"] == 1
+      attributes["socprevten"] ||= 10 if set_soctenant_fields?(attributes)
+      attributes["fromprop"] ||= 0 if set_soctenant_fields?(attributes)
       attributes["mortgagelender"] ||= 0 if attributes["mortgageused"] == 1
 
       # buyer 1 characteristics

config/forms/2022_2023.json

@@ -7081,6 +7081,9 @@
                     "13": {
                       "value": "Youth offending team"
                     },
+                    "17": {
+                      "value": "Children’s Social Care"
+                    },
                     "16": {
                       "value": "Other"
                     }
@@ -7138,6 +7141,9 @@
                     "13": {
                       "value": "Youth offending team"
                     },
+                    "17": {
+                      "value": "Children’s Social Care"
+                    },
                     "16": {
                       "value": "Other"
                     }
@@ -7198,6 +7204,9 @@
                     "13": {
                       "value": "Youth offending team"
                     },
+                    "17": {
+                      "value": "Children’s Social Care"
+                    },
                     "16": {
                       "value": "Other"
                     }

lib/tasks/data_import_field.rake

@@ -7,7 +7,7 @@ namespace :core do
 
     # We only allow a reduced list of known fields to be updatable
     case field
-    when "tenancycode", "major_repairs", "lettings_allocation", "offered"
+    when "tenancycode", "major_repairs", "lettings_allocation", "offered", "address", "reason"
       s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
       archive_io = s3_service.get_file_io(path)
       archive_service = Storage::ArchiveService.new(archive_io)

lib/tasks/full_import.rake

@@ -117,6 +117,15 @@ namespace :import do
     Imports::ImportReportService.new(s3_service, institutions_csv).create_reports(institutions_csv_name)
   end
 
+  desc "Generate missing answers report"
+  task :generate_missing_answers_report, %i[file_suffix] => :environment do |_task, args|
+    file_suffix = args[:file_suffix]
+    raise "Usage: rake import:generate_reports['file_suffix']" if file_suffix.blank?
+
+    s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
+    Imports::ImportReportService.new(s3_service, nil).generate_missing_answers_report(file_suffix)
+  end
+
   desc "Run import from logs step to end"
   task :logs_onwards, %i[institutions_csv_name] => %i[environment logs trigger_invites generate_reports]
 

lib/tasks/import_address_from_csv.rake

@@ -0,0 +1,42 @@
+namespace :data_import do
+  desc "Import address data from a csv file"
+  task :import_address_from_csv, %i[file_name] => :environment do |_task, args|
+    file_name = args[:file_name]
+
+    raise "Usage: rake data_import:import_address_from_csv['csv_file_name']" if file_name.blank?
+
+    s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
+    addresses_csv = CSV.parse(s3_service.get_file_io(file_name), headers: true)
+
+    addresses_csv.each do |row|
+      lettings_log_id = row[0]
+
+      if lettings_log_id.blank?
+        Rails.logger.info("Lettings log ID not provided for address: #{[row[1], row[2], row[3], row[4]].join(', ')}")
+        next
+      end
+
+      lettings_log = LettingsLog.find_by(id: lettings_log_id)
+      if lettings_log.blank?
+        Rails.logger.info("Could not find a lettings log with id #{lettings_log_id}")
+        next
+      end
+
+      lettings_log.uprn_known = 0
+      lettings_log.uprn = nil
+      lettings_log.uprn_confirmed = nil
+      lettings_log.address_line1 = row[1]
+      lettings_log.address_line2 = row[2]
+      lettings_log.town_or_city = row[3]
+      lettings_log.postcode_full = row[4]
+      lettings_log.postcode_known = lettings_log.postcode_full.present? ? 1 : nil
+      lettings_log.county = nil
+      lettings_log.is_la_inferred = nil
+      lettings_log.la = nil
+      lettings_log.values_updated_at = Time.zone.now
+
+      lettings_log.save!
+      Rails.logger.info("Updated lettings log #{lettings_log_id}, with address: #{[lettings_log.address_line1, lettings_log.address_line2, lettings_log.town_or_city, lettings_log.postcode_full].join(', ')}")
+    end
+  end
+end

spec/fixtures/files/addresses_reimport.csv

@@ -0,0 +1,6 @@
+lettings_log_id,address_line1,address_line2,town_or_city,postcode_full
+{id},address 1,address 2,town,B1 1BB
+{id2},address 3,address 4,city,B1 1BB
+{id3},,,,C1 1CC
+,address 5,address 6,city,D1 1DD
+fake_id,address 7,address 8,city,D1 1DD

spec/fixtures/files/imported_lettings_logs_missing_answers_examples.csv

@@ -0,0 +1,16 @@
+Missing answers,Organisation ID,Log ID,Old Form ID
+age1_known,{org_id0},{id0},1000
+age1_known,{org_id1},{id1},1001
+age1_known,{org_id2},{id2},1002
+age1_known,{org_id3},{id3},1003
+age1_known,{org_id4},{id4},1004
+age1_known,{org_id5},{id5},1005
+age1_known,{org_id6},{id6},1006
+age1_known,{org_id7},{id7},1007
+age1_known,{org_id8},{id8},1008
+age1_known,{org_id9},{id9},1009
+beds,{org_id2_0},{id2_0},2000
+beds,{org_id2_1},{id2_1},2001
+beds,{org_id2_2},{id2_2},2002
+beds,{org_id2_3},{id2_3},2003
+"beds, age1_known",{org_id},{id},300

spec/fixtures/files/imported_lettings_logs_missing_answers_report.csv

@@ -0,0 +1,4 @@
+Missing answers,Total number of affected logs
+age1_known,11
+beds,4
+"beds, age1_known",1

spec/lib/tasks/correct_address_from_csv_spec.rb

@@ -0,0 +1,154 @@
+require "rails_helper"
+require "rake"
+
+RSpec.describe "data_import" do
+  def replace_entity_ids(lettings_log, second_lettings_log, third_lettings_log, export_template)
+    export_template.sub!(/\{id\}/, lettings_log.id.to_s)
+    export_template.sub!(/\{id2\}/, second_lettings_log.id.to_s)
+    export_template.sub!(/\{id3\}/, third_lettings_log.id.to_s)
+  end
+
+  describe ":import_address_from_csv", type: :task do
+    subject(:task) { Rake::Task["data_import:import_address_from_csv"] }
+
+    let(:instance_name) { "paas_import_instance" }
+    let(:storage_service) { instance_double(Storage::S3Service) }
+    let(:env_config_service) { instance_double(Configuration::EnvConfigurationService) }
+    let(:paas_config_service) { instance_double(Configuration::PaasConfigurationService) }
+
+    before do
+      allow(Storage::S3Service).to receive(:new).and_return(storage_service)
+      allow(Configuration::EnvConfigurationService).to receive(:new).and_return(env_config_service)
+      allow(Configuration::PaasConfigurationService).to receive(:new).and_return(paas_config_service)
+      allow(ENV).to receive(:[])
+      allow(ENV).to receive(:[]).with("IMPORT_PAAS_INSTANCE").and_return(instance_name)
+      allow(ENV).to receive(:[]).with("VCAP_SERVICES").and_return("dummy")
+
+      Rake.application.rake_require("tasks/import_address_from_csv")
+      Rake::Task.define_task(:environment)
+      task.reenable
+
+      WebMock.stub_request(:get, /api.postcodes.io/)
+        .to_return(status: 200, body: "{\"status\":404,\"error\":\"Postcode not found\"}", headers: {})
+      WebMock.stub_request(:get, /api.postcodes.io\/postcodes\/B11BB/)
+        .to_return(status: 200, body: '{"status":200,"result":{"admin_district":"Westminster","codes":{"admin_district":"E08000035"}}}', headers: {})
+    end
+
+    context "when the rake task is run" do
+      let(:addresses_csv_path) { "addresses_reimport_123.csv" }
+      let(:wrong_file_path) { "/test/no_csv_here.csv" }
+      let!(:lettings_log) do
+        create(:lettings_log,
+               uprn_known: nil,
+               uprn: nil,
+               uprn_confirmed: nil,
+               address_line1: nil,
+               address_line2: nil,
+               town_or_city: nil,
+               county: nil,
+               postcode_known: 1,
+               postcode_full: "A1 1AA",
+               la: "E06000064",
+               is_la_inferred: true)
+      end
+
+      let!(:second_lettings_log) do
+        create(:lettings_log,
+               uprn_known: 1,
+               uprn: "1",
+               uprn_confirmed: nil,
+               address_line1: "wrong address line1",
+               address_line2: "wrong address 2",
+               town_or_city: "wrong town",
+               county: "wrong city",
+               postcode_known: 1,
+               postcode_full: "A1 1AA",
+               la: "E06000064",
+               is_la_inferred: true)
+      end
+
+      let!(:third_lettings_log) do
+        create(:lettings_log,
+               uprn_known: 1,
+               uprn: "1",
+               uprn_confirmed: nil,
+               address_line1: "wrong address line1",
+               address_line2: "wrong address 2",
+               town_or_city: "wrong town",
+               county: "wrong city",
+               postcode_known: 1,
+               postcode_full: "A1 1AA",
+               la: "E06000064",
+               is_la_inferred: true)
+      end
+
+      before do
+        allow(storage_service).to receive(:get_file_io)
+        .with("addresses_reimport_123.csv")
+        .and_return(replace_entity_ids(lettings_log, second_lettings_log, third_lettings_log, File.open("./spec/fixtures/files/addresses_reimport.csv").read))
+      end
+
+      it "updates the log address when old address was not given" do
+        task.invoke(addresses_csv_path)
+        lettings_log.reload
+        expect(lettings_log.uprn_known).to eq(0)
+        expect(lettings_log.uprn).to eq(nil)
+        expect(lettings_log.uprn_confirmed).to eq(nil)
+        expect(lettings_log.address_line1).to eq("address 1")
+        expect(lettings_log.address_line2).to eq("address 2")
+        expect(lettings_log.town_or_city).to eq("town")
+        expect(lettings_log.county).to eq(nil)
+        expect(lettings_log.postcode_known).to eq(1)
+        expect(lettings_log.postcode_full).to eq("B1 1BB")
+        expect(lettings_log.la).to eq("E08000035")
+        expect(lettings_log.is_la_inferred).to eq(true)
+      end
+
+      it "updates the log address when old address was given" do
+        task.invoke(addresses_csv_path)
+        second_lettings_log.reload
+        expect(second_lettings_log.uprn_known).to eq(0)
+        expect(second_lettings_log.uprn).to eq(nil)
+        expect(second_lettings_log.uprn_confirmed).to eq(nil)
+        expect(second_lettings_log.address_line1).to eq("address 3")
+        expect(second_lettings_log.address_line2).to eq("address 4")
+        expect(second_lettings_log.town_or_city).to eq("city")
+        expect(second_lettings_log.county).to eq(nil)
+        expect(second_lettings_log.postcode_known).to eq(1)
+        expect(second_lettings_log.postcode_full).to eq("B1 1BB")
+        expect(second_lettings_log.la).to eq("E08000035")
+        expect(second_lettings_log.is_la_inferred).to eq(true)
+      end
+
+      it "updates the log address when new address is not given" do
+        task.invoke(addresses_csv_path)
+        third_lettings_log.reload
+        expect(third_lettings_log.uprn_known).to eq(0)
+        expect(third_lettings_log.uprn).to eq(nil)
+        expect(third_lettings_log.uprn_confirmed).to eq(nil)
+        expect(third_lettings_log.address_line1).to eq(nil)
+        expect(third_lettings_log.address_line2).to eq(nil)
+        expect(third_lettings_log.town_or_city).to eq(nil)
+        expect(third_lettings_log.county).to eq(nil)
+        expect(third_lettings_log.postcode_known).to eq(1)
+        expect(third_lettings_log.postcode_full).to eq("C11CC")
+        expect(third_lettings_log.la).to eq(nil)
+        expect(third_lettings_log.is_la_inferred).to eq(false)
+      end
+
+      it "logs the progress of the update" do
+        expect(Rails.logger).to receive(:info).with("Updated lettings log #{lettings_log.id}, with address: address 1, address 2, town, B1 1BB")
+        expect(Rails.logger).to receive(:info).with("Updated lettings log #{second_lettings_log.id}, with address: address 3, address 4, city, B1 1BB")
+        expect(Rails.logger).to receive(:info).with("Updated lettings log #{third_lettings_log.id}, with address: , , , C11CC")
+        expect(Rails.logger).to receive(:info).with("Lettings log ID not provided for address: address 5, address 6, city, D1 1DD")
+        expect(Rails.logger).to receive(:info).with("Could not find a lettings log with id fake_id")
+
+        task.invoke(addresses_csv_path)
+      end
+
+      it "raises an error when no path is given" do
+        expect { task.invoke(nil) }.to raise_error(RuntimeError, "Usage: rake data_import:import_address_from_csv['csv_file_name']")
+      end
+    end
+  end
+end

spec/lib/tasks/data_import_field_spec.rb

@@ -92,6 +92,30 @@ describe "data_import_field imports" do
         end
       end
 
+      context "and we update the address fields" do
+        let(:field) { "address" }
+
+        it "updates the 2023 logs from the given XML file" do
+          expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name)
+          expect(storage_service).to receive(:get_file_io).with("spec/fixtures/imports/logs")
+          expect(Imports::LettingsLogsFieldImportService).to receive(:new).with(archive_service)
+          expect(import_service).to receive(:update_field).with(field, "logs")
+          task.invoke(field, fixture_path)
+        end
+      end
+
+      context "and we update the reason field" do
+        let(:field) { "reason" }
+
+        it "updates the 2023 logs from the given XML file" do
+          expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name)
+          expect(storage_service).to receive(:get_file_io).with("spec/fixtures/imports/logs")
+          expect(Imports::LettingsLogsFieldImportService).to receive(:new).with(archive_service)
+          expect(import_service).to receive(:update_field).with(field, "logs")
+          task.invoke(field, fixture_path)
+        end
+      end
+
       it "raises an exception if no parameters are provided" do
         expect { task.invoke }.to raise_error(/Usage/)
       end

spec/lib/tasks/full_import_spec.rb

@@ -54,6 +54,32 @@ describe "full import", type: :task do
     end
   end
 
+  describe "import:generate_missing_answers_report" do
+    subject(:task) { Rake::Task["import:generate_missing_answers_report"] }
+
+    before do
+      Rake.application.rake_require("tasks/full_import")
+      Rake::Task.define_task(:environment)
+      task.reenable
+    end
+
+    context "when generating a missing answers report" do
+      let(:import_report_service) { instance_double(Imports::ImportReportService) }
+
+      before do
+        allow(Imports::ImportReportService).to receive(:new).and_return(import_report_service)
+        allow(ENV).to receive(:[]).with("VCAP_SERVICES").and_return("dummy")
+      end
+
+      it "creates a missing answers report" do
+        expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name)
+        expect(Imports::ImportReportService).to receive(:new).with(storage_service, nil)
+        expect(import_report_service).to receive(:generate_missing_answers_report).with("some_name")
+        task.invoke("some_name")
+      end
+    end
+  end
+
   describe "import:initial" do
     subject(:task) { Rake::Task["import:initial"] }
 

spec/services/imports/import_report_service_spec.rb

@@ -98,4 +98,82 @@ RSpec.describe Imports::ImportReportService do
       end
     end
   end
+
+  describe "#generate_missing_answers_report" do
+    context "when there are in progress imported lettings logs" do
+      let(:institutions_csv) { nil }
+      let(:expected_content) { File.read("spec/fixtures/files/imported_lettings_logs_missing_answers_report.csv") }
+      let(:expected_answers_examples_content) { File.read("spec/fixtures/files/imported_lettings_logs_missing_answers_examples.csv") }
+
+      before do
+        create_list(:lettings_log, 11, :completed, age1_known: nil) do |log, i|
+          log.old_form_id = "100#{i}"
+          log.old_id = "old_id_age1_known_#{i}"
+          log.save!
+          expected_answers_examples_content.sub!("{id#{i}}", log.id.to_s)
+          expected_answers_examples_content.sub!("{org_id#{i}}", log.owning_organisation_id.to_s)
+        end
+        create_list(:lettings_log, 4, :completed, beds: nil) do |log, i|
+          log.old_form_id = "200#{i}"
+          log.old_id = "old_id_beds_#{i}"
+          expected_answers_examples_content.sub!("{id2_#{i}}", log.id.to_s)
+          expected_answers_examples_content.sub!("{org_id2_#{i}}", log.owning_organisation_id.to_s)
+          log.save!
+        end
+        create(:lettings_log, :completed, age1_known: nil, beds: nil, old_form_id: "300", old_id: "123") do |log|
+          expected_answers_examples_content.sub!("{id}", log.id.to_s)
+          expected_answers_examples_content.sub!("{org_id}", log.owning_organisation_id.to_s)
+        end
+
+        create_list(:lettings_log, 2, :completed, age1_known: nil)
+      end
+
+      it "generates a csv with expected missing fields" do
+        expect(storage_service).to receive(:write_file).with("MissingAnswersReportLettingsLog_report_suffix.csv", "#{expected_content}")
+        expect(storage_service).to receive(:write_file).with("MissingAnswersExamplesLettingsLog_report_suffix.csv", "#{expected_answers_examples_content}")
+        expect(storage_service).to receive(:write_file).with("MissingAnswersReportSalesLog_report_suffix.csv", "\uFEFFMissing answers,Total number of affected logs\n")
+        expect(storage_service).to receive(:write_file).with("MissingAnswersExamplesSalesLog_report_suffix.csv", "\uFEFFMissing answers,Organisation ID,Log ID,Old Form ID\n")
+
+        report_service.generate_missing_answers_report("report_suffix")
+      end
+    end
+
+    context "when there are in progress imported sales logs" do
+      let(:institutions_csv) { nil }
+      let(:expected_content) { File.read("spec/fixtures/files/imported_lettings_logs_missing_answers_report.csv") }
+      let(:expected_answers_examples_content) { File.read("spec/fixtures/files/imported_lettings_logs_missing_answers_examples.csv") }
+
+      before do
+        create_list(:sales_log, 11, :completed, age1_known: nil) do |log, i|
+          log.old_id = "age1_known_#{i}"
+          log.old_form_id = "100#{i}"
+          log.save!
+          expected_answers_examples_content.sub!("{id#{i}}", log.id.to_s)
+          expected_answers_examples_content.sub!("{org_id#{i}}", log.owning_organisation_id.to_s)
+        end
+        create_list(:sales_log, 4, :completed, beds: nil) do |log, i|
+          log.old_id = "beds_#{i}"
+          log.old_form_id = "200#{i}"
+          expected_answers_examples_content.sub!("{id2_#{i}}", log.id.to_s)
+          expected_answers_examples_content.sub!("{org_id2_#{i}}", log.owning_organisation_id.to_s)
+          log.save!
+        end
+        create(:sales_log, :completed, age1_known: nil, beds: nil, old_id: "beds_and_age", old_form_id: "300") do |log|
+          expected_answers_examples_content.sub!("{id}", log.id.to_s)
+          expected_answers_examples_content.sub!("{org_id}", log.owning_organisation_id.to_s)
+        end
+
+        create_list(:sales_log, 2, :completed, age1_known: nil)
+      end
+
+      it "generates a csv with expected missing fields" do
+        expect(storage_service).to receive(:write_file).with("MissingAnswersReportLettingsLog_report_suffix.csv", "\uFEFFMissing answers,Total number of affected logs\n")
+        expect(storage_service).to receive(:write_file).with("MissingAnswersExamplesLettingsLog_report_suffix.csv", "\uFEFFMissing answers,Organisation ID,Log ID,Old Form ID\n")
+        expect(storage_service).to receive(:write_file).with("MissingAnswersReportSalesLog_report_suffix.csv", "#{expected_content}")
+        expect(storage_service).to receive(:write_file).with("MissingAnswersExamplesSalesLog_report_suffix.csv", "#{expected_answers_examples_content}")
+
+        report_service.generate_missing_answers_report("report_suffix")
+      end
+    end
+  end
 end

spec/services/imports/lettings_logs_field_import_service_spec.rb

@@ -323,4 +323,226 @@ RSpec.describe Imports::LettingsLogsFieldImportService do
       end
     end
   end
+
+  context "when updating address" do
+    let(:field) { "address" }
+
+    before do
+      WebMock.stub_request(:get, /api.postcodes.io\/postcodes\/B11BB/)
+      .to_return(status: 200, body: '{"status":200,"result":{"admin_district":"Westminster","codes":{"admin_district":"E08000035"}}}', headers: {})
+
+      stub_request(:get, "https://api.os.uk/search/places/v1/uprn?key=OS_DATA_KEY&uprn=123")
+        .to_return(status: 500, body: "{}", headers: {})
+
+      Timecop.freeze(2023, 5, 5)
+      Singleton.__init__(FormHandler)
+      Imports::LettingsLogsImportService.new(storage_service, logger).create_logs(fixture_directory)
+      lettings_log_file.rewind
+    end
+
+    after do
+      Timecop.unfreeze
+      Singleton.__init__(FormHandler)
+    end
+
+    context "when the lettings log has no address values" do
+      let(:lettings_log_id) { "00d2343e-d5fa-4c89-8400-ec3854b0f2b4" }
+      let(:lettings_log) { LettingsLog.find_by(old_id: lettings_log_id) }
+
+      before do
+        lettings_log.update!(uprn_known: nil,
+                             startdate: Time.zone.local(2023, 5, 5),
+                             uprn: nil,
+                             uprn_confirmed: nil,
+                             address_line1: nil,
+                             address_line2: nil,
+                             town_or_city: nil,
+                             county: nil,
+                             postcode_known: nil,
+                             postcode_full: nil,
+                             la: nil,
+                             is_la_inferred: nil)
+      end
+
+      context "and new address values include address" do
+        before do
+          lettings_log_xml.at_xpath("//xmlns:UPRN").content = "123456781234"
+          lettings_log_xml.at_xpath("//xmlns:AddressLine1").content = "address 1"
+          lettings_log_xml.at_xpath("//xmlns:AddressLine2").content = "address 2"
+          lettings_log_xml.at_xpath("//xmlns:TownCity").content = "towncity"
+          lettings_log_xml.at_xpath("//xmlns:County").content = "county"
+          lettings_log_xml.at_xpath("//xmlns:POSTCODE").content = "B1"
+          lettings_log_xml.at_xpath("//xmlns:POSTCOD2").content = "1BB"
+          lettings_log_xml.at_xpath("//xmlns:Q28ONS").content = nil
+        end
+
+        it "updates the lettings_log prioritising address values" do
+          expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} address_line1 value has been set to address 1/)
+          expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} address_line2 value has been set to address 2/)
+          expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} town_or_city value has been set to towncity/)
+          expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} county value has been set to county/)
+          expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} postcode_full value has been set to B1 1BB/)
+          import_service.send(:update_address, lettings_log_xml)
+          lettings_log.reload
+
+          expect(lettings_log.uprn_known).to eq(0)
+          expect(lettings_log.uprn).to eq(nil)
+          expect(lettings_log.uprn_confirmed).to eq(nil)
+          expect(lettings_log.address_line1).to eq("address 1")
+          expect(lettings_log.address_line2).to eq("address 2")
+          expect(lettings_log.town_or_city).to eq("towncity")
+          expect(lettings_log.county).to eq("county")
+          expect(lettings_log.postcode_known).to eq(1)
+          expect(lettings_log.postcode_full).to eq("B1 1BB")
+          expect(lettings_log.la).to eq("E08000035")
+          expect(lettings_log.is_la_inferred).to eq(true)
+        end
+      end
+
+      context "and new address values don't include address" do
+        it "skips the update" do
+          expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} is missing either or both of address_line1 and town or city, skipping/)
+          import_service.send(:update_address, lettings_log_xml)
+        end
+      end
+    end
+
+    context "when the lettings log has address values" do
+      let(:lettings_log_id) { "00d2343e-d5fa-4c89-8400-ec3854b0f2b4" }
+      let(:lettings_log) { LettingsLog.find_by(old_id: lettings_log_id) }
+
+      before do
+        lettings_log_xml.at_xpath("//xmlns:UPRN").content = "123456781234"
+        lettings_log_xml.at_xpath("//xmlns:AddressLine1").content = "address 1"
+        lettings_log_xml.at_xpath("//xmlns:AddressLine2").content = "address 2"
+        lettings_log_xml.at_xpath("//xmlns:TownCity").content = "towncity"
+        lettings_log_xml.at_xpath("//xmlns:County").content = "county"
+        lettings_log_xml.at_xpath("//xmlns:POSTCODE").content = "B1"
+        lettings_log_xml.at_xpath("//xmlns:POSTCOD2").content = "1BC"
+        lettings_log_xml.at_xpath("//xmlns:Q28ONS").content = nil
+        lettings_log.update!(uprn_known: 1,
+                             startdate: Time.zone.local(2023, 5, 5),
+                             uprn: "123",
+                             uprn_confirmed: 0,
+                             address_line1: "wrong address line1",
+                             address_line2: "wrong address 2",
+                             town_or_city: "wrong town",
+                             county: "wrong city",
+                             postcode_known: 1,
+                             postcode_full: "A11AA",
+                             la: "E06000064",
+                             is_la_inferred: true)
+      end
+
+      it "replaces the lettings_log address values" do
+        expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} address_line1 value has been set to address 1/)
+        expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} address_line2 value has been set to address 2/)
+        expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} town_or_city value has been set to towncity/)
+        expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} county value has been set to county/)
+        expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} postcode_full value has been set to B11BC/)
+        import_service.send(:update_address, lettings_log_xml)
+        lettings_log.reload
+
+        expect(lettings_log.uprn_known).to eq(0)
+        expect(lettings_log.uprn).to eq(nil)
+        expect(lettings_log.uprn_confirmed).to eq(nil)
+        expect(lettings_log.address_line1).to eq("address 1")
+        expect(lettings_log.address_line2).to eq("address 2")
+        expect(lettings_log.town_or_city).to eq("towncity")
+        expect(lettings_log.county).to eq("county")
+        expect(lettings_log.postcode_known).to eq(1)
+        expect(lettings_log.postcode_full).to eq("B11BC")
+        expect(lettings_log.la).to eq(nil)
+        expect(lettings_log.is_la_inferred).to eq(false)
+      end
+    end
+
+    context "when the lettings log is from before collection 23/24" do
+      let(:lettings_log_id) { "00d2343e-d5fa-4c89-8400-ec3854b0f2b4" }
+      let(:lettings_log) { LettingsLog.find_by(old_id: lettings_log_id) }
+
+      before do
+        lettings_log.update!(startdate: Time.zone.local(2022, 5, 5))
+      end
+
+      it "skips the update" do
+        expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} is from previous collection year, skipping/)
+        import_service.send(:update_address, lettings_log_xml)
+      end
+    end
+  end
+
+  context "when updating reason" do
+    let(:field) { "reason" }
+
+    context "when the lettings log has no reason value" do
+      let(:lettings_log) { LettingsLog.find_by(old_id: lettings_log_id) }
+
+      before do
+        Imports::LettingsLogsImportService.new(storage_service, logger).create_logs(fixture_directory)
+        lettings_log_file.rewind
+        lettings_log.update!(reason: nil, values_updated_at: nil)
+        lettings_log_xml.at_xpath("//xmlns:Q9a").content = "47"
+      end
+
+      it "updates the lettings_log reason value" do
+        expect(logger).to receive(:info).with(/lettings log \d+'s reason value has been set to 47/)
+        expect { import_service.send(:update_reason, lettings_log_xml) }
+          .to(change { lettings_log.reload.reason }.from(nil).to(47))
+        expect(lettings_log.values_updated_at).not_to be_nil
+      end
+    end
+
+    context "when the lettings log has a different reason value" do
+      let(:lettings_log) { LettingsLog.find_by(old_id: lettings_log_id) }
+
+      before do
+        Imports::LettingsLogsImportService.new(storage_service, logger).create_logs(fixture_directory)
+        lettings_log_file.rewind
+        lettings_log.update!(reason: 18, values_updated_at: nil)
+        lettings_log_xml.at_xpath("//xmlns:Q9a").content = "47"
+      end
+
+      it "does not update the lettings_log reason value" do
+        expect(logger).to receive(:info).with(/lettings log \d+ has a value for reason, skipping update/)
+        expect { import_service.send(:update_reason, lettings_log_xml) }
+          .not_to(change { lettings_log.reload.reason })
+        expect(lettings_log.values_updated_at).to be_nil
+      end
+    end
+
+    context "when the new value is 'other'" do
+      let(:lettings_log) { LettingsLog.find_by(old_id: lettings_log_id) }
+
+      before do
+        Imports::LettingsLogsImportService.new(storage_service, logger).create_logs(fixture_directory)
+        lettings_log_file.rewind
+        lettings_log.update!(reason: nil, values_updated_at: nil)
+        lettings_log_xml.at_xpath("//xmlns:Q9a").content = "20"
+      end
+
+      context "and other value is given" do
+        before do
+          lettings_log_xml.at_xpath("//xmlns:Q9aa").content = "other"
+        end
+
+        it "updates the lettings_log reason value" do
+          expect(logger).to receive(:info).with(/lettings log \d+'s reason value has been set to 20/)
+          expect(logger).to receive(:info).with(/lettings log \d+'s reasonother value has been set to other/)
+          expect { import_service.send(:update_reason, lettings_log_xml) }
+            .to(change { lettings_log.reload.reason }.from(nil).to(20))
+          expect(lettings_log.values_updated_at).not_to be_nil
+        end
+      end
+
+      context "and other value is not given" do
+        it "does not update the lettings_log reason value" do
+          expect(logger).to receive(:info).with(/lettings log \d+'s reason is other but other reason is not provided, skipping update/)
+          expect { import_service.send(:update_reason, lettings_log_xml) }
+            .not_to(change { lettings_log.reload.reason })
+          expect(lettings_log.values_updated_at).to be_nil
+        end
+      end
+    end
+  end
 end

spec/services/imports/lettings_logs_import_service_spec.rb

@@ -172,6 +172,46 @@ RSpec.describe Imports::LettingsLogsImportService do
         end
       end
 
+      context "and the user exists on a different organisation" do
+        before do
+          create(:legacy_user, old_user_id: "fake_id")
+          lettings_log_xml.at_xpath("//meta:owner-user-id").content = "fake_id"
+        end
+
+        it "creates a new unassigned user" do
+          expect(logger).to receive(:error).with("Lettings log '0ead17cb-1668-442d-898c-0d52879ff592' belongs to legacy user with owner-user-id: 'fake_id' which belongs to a different organisation. Assigning log to 'Unassigned' user.")
+          lettings_log_service.send(:create_log, lettings_log_xml)
+
+          lettings_log = LettingsLog.where(old_id: lettings_log_id).first
+          expect(lettings_log&.created_by&.name).to eq("Unassigned")
+        end
+
+        it "only creates one unassigned user" do
+          expect(logger).to receive(:error).with("Lettings log '0ead17cb-1668-442d-898c-0d52879ff592' belongs to legacy user with owner-user-id: 'fake_id' which belongs to a different organisation. Assigning log to 'Unassigned' user.")
+          expect(logger).to receive(:error).with("Lettings log 'fake_id' belongs to legacy user with owner-user-id: 'fake_id' which belongs to a different organisation. Assigning log to 'Unassigned' user.")
+          lettings_log_service.send(:create_log, lettings_log_xml)
+          lettings_log_xml.at_xpath("//meta:document-id").content = "fake_id"
+          lettings_log_service.send(:create_log, lettings_log_xml)
+
+          lettings_log = LettingsLog.where(old_id: lettings_log_id).first
+          second_lettings_log = LettingsLog.where(old_id: "fake_id").first
+          expect(lettings_log&.created_by).to eq(second_lettings_log&.created_by)
+        end
+
+        context "when unassigned user exist for a different organisation" do
+          let!(:other_unassigned_user) { create(:user, name: "Unassigned") }
+
+          it "creates a new unassigned user for current organisation" do
+            expect(logger).to receive(:error).with("Lettings log '0ead17cb-1668-442d-898c-0d52879ff592' belongs to legacy user with owner-user-id: 'fake_id' which belongs to a different organisation. Assigning log to 'Unassigned' user.")
+            lettings_log_service.send(:create_log, lettings_log_xml)
+
+            lettings_log = LettingsLog.where(old_id: lettings_log_id).first
+            expect(lettings_log&.created_by&.name).to eq("Unassigned")
+            expect(lettings_log&.created_by).not_to eq(other_unassigned_user)
+          end
+        end
+      end
+
       it "correctly sets imported at date" do
         lettings_log_service.send(:create_log, lettings_log_xml)
 
@@ -285,7 +325,7 @@ RSpec.describe Imports::LettingsLogsImportService do
             .not_to raise_error
         end
 
-        it "clears out the invalid answers" do
+        it "clears out the invalid tenancy length" do
           allow(logger).to receive(:warn)
 
           lettings_log_service.send(:create_log, lettings_log_xml)
@@ -293,7 +333,7 @@ RSpec.describe Imports::LettingsLogsImportService do
 
           expect(lettings_log).not_to be_nil
           expect(lettings_log.tenancylength).to be_nil
-          expect(lettings_log.tenancy).to be_nil
+          expect(lettings_log.tenancy).to eq(4)
         end
       end
 
@@ -505,6 +545,48 @@ RSpec.describe Imports::LettingsLogsImportService do
         end
       end
 
+      context "and the number of times the property was relet is 0.01" do
+        before do
+          lettings_log_xml.at_xpath("//xmlns:Q20").content = "0.01"
+        end
+
+        it "intercepts the relevant validation error" do
+          expect { lettings_log_service.send(:create_log, lettings_log_xml) }
+            .not_to raise_error
+        end
+
+        it "clears out the number offered answer" do
+          allow(logger).to receive(:warn)
+
+          lettings_log_service.send(:create_log, lettings_log_xml)
+          lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
+
+          expect(lettings_log).not_to be_nil
+          expect(lettings_log.offered).to be_nil
+        end
+      end
+
+      context "and the number of times the property was relet is a non nil string that is nil as a BigDecimal" do
+        before do
+          lettings_log_xml.at_xpath("//xmlns:Q20").content = "a"
+        end
+
+        it "doesn't throw an error" do
+          expect { lettings_log_service.send(:create_log, lettings_log_xml) }
+            .not_to raise_error
+        end
+
+        it "clears out the number offered answer" do
+          allow(logger).to receive(:warn)
+
+          lettings_log_service.send(:create_log, lettings_log_xml)
+          lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
+
+          expect(lettings_log).not_to be_nil
+          expect(lettings_log.offered).to be_nil
+        end
+      end
+
       context "and the gender identity is refused" do
         before do
           lettings_log_xml.at_xpath("//xmlns:P1Sex").content = "Person prefers not to say"
@@ -735,6 +817,29 @@ RSpec.describe Imports::LettingsLogsImportService do
         end
       end
 
+      context "and scharge is ever so slightly positive" do
+        let(:lettings_log_id) { "0b4a68df-30cc-474a-93c0-a56ce8fdad3b" }
+
+        before do
+          lettings_log_xml.at_xpath("//xmlns:Q18aii").content = "1.66533E-16"
+        end
+
+        it "does not raise an error" do
+          expect { lettings_log_service.send(:create_log, lettings_log_xml) }
+            .not_to raise_error
+        end
+
+        it "sets scharge to 0" do
+          allow(logger).to receive(:warn)
+
+          lettings_log_service.send(:create_log, lettings_log_xml)
+          lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
+
+          expect(lettings_log).not_to be_nil
+          expect(lettings_log.scharge).to eq(0)
+        end
+      end
+
       context "and tshortfall is not positive" do
         let(:lettings_log_id) { "0b4a68df-30cc-474a-93c0-a56ce8fdad3b" }
 
@@ -1164,6 +1269,25 @@ RSpec.describe Imports::LettingsLogsImportService do
         end
       end
 
+      context "and the scheme and location are not valid" do
+        let(:lettings_log_id) { "0b4a68df-30cc-474a-93c0-a56ce8fdad3b" }
+
+        before do
+          lettings_log_xml.at_xpath("//xmlns:_1cmangroupcode").content = "999"
+          lettings_log_xml.at_xpath("//xmlns:_1cschemecode").content = "999"
+        end
+
+        it "saves log without location and scheme" do
+          expect(logger).not_to receive(:warn)
+          lettings_log_service.send(:create_log, lettings_log_xml)
+          lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
+
+          expect(lettings_log.scheme_id).to be_nil
+          expect(lettings_log.location_id).to be_nil
+          expect(lettings_log.status).to eq("in_progress")
+        end
+      end
+
       context "and this is a supported housing log with a single location under a scheme" do
         let(:lettings_log_id) { "0b4a68df-30cc-474a-93c0-a56ce8fdad3b" }
 
@@ -1179,6 +1303,226 @@ RSpec.describe Imports::LettingsLogsImportService do
           expect(lettings_log.status).to eq("completed")
         end
       end
+
+      context "and there are several household members" do
+        context "and one person details are skipped" do
+          before do
+            lettings_log_xml.at_xpath("//xmlns:HHMEMB").content = 3
+            lettings_log_xml.at_xpath("//xmlns:P2AR").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Age").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Sex").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Rel").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Eco").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P3Age").content = 7
+            lettings_log_xml.at_xpath("//xmlns:P3Sex").content = "Male"
+            lettings_log_xml.at_xpath("//xmlns:P3Rel").content = "Child"
+            lettings_log_xml.at_xpath("//xmlns:P3Eco").content = "9) Child under 16"
+          end
+
+          it "correctly moves person details" do
+            lettings_log_service.send(:create_log, lettings_log_xml)
+
+            lettings_log = LettingsLog.where(old_id: lettings_log_id).first
+            expect(lettings_log&.hhmemb).to eq(3)
+            expect(lettings_log&.details_known_2).to eq(0)
+            expect(lettings_log&.age2_known).to eq(0)
+            expect(lettings_log&.age2).to eq(7)
+            expect(lettings_log&.sex2).to eq("M")
+            expect(lettings_log&.relat2).to eq("C")
+
+            expect(lettings_log&.details_known_3).to eq(0)
+            expect(lettings_log&.age3_known).to eq(0)
+            expect(lettings_log&.age3).to eq(nil)
+            expect(lettings_log&.sex3).to eq(nil)
+            expect(lettings_log&.relat3).to eq(nil)
+
+            [4, 5].each do |i|
+              expect(lettings_log&.send("details_known_#{i}")).to eq(nil)
+              expect(lettings_log&.send("age#{i}_known")).to eq(nil)
+              expect(lettings_log&.send("age#{i}")).to eq(nil)
+              expect(lettings_log&.send("sex#{i}")).to eq(nil)
+              expect(lettings_log&.send("relat#{i}")).to eq(nil)
+              expect(lettings_log&.send("ecstat#{i}")).to eq(nil)
+            end
+          end
+        end
+
+        context "and several consecutive person details are skipped" do
+          before do
+            lettings_log_xml.at_xpath("//xmlns:HHMEMB").content = 4
+
+            lettings_log_xml.at_xpath("//xmlns:P2AR").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Age").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Sex").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Rel").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Eco").content = nil
+
+            lettings_log_xml.at_xpath("//xmlns:P3AR").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P3Age").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P3Sex").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P3Rel").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P3Eco").content = nil
+
+            lettings_log_xml.at_xpath("//xmlns:P4Age").content = 7
+            lettings_log_xml.at_xpath("//xmlns:P4Sex").content = "Male"
+            lettings_log_xml.at_xpath("//xmlns:P4Rel").content = "Child"
+            lettings_log_xml.at_xpath("//xmlns:P4Eco").content = "9) Child under 16"
+
+            lettings_log_xml.at_xpath("//xmlns:P5Age").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P5Sex").content = "Male"
+            lettings_log_xml.at_xpath("//xmlns:P5Rel").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P5Eco").content = nil
+
+            lettings_log_xml.at_xpath("//xmlns:P6Age").content = 12
+            lettings_log_xml.at_xpath("//xmlns:P6Sex").content = "Female"
+            lettings_log_xml.at_xpath("//xmlns:P6Rel").content = "Child"
+            lettings_log_xml.at_xpath("//xmlns:P6Eco").content = "9) Child under 16"
+          end
+
+          it "correctly moves person details" do
+            lettings_log_service.send(:create_log, lettings_log_xml)
+
+            lettings_log = LettingsLog.where(old_id: lettings_log_id).first
+            expect(lettings_log&.hhmemb).to eq(4)
+            expect(lettings_log&.details_known_2).to eq(0)
+            expect(lettings_log&.age2_known).to eq(0)
+            expect(lettings_log&.age2).to eq(7)
+            expect(lettings_log&.sex2).to eq("M")
+            expect(lettings_log&.relat2).to eq("C")
+
+            expect(lettings_log&.details_known_3).to eq(0)
+            expect(lettings_log&.age3_known).to eq(0)
+            expect(lettings_log&.age3).to eq(nil)
+            expect(lettings_log&.sex3).to eq("M")
+            expect(lettings_log&.relat3).to eq(nil)
+
+            expect(lettings_log&.details_known_4).to eq(0)
+            expect(lettings_log&.age4_known).to eq(0)
+            expect(lettings_log&.age4).to eq(12)
+            expect(lettings_log&.sex4).to eq("F")
+            expect(lettings_log&.relat4).to eq("C")
+
+            [5, 6, 7, 8].each do |i|
+              expect(lettings_log&.send("details_known_#{i}")).to eq(nil)
+              expect(lettings_log&.send("age#{i}_known")).to eq(nil)
+              expect(lettings_log&.send("age#{i}")).to eq(nil)
+              expect(lettings_log&.send("sex#{i}")).to eq(nil)
+              expect(lettings_log&.send("relat#{i}")).to eq(nil)
+              expect(lettings_log&.send("ecstat#{i}")).to eq(nil)
+            end
+          end
+        end
+
+        context "and several non consecutive person details are skipped" do
+          before do
+            lettings_log_xml.at_xpath("//xmlns:HHMEMB").content = 4
+            lettings_log_xml.at_xpath("//xmlns:P2AR").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Age").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Sex").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Rel").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Eco").content = nil
+
+            lettings_log_xml.at_xpath("//xmlns:P3Age").content = 7
+            lettings_log_xml.at_xpath("//xmlns:P3Sex").content = "Male"
+            lettings_log_xml.at_xpath("//xmlns:P3Rel").content = "Child"
+            lettings_log_xml.at_xpath("//xmlns:P3Eco").content = "9) Child under 16"
+
+            lettings_log_xml.at_xpath("//xmlns:P4AR").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P4Age").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P4Sex").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P4Rel").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P4Eco").content = nil
+
+            lettings_log_xml.at_xpath("//xmlns:P5Age").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P5Sex").content = "Male"
+            lettings_log_xml.at_xpath("//xmlns:P5Rel").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P5Eco").content = nil
+
+            lettings_log_xml.at_xpath("//xmlns:P6Age").content = 12
+            lettings_log_xml.at_xpath("//xmlns:P6Sex").content = "Female"
+            lettings_log_xml.at_xpath("//xmlns:P6Rel").content = "Child"
+            lettings_log_xml.at_xpath("//xmlns:P6Eco").content = "9) Child under 16"
+          end
+
+          it "correctly moves person details" do
+            lettings_log_service.send(:create_log, lettings_log_xml)
+
+            lettings_log = LettingsLog.where(old_id: lettings_log_id).first
+            expect(lettings_log&.hhmemb).to eq(4)
+            expect(lettings_log&.details_known_2).to eq(0)
+            expect(lettings_log&.age2_known).to eq(0)
+            expect(lettings_log&.age2).to eq(7)
+            expect(lettings_log&.sex2).to eq("M")
+            expect(lettings_log&.relat2).to eq("C")
+
+            expect(lettings_log&.details_known_3).to eq(0)
+            expect(lettings_log&.age3_known).to eq(0)
+            expect(lettings_log&.age3).to eq(nil)
+            expect(lettings_log&.sex3).to eq("M")
+            expect(lettings_log&.relat3).to eq(nil)
+
+            expect(lettings_log&.details_known_4).to eq(0)
+            expect(lettings_log&.age4_known).to eq(0)
+            expect(lettings_log&.age4).to eq(12)
+            expect(lettings_log&.sex4).to eq("F")
+            expect(lettings_log&.relat4).to eq("C")
+
+            [5, 6, 7, 8].each do |i|
+              expect(lettings_log&.send("details_known_#{i}")).to eq(nil)
+              expect(lettings_log&.send("age#{i}_known")).to eq(nil)
+              expect(lettings_log&.send("age#{i}")).to eq(nil)
+              expect(lettings_log&.send("sex#{i}")).to eq(nil)
+              expect(lettings_log&.send("relat#{i}")).to eq(nil)
+              expect(lettings_log&.send("ecstat#{i}")).to eq(nil)
+            end
+          end
+        end
+
+        context "with 3 houusehold members without any person data" do
+          before do
+            lettings_log_xml.at_xpath("//xmlns:HHMEMB").content = 3
+            lettings_log_xml.at_xpath("//xmlns:P2AR").content = "No"
+            lettings_log_xml.at_xpath("//xmlns:P2Age").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Sex").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Rel").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Eco").content = nil
+
+            lettings_log_xml.at_xpath("//xmlns:P3AR").content = "No"
+            lettings_log_xml.at_xpath("//xmlns:P3Age").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P3Sex").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P3Rel").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P3Eco").content = nil
+          end
+
+          it "correctly sets person details" do
+            lettings_log_service.send(:create_log, lettings_log_xml)
+
+            lettings_log = LettingsLog.where(old_id: lettings_log_id).first
+            expect(lettings_log&.hhmemb).to eq(3)
+
+            expect(lettings_log&.details_known_2).to eq(0)
+            expect(lettings_log&.age2_known).to eq(1)
+            expect(lettings_log&.age2).to eq(nil)
+            expect(lettings_log&.sex2).to eq(nil)
+            expect(lettings_log&.relat2).to eq(nil)
+
+            expect(lettings_log&.details_known_3).to eq(0)
+            expect(lettings_log&.age3_known).to eq(1)
+            expect(lettings_log&.age3).to eq(nil)
+            expect(lettings_log&.sex3).to eq(nil)
+            expect(lettings_log&.relat3).to eq(nil)
+
+            [4, 5, 6, 7, 8].each do |i|
+              expect(lettings_log&.send("details_known_#{i}")).to eq(nil)
+              expect(lettings_log&.send("age#{i}_known")).to eq(nil)
+              expect(lettings_log&.send("age#{i}")).to eq(nil)
+              expect(lettings_log&.send("sex#{i}")).to eq(nil)
+              expect(lettings_log&.send("relat#{i}")).to eq(nil)
+              expect(lettings_log&.send("ecstat#{i}")).to eq(nil)
+            end
+          end
+        end
+      end
     end
   end
 

spec/services/imports/sales_logs_import_service_spec.rb

@@ -163,6 +163,48 @@ RSpec.describe Imports::SalesLogsImportService do
       end
     end
 
+    context "and the user exists on a different organisation" do
+      let(:sales_log_id) { "shared_ownership_sales_log" }
+
+      before do
+        create(:legacy_user, old_user_id: "fake_id")
+        sales_log_xml.at_xpath("//meta:owner-user-id").content = "fake_id"
+      end
+
+      it "creates a new unassigned user" do
+        expect(logger).to receive(:error).with("Sales log 'shared_ownership_sales_log' belongs to legacy user with owner-user-id: 'fake_id' which belongs to a different organisation. Assigning log to 'Unassigned' user.")
+        sales_log_service.send(:create_log, sales_log_xml)
+
+        sales_log = SalesLog.where(old_id: sales_log_id).first
+        expect(sales_log&.created_by&.name).to eq("Unassigned")
+      end
+
+      it "only creates one unassigned user" do
+        expect(logger).to receive(:error).with("Sales log 'shared_ownership_sales_log' belongs to legacy user with owner-user-id: 'fake_id' which belongs to a different organisation. Assigning log to 'Unassigned' user.")
+        expect(logger).to receive(:error).with("Sales log 'fake_id' belongs to legacy user with owner-user-id: 'fake_id' which belongs to a different organisation. Assigning log to 'Unassigned' user.")
+        sales_log_service.send(:create_log, sales_log_xml)
+        sales_log_xml.at_xpath("//meta:document-id").content = "fake_id"
+        sales_log_service.send(:create_log, sales_log_xml)
+
+        sales_log = SalesLog.where(old_id: sales_log_id).first
+        second_sales_log = SalesLog.where(old_id: "fake_id").first
+        expect(sales_log&.created_by).to eq(second_sales_log&.created_by)
+      end
+
+      context "when unassigned user exist for a different organisation" do
+        let!(:other_unassigned_user) { create(:user, name: "Unassigned") }
+
+        it "creates a new unassigned user for current organisation" do
+          expect(logger).to receive(:error).with("Sales log 'shared_ownership_sales_log' belongs to legacy user with owner-user-id: 'fake_id' which belongs to a different organisation. Assigning log to 'Unassigned' user.")
+          sales_log_service.send(:create_log, sales_log_xml)
+
+          sales_log = SalesLog.where(old_id: sales_log_id).first
+          expect(sales_log&.created_by&.name).to eq("Unassigned")
+          expect(sales_log&.created_by).not_to eq(other_unassigned_user)
+        end
+      end
+    end
+
     context "and the log startdate is before 22/23 collection period" do
       let(:sales_log_id) { "shared_ownership_sales_log" }
 
@@ -186,6 +228,31 @@ RSpec.describe Imports::SalesLogsImportService do
       end
     end
 
+    context "and the log startdate is only present in the CompletionDate field" do
+      let(:sales_log_id) { "shared_ownership_sales_log" }
+
+      before do
+        sales_log_xml.at_xpath("//xmlns:DAY").content = nil
+        sales_log_xml.at_xpath("//xmlns:MONTH").content = nil
+        sales_log_xml.at_xpath("//xmlns:YEAR").content = nil
+        sales_log_xml.at_xpath("//xmlns:CompletionDate").content = "2022-10-9"
+        sales_log_xml.at_xpath("//xmlns:HODAY").content = 9
+        sales_log_xml.at_xpath("//xmlns:HOMONTH").content = 10
+        sales_log_xml.at_xpath("//xmlns:HOYEAR").content = 2022
+        sales_log_xml.at_xpath("//xmlns:EXDAY").content = 9
+        sales_log_xml.at_xpath("//xmlns:EXMONTH").content = 10
+        sales_log_xml.at_xpath("//xmlns:EXYEAR").content = 2022
+      end
+
+      it "creates the log with the correct saledate" do
+        expect(logger).not_to receive(:error)
+        expect(logger).not_to receive(:warn)
+        expect { sales_log_service.send(:create_log, sales_log_xml) }
+          .to change(SalesLog, :count).by(1)
+        expect(SalesLog.last.saledate).to eq(Time.zone.local(2022, 10, 9))
+      end
+    end
+
     context "when the log is valid" do
       let(:sales_log_id) { "shared_ownership_sales_log" }
 
@@ -342,6 +409,42 @@ RSpec.describe Imports::SalesLogsImportService do
           sales_log = SalesLog.find_by(old_id: sales_log_id)
           expect(sales_log.proplen_asked).to eq(1)
         end
+
+        context "when setting soctenant fields" do
+          it "does not set soctenant value if none of the soctenant questions are answered" do
+            sales_log_xml.at_xpath("//xmlns:Q20Bedrooms").content = nil
+            sales_log_xml.at_xpath("//xmlns:PrevRentType").content = nil
+            sales_log_xml.at_xpath("//xmlns:Q21PropertyType").content = nil
+
+            expect(logger).not_to receive(:error)
+            expect(logger).not_to receive(:warn)
+            expect(logger).not_to receive(:info)
+            expect { sales_log_service.send(:create_log, sales_log_xml) }
+            .to change(SalesLog, :count).by(1)
+            sales_log = SalesLog.find_by(old_id: sales_log_id)
+            expect(sales_log.soctenant).to eq(nil)
+            expect(sales_log.frombeds).to eq(nil)
+            expect(sales_log.fromprop).to eq(nil)
+            expect(sales_log.socprevten).to eq(nil)
+          end
+
+          it "sets soctenant to don't know if any of the soctenant questions are answered" do
+            sales_log_xml.at_xpath("//xmlns:Q20Bedrooms").content = "2"
+            sales_log_xml.at_xpath("//xmlns:PrevRentType").content = nil
+            sales_log_xml.at_xpath("//xmlns:Q21PropertyType").content = nil
+
+            expect(logger).not_to receive(:error)
+            expect(logger).not_to receive(:warn)
+            expect(logger).not_to receive(:info)
+            expect { sales_log_service.send(:create_log, sales_log_xml) }
+            .to change(SalesLog, :count).by(1)
+            sales_log = SalesLog.find_by(old_id: sales_log_id)
+            expect(sales_log.soctenant).to eq(0)
+            expect(sales_log.frombeds).to eq(2)
+            expect(sales_log.fromprop).to eq(0)
+            expect(sales_log.socprevten).to eq(10)
+          end
+        end
       end
 
       context "with discounted sale type" do
@@ -553,6 +656,56 @@ RSpec.describe Imports::SalesLogsImportService do
       end
     end
 
+    context "and the hodate soft validation is triggered (hodate_value_check)" do
+      let(:sales_log_id) { "shared_ownership_sales_log" }
+
+      before do
+        sales_log_xml.at_xpath("//xmlns:HODAY").content = "1"
+        sales_log_xml.at_xpath("//xmlns:HOMONTH").content = "1"
+        sales_log_xml.at_xpath("//xmlns:HOYEAR").content = "2018"
+      end
+
+      it "completes the log" do
+        sales_log_service.send(:create_log, sales_log_xml)
+        sales_log = SalesLog.find_by(old_id: sales_log_id)
+        expect(sales_log.status).to eq("completed")
+      end
+    end
+
+    context "and the combined income soft validation is triggered (combined_income_value_check)" do
+      let(:sales_log_id) { "shared_ownership_sales_log" }
+
+      before do
+        sales_log_xml.at_xpath("//xmlns:joint").content = "1 Yes"
+        sales_log_xml.at_xpath("//xmlns:Q2Person1Income").content = "45000"
+        sales_log_xml.at_xpath("//xmlns:P2IncKnown").content = "1 Yes"
+        sales_log_xml.at_xpath("//xmlns:Q2Person2Income").content = "45000"
+      end
+
+      it "completes the log" do
+        sales_log_service.send(:create_log, sales_log_xml)
+        sales_log = SalesLog.find_by(old_id: sales_log_id)
+        expect(sales_log.status).to eq("completed")
+      end
+    end
+
+    context "and the stairowned soft validation is triggered (stairowned_value_check)" do
+      let(:sales_log_id) { "shared_ownership_sales_log" }
+
+      before do
+        sales_log_xml.at_xpath("//xmlns:DerSaleType").content = "24"
+        sales_log_xml.at_xpath("//xmlns:PercentOwns").content = "77"
+        sales_log_xml.at_xpath("//xmlns:Q17aStaircase").content = "1"
+        sales_log_xml.at_xpath("//xmlns:PercentBought").content = "10"
+      end
+
+      it "completes the log" do
+        sales_log_service.send(:create_log, sales_log_xml)
+        sales_log = SalesLog.find_by(old_id: sales_log_id)
+        expect(sales_log.status).to eq("completed")
+      end
+    end
+
     context "and it has an invalid record with invalid child age" do
       let(:sales_log_id) { "discounted_ownership_sales_log" }
 
@@ -808,7 +961,7 @@ RSpec.describe Imports::SalesLogsImportService do
           .not_to raise_error
       end
 
-      it "clears out the referral answer" do
+      it "clears out the mortgage answer" do
         allow(logger).to receive(:warn)
 
         sales_log_service.send(:create_log, sales_log_xml)
@@ -819,6 +972,29 @@ RSpec.describe Imports::SalesLogsImportService do
       end
     end
 
+    context "when proplen is out of range" do
+      let(:sales_log_id) { "shared_ownership_sales_log" }
+
+      before do
+        sales_log_xml.at_xpath("//xmlns:Q16aProplen2").content = "2000"
+      end
+
+      it "intercepts the relevant validation error" do
+        expect { sales_log_service.send(:create_log, sales_log_xml) }
+          .not_to raise_error
+      end
+
+      it "clears out the proplen answer" do
+        allow(logger).to receive(:warn)
+
+        sales_log_service.send(:create_log, sales_log_xml)
+        sales_log = SalesLog.find_by(old_id: sales_log_id)
+
+        expect(sales_log).not_to be_nil
+        expect(sales_log.proplen).to be_nil
+      end
+    end
+
     context "and it has an invalid income 1" do
       let(:sales_log_id) { "shared_ownership_sales_log" }
 
@@ -869,6 +1045,34 @@ RSpec.describe Imports::SalesLogsImportService do
       end
     end
 
+    context "and it has an invalid combined income outside london" do
+      let(:sales_log_id) { "shared_ownership_sales_log" }
+
+      before do
+        sales_log_xml.at_xpath("//xmlns:joint").content = "1 Yes"
+        sales_log_xml.at_xpath("//xmlns:Q2Person1Income").content = "45000"
+        sales_log_xml.at_xpath("//xmlns:P2IncKnown").content = "1 Yes"
+        sales_log_xml.at_xpath("//xmlns:Q2Person2Income").content = "40000"
+        sales_log_xml.at_xpath("//xmlns:Q14ONSLACode").content = "E07000223"
+      end
+
+      it "intercepts the relevant validation error" do
+        expect { sales_log_service.send(:create_log, sales_log_xml) }
+          .not_to raise_error
+      end
+
+      it "clears out the invalid answers" do
+        allow(logger).to receive(:warn)
+
+        sales_log_service.send(:create_log, sales_log_xml)
+        sales_log = SalesLog.find_by(old_id: sales_log_id)
+
+        expect(sales_log).not_to be_nil
+        expect(sales_log.income1).to be_nil
+        expect(sales_log.income2).to be_nil
+      end
+    end
+
     context "and it has an invalid income 1 for london" do
       let(:sales_log_id) { "shared_ownership_sales_log" }
 
@@ -919,6 +1123,34 @@ RSpec.describe Imports::SalesLogsImportService do
       end
     end
 
+    context "and it has an invalid combined income for london" do
+      let(:sales_log_id) { "shared_ownership_sales_log" }
+
+      before do
+        sales_log_xml.at_xpath("//xmlns:joint").content = "1 Yes"
+        sales_log_xml.at_xpath("//xmlns:Q2Person1Income").content = "50000"
+        sales_log_xml.at_xpath("//xmlns:P2IncKnown").content = "1 Yes"
+        sales_log_xml.at_xpath("//xmlns:Q2Person2Income").content = "45000"
+        sales_log_xml.at_xpath("//xmlns:Q14ONSLACode").content = "E09000012"
+      end
+
+      it "intercepts the relevant validation error" do
+        expect { sales_log_service.send(:create_log, sales_log_xml) }
+          .not_to raise_error
+      end
+
+      it "clears out the invalid answers" do
+        allow(logger).to receive(:warn)
+
+        sales_log_service.send(:create_log, sales_log_xml)
+        sales_log = SalesLog.find_by(old_id: sales_log_id)
+
+        expect(sales_log).not_to be_nil
+        expect(sales_log.income1).to be_nil
+        expect(sales_log.income2).to be_nil
+      end
+    end
+
     context "and it has an invalid mscharge" do
       let(:sales_log_id) { "shared_ownership_sales_log" }
 
@@ -1895,6 +2127,7 @@ RSpec.describe Imports::SalesLogsImportService do
 
           sales_log = SalesLog.find_by(old_id: sales_log_id)
           expect(sales_log&.fromprop).to be(0)
+          expect(sales_log&.soctenant).to be(0)
         end
       end
 
@@ -1911,6 +2144,7 @@ RSpec.describe Imports::SalesLogsImportService do
 
           sales_log = SalesLog.find_by(old_id: sales_log_id)
           expect(sales_log&.socprevten).to be(10)
+          expect(sales_log&.soctenant).to be(0)
         end
       end