From 2256cc9d73517e8cd1fd201cd22285eb7c5fedcd Mon Sep 17 00:00:00 2001
From: natdeanlewissoftwire
 <94526761+natdeanlewissoftwire@users.noreply.github.com>
Date: Fri, 1 Sep 2023 12:26:16 +0100
Subject: [PATCH 01/33] Migration fixes (#1881)

* feat: make offered unsafe as 0.00 is an expected value

* feat: add correct refused options for sex and relat questions

* refactor: lint

* feat: use safe_string_as_decimal
---
 .../imports/lettings_logs_import_service.rb   |  2 +-
 app/services/imports/logs_import_service.rb   |  4 +-
 .../lettings_logs_import_service_spec.rb      | 57 +++++++++++++++++++
 3 files changed, 60 insertions(+), 3 deletions(-)

diff --git a/app/services/imports/lettings_logs_import_service.rb b/app/services/imports/lettings_logs_import_service.rb
index a1c957b85..8bcaba27b 100644
--- a/app/services/imports/lettings_logs_import_service.rb
+++ b/app/services/imports/lettings_logs_import_service.rb
@@ -174,7 +174,7 @@ module Imports
                                      0
                                    end
 
-      attributes["offered"] = safe_string_as_integer(xml_doc, "Q20")
+      attributes["offered"] = safe_string_as_decimal(xml_doc, "Q20")
       attributes["propcode"] = string_or_nil(xml_doc, "Q21a")
       attributes["beds"] = safe_string_as_integer(xml_doc, "Q22")
       attributes["unittype_gn"] = unsafe_string_as_integer(xml_doc, "Q23")
diff --git a/app/services/imports/logs_import_service.rb b/app/services/imports/logs_import_service.rb
index c6c158089..930956f16 100644
--- a/app/services/imports/logs_import_service.rb
+++ b/app/services/imports/logs_import_service.rb
@@ -106,7 +106,7 @@ module Imports
         "F"
       when "Other", "Non-binary"
         "X"
-      when "Refused"
+      when "Refused", "Person prefers not to say", "Buyer prefers not to say"
         "R"
       end
     end
@@ -120,7 +120,7 @@ module Imports
         "P"
       when "Other", "Non-binary"
         "X"
-      when "Refused", "Buyer prefers not to say"
+      when "Refused", "Person prefers not to say", "Buyer prefers not to say"
         "R"
       end
     end
diff --git a/spec/services/imports/lettings_logs_import_service_spec.rb b/spec/services/imports/lettings_logs_import_service_spec.rb
index 07fd6996e..d055db112 100644
--- a/spec/services/imports/lettings_logs_import_service_spec.rb
+++ b/spec/services/imports/lettings_logs_import_service_spec.rb
@@ -442,6 +442,63 @@ RSpec.describe Imports::LettingsLogsImportService do
         end
       end
 
+      context "and the number of times the property was relet is 0.00" do
+        before do
+          lettings_log_xml.at_xpath("//xmlns:Q20").content = "0.00"
+        end
+
+        it "does not raise an error" do
+          expect { lettings_log_service.send(:create_log, lettings_log_xml) }
+            .not_to raise_error
+        end
+
+        it "does not clear offered answer" do
+          lettings_log_service.send(:create_log, lettings_log_xml)
+          lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
+
+          expect(lettings_log).not_to be_nil
+          expect(lettings_log.offered).to equal(0)
+        end
+      end
+
+      context "and the gender identity is refused" do
+        before do
+          lettings_log_xml.at_xpath("//xmlns:P1Sex").content = "Person prefers not to say"
+        end
+
+        it "does not raise an error" do
+          expect { lettings_log_service.send(:create_log, lettings_log_xml) }
+            .not_to raise_error
+        end
+
+        it "saves the correct answer" do
+          lettings_log_service.send(:create_log, lettings_log_xml)
+          lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
+
+          expect(lettings_log).not_to be_nil
+          expect(lettings_log.sex1).to eq("R")
+        end
+      end
+
+      context "and the relationship is refused" do
+        before do
+          lettings_log_xml.at_xpath("//xmlns:P2Rel").content = "Person prefers not to say"
+        end
+
+        it "does not raise an error" do
+          expect { lettings_log_service.send(:create_log, lettings_log_xml) }
+            .not_to raise_error
+        end
+
+        it "saves the correct answer" do
+          lettings_log_service.send(:create_log, lettings_log_xml)
+          lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
+
+          expect(lettings_log).not_to be_nil
+          expect(lettings_log.relat2).to eq("R")
+        end
+      end
+
       context "when the log being imported was manually entered" do
         it "sets the creation method correctly" do
           lettings_log_service.send(:create_log, lettings_log_xml)

From 3437f9725e4908cb473f3aad46955e6c31f09c89 Mon Sep 17 00:00:00 2001
From: kosiakkatrina <54268893+kosiakkatrina@users.noreply.github.com>
Date: Fri, 1 Sep 2023 14:04:38 +0200
Subject: [PATCH 02/33] CLDC-2753 Add unassigned user for imported logs (#1883)

* Add unassigned user if we can't find a legacy user for a log

* Add generate_unassigned_logs_report

* Log user email address if validation fails on user import
---
 app/services/imports/import_report_service.rb | 31 +++++++++++++++++++
 .../imports/lettings_logs_import_service.rb   | 17 ++++++++++
 .../imports/sales_logs_import_service.rb      | 17 ++++++++++
 app/services/imports/user_import_service.rb   | 12 +++++--
 .../imports/import_report_service_spec.rb     | 31 +++++++++++++++++++
 .../lettings_logs_import_service_spec.rb      | 24 ++++++++++++++
 .../imports/sales_logs_import_service_spec.rb | 26 ++++++++++++++++
 .../imports/user_import_service_spec.rb       | 16 +++++++++-
 8 files changed, 170 insertions(+), 4 deletions(-)

diff --git a/app/services/imports/import_report_service.rb b/app/services/imports/import_report_service.rb
index 61d43a9eb..4e387c4c4 100644
--- a/app/services/imports/import_report_service.rb
+++ b/app/services/imports/import_report_service.rb
@@ -11,6 +11,7 @@ module Imports
     def create_reports(report_suffix)
       generate_missing_data_coordinators_report(report_suffix)
       generate_logs_report(report_suffix)
+      generate_unassigned_logs_report(report_suffix)
     end
 
     def generate_missing_data_coordinators_report(report_suffix)
@@ -53,5 +54,35 @@ module Imports
 
       @logger.info("Logs report available in s3 import bucket at #{report_name}")
     end
+
+    def generate_unassigned_logs_report(report_suffix)
+      Rails.logger.info("Generating unassigned logs report")
+
+      rep = CSV.generate do |report|
+        headers = ["Owning Organisation ID", "Old Owning Organisation ID", "Managing Organisation ID", "Old Managing Organisation ID", "Log ID", "Old Log ID", "Tenancy code", "Purchaser code"]
+        report << headers
+
+        @institutions_csv.each do |row|
+          name = row[0]
+          organisation = Organisation.find_by(name:)
+          next unless organisation
+
+          unassigned_user = organisation.users.find_by(name: "Unassigned")
+          next unless unassigned_user
+
+          organisation.owned_lettings_logs.where(created_by: unassigned_user).each do |lettings_log|
+            report << [organisation.id, organisation.old_org_id, lettings_log.managing_organisation.id, lettings_log.managing_organisation.old_org_id, lettings_log.id, lettings_log.old_id, lettings_log.tenancycode, nil]
+          end
+          organisation.owned_sales_logs.where(created_by: unassigned_user).each do |sales_log|
+            report << [organisation.id, organisation.old_org_id, nil, nil, sales_log.id, sales_log.old_id, nil, sales_log.purchid]
+          end
+        end
+      end
+
+      report_name = "UnassignedLogsReport_#{report_suffix}"
+      @storage_service.write_file(report_name, BYTE_ORDER_MARK + rep)
+
+      @logger.info("Unassigned logs report available in s3 import bucket at #{report_name}")
+    end
   end
 end
diff --git a/app/services/imports/lettings_logs_import_service.rb b/app/services/imports/lettings_logs_import_service.rb
index 8bcaba27b..7b749524a 100644
--- a/app/services/imports/lettings_logs_import_service.rb
+++ b/app/services/imports/lettings_logs_import_service.rb
@@ -251,6 +251,23 @@ module Imports
       owner_id = meta_field_value(xml_doc, "owner-user-id").strip
       if owner_id.present?
         user = LegacyUser.find_by(old_user_id: owner_id)&.user
+        if user.blank?
+          @logger.error("Lettings log '#{attributes['old_id']}' belongs to legacy user with owner-user-id: '#{owner_id}' which cannot be found. Assigning log to 'Unassigned' user.")
+          if User.find_by(name: "Unassigned")
+            user = User.find_by(name: "Unassigned")
+          else
+            user = User.new(
+              name: "Unassigned",
+              organisation_id: attributes["managing_organisation_id"],
+              is_dpo: false,
+              encrypted_password: SecureRandom.hex(10),
+              email: SecureRandom.uuid,
+              confirmed_at: Time.zone.now,
+              active: false,
+            )
+            user.save!(validate: false)
+          end
+        end
 
         attributes["created_by"] = user
       end
diff --git a/app/services/imports/sales_logs_import_service.rb b/app/services/imports/sales_logs_import_service.rb
index 0d780c87d..723159198 100644
--- a/app/services/imports/sales_logs_import_service.rb
+++ b/app/services/imports/sales_logs_import_service.rb
@@ -179,6 +179,23 @@ module Imports
       if owner_id.present?
         user = LegacyUser.find_by(old_user_id: owner_id)&.user
 
+        if user.blank?
+          @logger.error("Sales log '#{attributes['old_id']}' belongs to legacy user with owner-user-id: '#{owner_id}' which cannot be found. Assigning log to 'Unassigned' user.")
+          if User.find_by(name: "Unassigned")
+            user = User.find_by(name: "Unassigned")
+          else
+            user = User.new(
+              name: "Unassigned",
+              organisation_id: attributes["managing_organisation_id"],
+              is_dpo: false,
+              encrypted_password: SecureRandom.hex(10),
+              email: SecureRandom.uuid,
+              confirmed_at: Time.zone.now,
+              active: false,
+            )
+            user.save!(validate: false)
+          end
+        end
         attributes["created_by"] = user
       end
 
diff --git a/app/services/imports/user_import_service.rb b/app/services/imports/user_import_service.rb
index b0fe32bef..e280dcfe1 100644
--- a/app/services/imports/user_import_service.rb
+++ b/app/services/imports/user_import_service.rb
@@ -39,9 +39,15 @@ module Imports
         user.active = user_field_value(xml_document, "active")
 
         user.skip_confirmation_notification!
-        user.save!
-        user.legacy_users.create!(old_user_id:)
-        user
+
+        begin
+          user.save!
+          user.legacy_users.create!(old_user_id:)
+          user
+        rescue ActiveRecord::RecordInvalid => e
+          @logger.error(e.message)
+          @logger.error("Could not save user with email: #{email}")
+        end
       end
     end
 
diff --git a/spec/services/imports/import_report_service_spec.rb b/spec/services/imports/import_report_service_spec.rb
index 1d241754d..abd318a8d 100644
--- a/spec/services/imports/import_report_service_spec.rb
+++ b/spec/services/imports/import_report_service_spec.rb
@@ -67,4 +67,35 @@ RSpec.describe Imports::ImportReportService do
       report_service.generate_logs_report("report_suffix.csv")
     end
   end
+
+  describe "#generate_unassigned_logs_report" do
+    context "when there is no unassigned user (all the logs have ben assigned)" do
+      let(:institutions_csv) { CSV.parse("Institution name,Id,Old Completed lettings logs,Old In progress lettings logs,Old Completed sales logs,Old In progress sales logs\norg1,1,2,1,4,3", headers: true) }
+
+      it "writes an empty unassigned logs report" do
+        expect(storage_service).to receive(:write_file).with("UnassignedLogsReport_report_suffix.csv", "\uFEFFOwning Organisation ID,Old Owning Organisation ID,Managing Organisation ID,Old Managing Organisation ID,Log ID,Old Log ID,Tenancy code,Purchaser code\n")
+
+        report_service.generate_unassigned_logs_report("report_suffix.csv")
+      end
+    end
+
+    context "when some logs have been added to Unassigned user" do
+      let(:organisation) { create(:organisation, old_org_id: "1", name: "org1") }
+      let(:organisation2) { create(:organisation, old_org_id: "2", name: "org2") }
+      let(:unassigned_user) { create(:user, name: "Unassigned", organisation:) }
+      let(:institutions_csv) { CSV.parse("Institution name,Id,Old Completed lettings logs,Old In progress lettings logs,Old Completed sales logs,Old In progress sales logs\norg1,1,2,1,4,3", headers: true) }
+      let!(:lettings_log) { create(:lettings_log, owning_organisation: organisation, managing_organisation: organisation2, created_by: unassigned_user, tenancycode: "tenancycode", old_id: "12") }
+      let!(:sales_log) { create(:sales_log, owning_organisation: organisation, created_by: unassigned_user, purchid: "purchid", old_id: "23") }
+
+      before do
+        create(:organisation_relationship, parent_organisation: organisation, child_organisation: organisation2)
+      end
+
+      it "writes a report with all unassigned logs" do
+        expect(storage_service).to receive(:write_file).with("UnassignedLogsReport_report_suffix.csv", "\uFEFFOwning Organisation ID,Old Owning Organisation ID,Managing Organisation ID,Old Managing Organisation ID,Log ID,Old Log ID,Tenancy code,Purchaser code\n#{organisation.id},1,#{organisation2.id},2,#{lettings_log.id},12,tenancycode,\n#{organisation.id},1,,,#{sales_log.id},23,,purchid\n")
+
+        report_service.generate_unassigned_logs_report("report_suffix.csv")
+      end
+    end
+  end
 end
diff --git a/spec/services/imports/lettings_logs_import_service_spec.rb b/spec/services/imports/lettings_logs_import_service_spec.rb
index d055db112..a95cdac5b 100644
--- a/spec/services/imports/lettings_logs_import_service_spec.rb
+++ b/spec/services/imports/lettings_logs_import_service_spec.rb
@@ -135,6 +135,30 @@ RSpec.describe Imports::LettingsLogsImportService do
       let(:lettings_log_file) { open_file(fixture_directory, lettings_log_id) }
       let(:lettings_log_xml) { Nokogiri::XML(lettings_log_file) }
 
+      context "and the user does not exist" do
+        before { lettings_log_xml.at_xpath("//meta:owner-user-id").content = "fake_id" }
+
+        it "creates a new unassigned user" do
+          expect(logger).to receive(:error).with("Lettings log '0ead17cb-1668-442d-898c-0d52879ff592' belongs to legacy user with owner-user-id: 'fake_id' which cannot be found. Assigning log to 'Unassigned' user.")
+          lettings_log_service.send(:create_log, lettings_log_xml)
+
+          lettings_log = LettingsLog.where(old_id: lettings_log_id).first
+          expect(lettings_log&.created_by&.name).to eq("Unassigned")
+        end
+
+        it "only creates one unassigned user" do
+          expect(logger).to receive(:error).with("Lettings log '0ead17cb-1668-442d-898c-0d52879ff592' belongs to legacy user with owner-user-id: 'fake_id' which cannot be found. Assigning log to 'Unassigned' user.")
+          expect(logger).to receive(:error).with("Lettings log 'fake_id' belongs to legacy user with owner-user-id: 'fake_id' which cannot be found. Assigning log to 'Unassigned' user.")
+          lettings_log_service.send(:create_log, lettings_log_xml)
+          lettings_log_xml.at_xpath("//meta:document-id").content = "fake_id"
+          lettings_log_service.send(:create_log, lettings_log_xml)
+
+          lettings_log = LettingsLog.where(old_id: lettings_log_id).first
+          second_lettings_log = LettingsLog.where(old_id: "fake_id").first
+          expect(lettings_log&.created_by).to eq(second_lettings_log&.created_by)
+        end
+      end
+
       context "and the void date is after the start date" do
         before { lettings_log_xml.at_xpath("//xmlns:VYEAR").content = 2023 }
 
diff --git a/spec/services/imports/sales_logs_import_service_spec.rb b/spec/services/imports/sales_logs_import_service_spec.rb
index 740614775..8f12be392 100644
--- a/spec/services/imports/sales_logs_import_service_spec.rb
+++ b/spec/services/imports/sales_logs_import_service_spec.rb
@@ -116,6 +116,32 @@ RSpec.describe Imports::SalesLogsImportService do
       end
     end
 
+    context "and the user does not exist" do
+      let(:sales_log_id) { "shared_ownership_sales_log" }
+
+      before { sales_log_xml.at_xpath("//meta:owner-user-id").content = "fake_id" }
+
+      it "creates a new unassigned user" do
+        expect(logger).to receive(:error).with("Sales log 'shared_ownership_sales_log' belongs to legacy user with owner-user-id: 'fake_id' which cannot be found. Assigning log to 'Unassigned' user.")
+        sales_log_service.send(:create_log, sales_log_xml)
+
+        sales_log = SalesLog.where(old_id: sales_log_id).first
+        expect(sales_log&.created_by&.name).to eq("Unassigned")
+      end
+
+      it "only creates one unassigned user" do
+        expect(logger).to receive(:error).with("Sales log 'shared_ownership_sales_log' belongs to legacy user with owner-user-id: 'fake_id' which cannot be found. Assigning log to 'Unassigned' user.")
+        expect(logger).to receive(:error).with("Sales log 'fake_id' belongs to legacy user with owner-user-id: 'fake_id' which cannot be found. Assigning log to 'Unassigned' user.")
+        sales_log_service.send(:create_log, sales_log_xml)
+        sales_log_xml.at_xpath("//meta:document-id").content = "fake_id"
+        sales_log_service.send(:create_log, sales_log_xml)
+
+        sales_log = SalesLog.where(old_id: sales_log_id).first
+        second_sales_log = SalesLog.where(old_id: "fake_id").first
+        expect(sales_log&.created_by).to eq(second_sales_log&.created_by)
+      end
+    end
+
     context "and the log startdate is before 22/23 collection period" do
       let(:sales_log_id) { "shared_ownership_sales_log" }
 
diff --git a/spec/services/imports/user_import_service_spec.rb b/spec/services/imports/user_import_service_spec.rb
index ea3388169..5e56b2782 100644
--- a/spec/services/imports/user_import_service_spec.rb
+++ b/spec/services/imports/user_import_service_spec.rb
@@ -44,10 +44,24 @@ RSpec.describe Imports::UserImportService do
     end
 
     it "refuses to create a user belonging to a non existing organisation" do
-      expect(logger).to receive(:error).with(/ActiveRecord::RecordInvalid/)
+      expect(logger).to receive(:error).with(/Could not save user with email: john.doe@gov.uk/)
+      expect(logger).to receive(:error).with(/Validation failed: Organisation Select the user’s organisation/)
       import_service.create_users("user_directory")
     end
 
+    context "when the user with the same email already exists" do
+      before do
+        create(:organisation, old_org_id:)
+        create(:user, email: "john.doe@gov.uk")
+      end
+
+      it "logs an error and user email" do
+        expect(logger).to receive(:error).with(/Could not save user with email: john.doe@gov.uk/)
+        expect(logger).to receive(:error).with(/Validation failed: email Enter an email address that hasn’t already been used to sign up/)
+        import_service.create_users("user_directory")
+      end
+    end
+
     context "when the user is a data coordinator" do
       let(:old_user_id) { "d4729b1a5dfb68bb1e01c08445830c0add40907c" }
 

From 4d275a5c0267f8dcd41a52be8034bd003dfe98f3 Mon Sep 17 00:00:00 2001
From: kosiakkatrina <54268893+kosiakkatrina@users.noreply.github.com>
Date: Fri, 1 Sep 2023 17:13:33 +0200
Subject: [PATCH 03/33] CLDC-2689 Export migrated logs (#1873)

* Add imported_at columns

* Set imported at date for lettings logs

* Set imported at for sales logs

* Export logs migrated since the previous partial export

* Rename column

* rebase changes

* Re-add test
---
 .../exports/lettings_log_export_service.rb    |  2 +-
 .../imports/lettings_logs_import_service.rb   |  1 +
 .../imports/sales_logs_import_service.rb      |  1 +
 .../20230828145454_add_migrated_on_fields.rb  |  6 ++++++
 db/schema.rb                                  |  4 +++-
 .../lettings_log_export_service_spec.rb       | 20 +++++++++++++++++++
 .../lettings_logs_import_service_spec.rb      |  7 +++++++
 .../imports/sales_logs_import_service_spec.rb |  8 ++++++++
 8 files changed, 47 insertions(+), 2 deletions(-)
 create mode 100644 db/migrate/20230828145454_add_migrated_on_fields.rb

diff --git a/app/services/exports/lettings_log_export_service.rb b/app/services/exports/lettings_log_export_service.rb
index e09af1c29..9bcc8676c 100644
--- a/app/services/exports/lettings_log_export_service.rb
+++ b/app/services/exports/lettings_log_export_service.rb
@@ -119,7 +119,7 @@ module Exports
     def retrieve_lettings_logs(start_time, recent_export, full_update)
       if !full_update && recent_export
         params = { from: recent_export.started_at, to: start_time }
-        LettingsLog.exportable.where("updated_at >= :from and updated_at <= :to", params)
+        LettingsLog.exportable.where("(updated_at >= :from AND updated_at <= :to) OR (values_updated_at IS NOT NULL AND values_updated_at >= :from AND values_updated_at <= :to)", params)
       else
         params = { to: start_time }
         LettingsLog.exportable.where("updated_at <= :to", params)
diff --git a/app/services/imports/lettings_logs_import_service.rb b/app/services/imports/lettings_logs_import_service.rb
index 7b749524a..b7e2673d0 100644
--- a/app/services/imports/lettings_logs_import_service.rb
+++ b/app/services/imports/lettings_logs_import_service.rb
@@ -271,6 +271,7 @@ module Imports
 
         attributes["created_by"] = user
       end
+      attributes["values_updated_at"] = Time.zone.now
 
       apply_date_consistency!(attributes)
       apply_household_consistency!(attributes)
diff --git a/app/services/imports/sales_logs_import_service.rb b/app/services/imports/sales_logs_import_service.rb
index 723159198..8a8156480 100644
--- a/app/services/imports/sales_logs_import_service.rb
+++ b/app/services/imports/sales_logs_import_service.rb
@@ -198,6 +198,7 @@ module Imports
         end
         attributes["created_by"] = user
       end
+      attributes["values_updated_at"] = Time.zone.now
 
       set_default_values(attributes) if previous_status.include?("submitted")
       sales_log = save_sales_log(attributes, previous_status)
diff --git a/db/migrate/20230828145454_add_migrated_on_fields.rb b/db/migrate/20230828145454_add_migrated_on_fields.rb
new file mode 100644
index 000000000..afac3925f
--- /dev/null
+++ b/db/migrate/20230828145454_add_migrated_on_fields.rb
@@ -0,0 +1,6 @@
+class AddMigratedOnFields < ActiveRecord::Migration[7.0]
+  def change
+    add_column :lettings_logs, :values_updated_at, :datetime
+    add_column :sales_logs, :values_updated_at, :datetime
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index fe5b6cde4..7218e75a5 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2023_08_16_083950) do
+ActiveRecord::Schema[7.0].define(version: 2023_08_28_145454) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
@@ -297,6 +297,7 @@ ActiveRecord::Schema[7.0].define(version: 2023_08_16_083950) do
     t.integer "status_cache", default: 0, null: false
     t.datetime "discarded_at"
     t.integer "creation_method", default: 1
+    t.datetime "values_updated_at"
     t.index ["bulk_upload_id"], name: "index_lettings_logs_on_bulk_upload_id"
     t.index ["created_by_id"], name: "index_lettings_logs_on_created_by_id"
     t.index ["location_id"], name: "index_lettings_logs_on_location_id"
@@ -617,6 +618,7 @@ ActiveRecord::Schema[7.0].define(version: 2023_08_16_083950) do
     t.integer "stairowned_value_check"
     t.integer "creation_method", default: 1
     t.integer "old_form_id"
+    t.datetime "values_updated_at"
     t.index ["bulk_upload_id"], name: "index_sales_logs_on_bulk_upload_id"
     t.index ["created_by_id"], name: "index_sales_logs_on_created_by_id"
     t.index ["old_id"], name: "index_sales_logs_on_old_id", unique: true
diff --git a/spec/services/exports/lettings_log_export_service_spec.rb b/spec/services/exports/lettings_log_export_service_spec.rb
index b37bfe2da..7b9f55b93 100644
--- a/spec/services/exports/lettings_log_export_service_spec.rb
+++ b/spec/services/exports/lettings_log_export_service_spec.rb
@@ -328,6 +328,26 @@ RSpec.describe Exports::LettingsLogExportService do
         expect(LogsExport.last.increment_number).to eq(1)
       end
     end
+
+    context "and a log has been migrated since the previous partial export" do
+      before do
+        FactoryBot.create(:lettings_log, startdate: Time.zone.local(2022, 2, 1), updated_at: Time.zone.local(2022, 4, 27), values_updated_at: Time.zone.local(2022, 4, 29))
+        FactoryBot.create(:lettings_log, startdate: Time.zone.local(2022, 2, 1), updated_at: Time.zone.local(2022, 4, 27), values_updated_at: Time.zone.local(2022, 4, 29))
+        LogsExport.create!(started_at: Time.zone.local(2022, 4, 28), base_number: 1, increment_number: 1)
+      end
+
+      it "generates an XML manifest file with the expected content within the ZIP file" do
+        expected_content = replace_record_number(local_manifest_file.read, 2)
+        expect(storage_service).to receive(:write_file).with(expected_master_manifest_rerun, any_args)
+        expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) do |_, content|
+          entry = Zip::File.open_buffer(content).find_entry(expected_manifest_filename)
+          expect(entry).not_to be_nil
+          expect(entry.get_input_stream.read).to eq(expected_content)
+        end
+
+        export_service.export_xml_lettings_logs
+      end
+    end
   end
 
   context "when exporting a supported housing lettings logs in XML" do
diff --git a/spec/services/imports/lettings_logs_import_service_spec.rb b/spec/services/imports/lettings_logs_import_service_spec.rb
index a95cdac5b..4b18da5f6 100644
--- a/spec/services/imports/lettings_logs_import_service_spec.rb
+++ b/spec/services/imports/lettings_logs_import_service_spec.rb
@@ -159,6 +159,13 @@ RSpec.describe Imports::LettingsLogsImportService do
         end
       end
 
+      it "correctly sets imported at date" do
+        lettings_log_service.send(:create_log, lettings_log_xml)
+
+        lettings_log = LettingsLog.where(old_id: lettings_log_id).first
+        expect(lettings_log&.values_updated_at).to eq(Time.zone.local(2022, 1, 1))
+      end
+
       context "and the void date is after the start date" do
         before { lettings_log_xml.at_xpath("//xmlns:VYEAR").content = 2023 }
 
diff --git a/spec/services/imports/sales_logs_import_service_spec.rb b/spec/services/imports/sales_logs_import_service_spec.rb
index 8f12be392..afe4f6750 100644
--- a/spec/services/imports/sales_logs_import_service_spec.rb
+++ b/spec/services/imports/sales_logs_import_service_spec.rb
@@ -102,9 +102,17 @@ RSpec.describe Imports::SalesLogsImportService do
   end
 
   context "when importing a specific log" do
+    let(:sales_log_id) { "shared_ownership_sales_log" }
     let(:sales_log_file) { open_file(fixture_directory, sales_log_id) }
     let(:sales_log_xml) { Nokogiri::XML(sales_log_file) }
 
+    it "correctly sets values updated at date" do
+      sales_log_service.send(:create_log, sales_log_xml)
+
+      sales_log = SalesLog.where(old_id: sales_log_id).first
+      expect(sales_log&.values_updated_at).to eq(Time.zone.local(2023, 2, 1))
+    end
+
     context "and the organisation legacy ID does not exist" do
       let(:sales_log_id) { "shared_ownership_sales_log" }
 

From 558a237e079243c99991c05997e1aab0375e7a43 Mon Sep 17 00:00:00 2001
From: kosiakkatrina <54268893+kosiakkatrina@users.noreply.github.com>
Date: Fri, 1 Sep 2023 17:17:09 +0200
Subject: [PATCH 04/33] Add extra options for renewal reason (#1876)

---
 ...n_for_leaving_last_settled_home_renewal.rb |  5 ++++-
 .../form/lettings/questions/reason_renewal.rb | 17 ++++++++++++++
 config/forms/2022_2023.json                   | 22 +++++++++++++++++++
 3 files changed, 43 insertions(+), 1 deletion(-)

diff --git a/app/models/form/lettings/pages/reason_for_leaving_last_settled_home_renewal.rb b/app/models/form/lettings/pages/reason_for_leaving_last_settled_home_renewal.rb
index ba025915f..acc8e7045 100644
--- a/app/models/form/lettings/pages/reason_for_leaving_last_settled_home_renewal.rb
+++ b/app/models/form/lettings/pages/reason_for_leaving_last_settled_home_renewal.rb
@@ -6,6 +6,9 @@ class Form::Lettings::Pages::ReasonForLeavingLastSettledHomeRenewal < ::Form::Pa
   end
 
   def questions
-    @questions ||= [Form::Lettings::Questions::ReasonRenewal.new(nil, nil, self)]
+    @questions ||= [
+      Form::Lettings::Questions::ReasonRenewal.new(nil, nil, self),
+      Form::Lettings::Questions::Reasonother.new(nil, nil, self),
+    ]
   end
 end
diff --git a/app/models/form/lettings/questions/reason_renewal.rb b/app/models/form/lettings/questions/reason_renewal.rb
index 86aae9c39..0158eba63 100644
--- a/app/models/form/lettings/questions/reason_renewal.rb
+++ b/app/models/form/lettings/questions/reason_renewal.rb
@@ -9,10 +9,27 @@ class Form::Lettings::Questions::ReasonRenewal < ::Form::Question
     @hint_text = "You told us this letting is a renewal. We have removed some options because of this."
     @answer_options = ANSWER_OPTIONS
     @question_number = 77
+    @conditional_for = {
+      "reasonother" => [
+        20,
+      ],
+    }
   end
 
   ANSWER_OPTIONS = {
     "40" => { "value" => "End of assured shorthold tenancy (no fault)" },
     "42" => { "value" => "End of fixed term tenancy (no fault)" },
+    "20" => {
+      "value" => "Other",
+    },
+    "47" => {
+      "value" => "Tenant prefers not to say",
+    },
+    "divider" => {
+      "value" => true,
+    },
+    "28" => {
+      "value" => "Don’t know",
+    },
   }.freeze
 end
diff --git a/config/forms/2022_2023.json b/config/forms/2022_2023.json
index f8eb4d4c5..db11b0950 100644
--- a/config/forms/2022_2023.json
+++ b/config/forms/2022_2023.json
@@ -6228,8 +6228,30 @@
                     },
                     "42": {
                       "value": "End of fixed term tenancy (no fault)"
+                    },
+                    "20": {
+                      "value": "Other"
+                    },
+                    "47": {
+                      "value":"Tenant prefers not to say"
+                    },
+                    "divider": {
+                      "value": true
+                    },
+                    "28": {
+                      "value": "Don’t know"
                     }
+                  },
+                  "conditional_for": {
+                    "reasonother": [
+                      20
+                    ]
                   }
+                },
+                "reasonother": {
+                  "header": "What is the reason?",
+                  "hint_text": "",
+                  "type": "text"
                 }
               },
               "depends_on": [

From 3b53e3e6963ad24e14a114c5608c4767b819d9e1 Mon Sep 17 00:00:00 2001
From: kosiakkatrina <54268893+kosiakkatrina@users.noreply.github.com>
Date: Fri, 1 Sep 2023 17:19:16 +0200
Subject: [PATCH 05/33] Remove homelessness validation (#1877)

---
 .../validations/household_validations.rb      |  4 ---
 .../lettings/year2022/row_parser.rb           | 12 +++-----
 .../lettings/year2023/row_parser.rb           | 12 +++-----
 config/locales/en.yml                         |  2 --
 .../validations/household_validations_spec.rb | 30 -------------------
 .../lettings/year2022/row_parser_spec.rb      |  8 -----
 .../lettings/year2023/row_parser_spec.rb      |  8 -----
 7 files changed, 8 insertions(+), 68 deletions(-)

diff --git a/app/models/validations/household_validations.rb b/app/models/validations/household_validations.rb
index 07a3fa467..1f4e0998d 100644
--- a/app/models/validations/household_validations.rb
+++ b/app/models/validations/household_validations.rb
@@ -4,10 +4,6 @@ module Validations::HouseholdValidations
   # Validations methods need to be called 'validate_<page_name>' to run on model save
   # or 'validate_' to run on submit as well
   def validate_reasonable_preference(record)
-    if record.is_not_homeless? && record.rp_homeless == 1
-      record.errors.add :reasonable_preference_reason, I18n.t("validations.household.reasonpref.not_homeless")
-      record.errors.add :homeless, I18n.t("validations.household.homeless.reasonpref.not_homeless")
-    end
     if !record.given_reasonable_preference? && [record.rp_homeless, record.rp_insan_unsat, record.rp_medwel, record.rp_hardship, record.rp_dontknow].any? { |a| a == 1 }
       record.errors.add :reasonable_preference_reason, I18n.t("validations.household.reasonable_preference_reason.reason_not_required")
     end
diff --git a/app/services/bulk_upload/lettings/year2022/row_parser.rb b/app/services/bulk_upload/lettings/year2022/row_parser.rb
index a5c598d5a..1f8839989 100644
--- a/app/services/bulk_upload/lettings/year2022/row_parser.rb
+++ b/app/services/bulk_upload/lettings/year2022/row_parser.rb
@@ -689,14 +689,10 @@ private
   end
 
   def validate_reasonable_preference_homeless
-    if field_69 == 1 && homeless == 1 && field_70 == 1
-      errors.add(:field_70, I18n.t("validations.household.reasonpref.not_homeless"))
-    else
-      reason_fields = %i[field_70 field_71 field_72 field_73 field_74]
-      if field_69 == 1 && reason_fields.all? { |field| attributes[field.to_s].blank? }
-        reason_fields.each do |field|
-          errors.add(field, I18n.t("validations.not_answered", question: "reason for reasonable preference"))
-        end
+    reason_fields = %i[field_70 field_71 field_72 field_73 field_74]
+    if field_69 == 1 && reason_fields.all? { |field| attributes[field.to_s].blank? }
+      reason_fields.each do |field|
+        errors.add(field, I18n.t("validations.not_answered", question: "reason for reasonable preference"))
       end
     end
   end
diff --git a/app/services/bulk_upload/lettings/year2023/row_parser.rb b/app/services/bulk_upload/lettings/year2023/row_parser.rb
index 4abfdac17..9839846f8 100644
--- a/app/services/bulk_upload/lettings/year2023/row_parser.rb
+++ b/app/services/bulk_upload/lettings/year2023/row_parser.rb
@@ -622,14 +622,10 @@ private
   end
 
   def validate_reasonable_preference_homeless
-    if field_110 == 1 && field_105 == 1 && field_111 == 1
-      errors.add(:field_111, I18n.t("validations.household.reasonpref.not_homeless"))
-    else
-      reason_fields = %i[field_111 field_112 field_113 field_114 field_115]
-      if field_110 == 1 && reason_fields.all? { |field| attributes[field.to_s].blank? }
-        reason_fields.each do |field|
-          errors.add(field, I18n.t("validations.not_answered", question: "reason for reasonable preference"))
-        end
+    reason_fields = %i[field_111 field_112 field_113 field_114 field_115]
+    if field_110 == 1 && reason_fields.all? { |field| attributes[field.to_s].blank? }
+      reason_fields.each do |field|
+        errors.add(field, I18n.t("validations.not_answered", question: "reason for reasonable preference"))
       end
     end
   end
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 4307158e2..5d0f435f9 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -441,8 +441,6 @@ en:
       mortgage: "Mortgage value cannot be £0 if a mortgage was used for the purchase of this property"
 
     household:
-      reasonpref:
-        not_homeless: "Answer cannot be ‘homeless or about to lose their home’ as the tenant was not homeless immediately prior to this letting"
       reasonable_preference_reason:
         reason_required: "Enter a reason if you've answered 'yes' to reasonable preference"
         reason_not_required: "Do not enter a reason if you've answered 'no' to reasonable preference"
diff --git a/spec/models/validations/household_validations_spec.rb b/spec/models/validations/household_validations_spec.rb
index f44c5d255..b3e20779d 100644
--- a/spec/models/validations/household_validations_spec.rb
+++ b/spec/models/validations/household_validations_spec.rb
@@ -12,36 +12,6 @@ RSpec.describe Validations::HouseholdValidations do
   end
 
   describe "reasonable preference validations" do
-    context "when reasonable preference is homeless" do
-      context "when the tenant was not previously homeless" do
-        it "adds an error" do
-          record.homeless = 1
-          record.rp_homeless = 1
-          household_validator.validate_reasonable_preference(record)
-          expect(record.errors["reasonable_preference_reason"])
-            .to include(match I18n.t("validations.household.reasonpref.not_homeless"))
-          expect(record.errors["homeless"])
-            .to include(match I18n.t("validations.household.homeless.reasonpref.not_homeless"))
-        end
-      end
-
-      context "when reasonable preference is given" do
-        context "when the tenant was previously homeless" do
-          it "does not add an error" do
-            record.homeless = 1
-            record.reasonpref = 1
-            household_validator.validate_reasonable_preference(record)
-            expect(record.errors["reasonpref"]).to be_empty
-            expect(record.errors["homeless"]).to be_empty
-            record.homeless = 0
-            household_validator.validate_reasonable_preference(record)
-            expect(record.errors["reasonpref"]).to be_empty
-            expect(record.errors["homeless"]).to be_empty
-          end
-        end
-      end
-    end
-
     context "when reasonable preference is not given" do
       it "validates that no reason is needed" do
         record.reasonpref = 1
diff --git a/spec/services/bulk_upload/lettings/year2022/row_parser_spec.rb b/spec/services/bulk_upload/lettings/year2022/row_parser_spec.rb
index 7380ca550..0a9f65d32 100644
--- a/spec/services/bulk_upload/lettings/year2022/row_parser_spec.rb
+++ b/spec/services/bulk_upload/lettings/year2022/row_parser_spec.rb
@@ -711,14 +711,6 @@ RSpec.describe BulkUpload::Lettings::Year2022::RowParser do
     end
 
     describe "#field_68 - 74" do
-      context "when not homeless but reasonable preference for homelessness" do
-        let(:attributes) { { bulk_upload:, field_68: "1", field_69: "1", field_70: "1" } }
-
-        it "is not permitted" do
-          expect(parser.errors[:field_70]).to be_present
-        end
-      end
-
       context "when there is a reasonable preference but none is given" do
         let(:attributes) { { bulk_upload:, field_69: "1", field_70: nil, field_71: nil, field_72: nil, field_73: nil, field_74: nil } }
 
diff --git a/spec/services/bulk_upload/lettings/year2023/row_parser_spec.rb b/spec/services/bulk_upload/lettings/year2023/row_parser_spec.rb
index c2f816ab6..a3b9152c5 100644
--- a/spec/services/bulk_upload/lettings/year2023/row_parser_spec.rb
+++ b/spec/services/bulk_upload/lettings/year2023/row_parser_spec.rb
@@ -803,14 +803,6 @@ RSpec.describe BulkUpload::Lettings::Year2023::RowParser do
     end
 
     describe "#field_105, field_110 - 15" do
-      context "when not homeless but reasonable preference for homelessness" do
-        let(:attributes) { { bulk_upload:, field_105: "1", field_110: "1", field_111: "1" } }
-
-        it "is not permitted" do
-          expect(parser.errors[:field_111]).to be_present
-        end
-      end
-
       context "when there is a reasonable preference but none is given" do
         let(:attributes) { { bulk_upload:, field_110: "1", field_111: nil, field_112: nil, field_113: nil, field_114: nil, field_115: nil } }
 

From 9ae32ac2664594f5b7e1ea02e2e7d1e5ddba7c50 Mon Sep 17 00:00:00 2001
From: kosiakkatrina <54268893+kosiakkatrina@users.noreply.github.com>
Date: Fri, 1 Sep 2023 17:23:14 +0200
Subject: [PATCH 06/33] remove
 validate_years_living_in_property_before_purchase (#1878)

---
 .../sales/sale_information_validations.rb     | 13 -----
 .../sale_information_validations_spec.rb      | 54 -------------------
 2 files changed, 67 deletions(-)

diff --git a/app/models/validations/sales/sale_information_validations.rb b/app/models/validations/sales/sale_information_validations.rb
index b802fa08f..3f4d06a30 100644
--- a/app/models/validations/sales/sale_information_validations.rb
+++ b/app/models/validations/sales/sale_information_validations.rb
@@ -11,19 +11,6 @@ module Validations::Sales::SaleInformationValidations
     end
   end
 
-  def validate_years_living_in_property_before_purchase(record)
-    return unless record.proplen&.nonzero?
-
-    case record.type
-    when 18
-      record.errors.add :type, I18n.t("validations.sale_information.proplen.social_homebuy")
-      record.errors.add :proplen, I18n.t("validations.sale_information.proplen.social_homebuy")
-    when 28, 29
-      record.errors.add :type, I18n.t("validations.sale_information.proplen.rent_to_buy")
-      record.errors.add :proplen, I18n.t("validations.sale_information.proplen.rent_to_buy")
-    end
-  end
-
   def validate_exchange_date(record)
     return unless record.exdate && record.saledate
 
diff --git a/spec/models/validations/sales/sale_information_validations_spec.rb b/spec/models/validations/sales/sale_information_validations_spec.rb
index cccb19d40..dbd4c1e49 100644
--- a/spec/models/validations/sales/sale_information_validations_spec.rb
+++ b/spec/models/validations/sales/sale_information_validations_spec.rb
@@ -171,60 +171,6 @@ RSpec.describe Validations::Sales::SaleInformationValidations do
     end
   end
 
-  describe "#validate_years_living_in_property_before_purchase" do
-    context "when proplen blank" do
-      let(:record) { build(:sales_log, proplen: nil) }
-
-      it "does not add an error" do
-        sale_information_validator.validate_years_living_in_property_before_purchase(record)
-
-        expect(record.errors).not_to be_present
-      end
-    end
-
-    context "when type blank" do
-      let(:record) { build(:sales_log, type: nil) }
-
-      it "does not add an error" do
-        sale_information_validator.validate_years_living_in_property_before_purchase(record)
-
-        expect(record.errors).not_to be_present
-      end
-    end
-
-    context "when proplen 0" do
-      let(:record) { build(:sales_log, proplen: 0) }
-
-      it "does not add an error" do
-        sale_information_validator.validate_years_living_in_property_before_purchase(record)
-
-        expect(record.errors).not_to be_present
-      end
-    end
-
-    context "when type Rent to Buy and proplen > 0" do
-      let(:record) { build(:sales_log, proplen: 1, type: 28) }
-
-      it "adds an error" do
-        sale_information_validator.validate_years_living_in_property_before_purchase(record)
-
-        expect(record.errors[:type]).to include(I18n.t("validations.sale_information.proplen.rent_to_buy"))
-        expect(record.errors[:proplen]).to include(I18n.t("validations.sale_information.proplen.rent_to_buy"))
-      end
-    end
-
-    context "when type Social HomeBuy and proplen > 0" do
-      let(:record) { build(:sales_log, proplen: 1, type: 18) }
-
-      it "adds an error" do
-        sale_information_validator.validate_years_living_in_property_before_purchase(record)
-
-        expect(record.errors[:type]).to include(I18n.t("validations.sale_information.proplen.social_homebuy"))
-        expect(record.errors[:proplen]).to include(I18n.t("validations.sale_information.proplen.social_homebuy"))
-      end
-    end
-  end
-
   describe "#validate_discounted_ownership_value" do
     context "when sale is on or after 24/25 collection window" do
       context "when grant is routed to" do

From 292091729216c4c4414907467f173b386c051e85 Mon Sep 17 00:00:00 2001
From: kosiakkatrina <54268893+kosiakkatrina@users.noreply.github.com>
Date: Mon, 4 Sep 2023 11:05:57 +0200
Subject: [PATCH 07/33] Create an unassigned user per organisation (#1885)

---
 .../imports/lettings_logs_import_service.rb         |  4 ++--
 app/services/imports/sales_logs_import_service.rb   |  6 +++---
 .../imports/lettings_logs_import_service_spec.rb    | 13 +++++++++++++
 .../imports/sales_logs_import_service_spec.rb       | 13 +++++++++++++
 4 files changed, 31 insertions(+), 5 deletions(-)

diff --git a/app/services/imports/lettings_logs_import_service.rb b/app/services/imports/lettings_logs_import_service.rb
index b7e2673d0..138126cbb 100644
--- a/app/services/imports/lettings_logs_import_service.rb
+++ b/app/services/imports/lettings_logs_import_service.rb
@@ -253,8 +253,8 @@ module Imports
         user = LegacyUser.find_by(old_user_id: owner_id)&.user
         if user.blank?
           @logger.error("Lettings log '#{attributes['old_id']}' belongs to legacy user with owner-user-id: '#{owner_id}' which cannot be found. Assigning log to 'Unassigned' user.")
-          if User.find_by(name: "Unassigned")
-            user = User.find_by(name: "Unassigned")
+          if User.find_by(name: "Unassigned", organisation_id: attributes["managing_organisation_id"])
+            user = User.find_by(name: "Unassigned", organisation_id: attributes["managing_organisation_id"])
           else
             user = User.new(
               name: "Unassigned",
diff --git a/app/services/imports/sales_logs_import_service.rb b/app/services/imports/sales_logs_import_service.rb
index 8a8156480..ecee55c22 100644
--- a/app/services/imports/sales_logs_import_service.rb
+++ b/app/services/imports/sales_logs_import_service.rb
@@ -181,12 +181,12 @@ module Imports
 
         if user.blank?
           @logger.error("Sales log '#{attributes['old_id']}' belongs to legacy user with owner-user-id: '#{owner_id}' which cannot be found. Assigning log to 'Unassigned' user.")
-          if User.find_by(name: "Unassigned")
-            user = User.find_by(name: "Unassigned")
+          if User.find_by(name: "Unassigned", organisation_id: attributes["owning_organisation_id"])
+            user = User.find_by(name: "Unassigned", organisation_id: attributes["owning_organisation_id"])
           else
             user = User.new(
               name: "Unassigned",
-              organisation_id: attributes["managing_organisation_id"],
+              organisation_id: attributes["owning_organisation_id"],
               is_dpo: false,
               encrypted_password: SecureRandom.hex(10),
               email: SecureRandom.uuid,
diff --git a/spec/services/imports/lettings_logs_import_service_spec.rb b/spec/services/imports/lettings_logs_import_service_spec.rb
index 4b18da5f6..1025fcd3b 100644
--- a/spec/services/imports/lettings_logs_import_service_spec.rb
+++ b/spec/services/imports/lettings_logs_import_service_spec.rb
@@ -157,6 +157,19 @@ RSpec.describe Imports::LettingsLogsImportService do
           second_lettings_log = LettingsLog.where(old_id: "fake_id").first
           expect(lettings_log&.created_by).to eq(second_lettings_log&.created_by)
         end
+
+        context "when unassigned user exist for a different organisation" do
+          let!(:other_unassigned_user) { create(:user, name: "Unassigned") }
+
+          it "creates a new unassigned user for current organisation" do
+            expect(logger).to receive(:error).with("Lettings log '0ead17cb-1668-442d-898c-0d52879ff592' belongs to legacy user with owner-user-id: 'fake_id' which cannot be found. Assigning log to 'Unassigned' user.")
+            lettings_log_service.send(:create_log, lettings_log_xml)
+
+            lettings_log = LettingsLog.where(old_id: lettings_log_id).first
+            expect(lettings_log&.created_by&.name).to eq("Unassigned")
+            expect(lettings_log&.created_by).not_to eq(other_unassigned_user)
+          end
+        end
       end
 
       it "correctly sets imported at date" do
diff --git a/spec/services/imports/sales_logs_import_service_spec.rb b/spec/services/imports/sales_logs_import_service_spec.rb
index afe4f6750..bbabc5c1a 100644
--- a/spec/services/imports/sales_logs_import_service_spec.rb
+++ b/spec/services/imports/sales_logs_import_service_spec.rb
@@ -148,6 +148,19 @@ RSpec.describe Imports::SalesLogsImportService do
         second_sales_log = SalesLog.where(old_id: "fake_id").first
         expect(sales_log&.created_by).to eq(second_sales_log&.created_by)
       end
+
+      context "when unassigned user exist for a different organisation" do
+        let!(:other_unassigned_user) { create(:user, name: "Unassigned") }
+
+        it "creates a new unassigned user for current organisation" do
+          expect(logger).to receive(:error).with("Sales log 'shared_ownership_sales_log' belongs to legacy user with owner-user-id: 'fake_id' which cannot be found. Assigning log to 'Unassigned' user.")
+          sales_log_service.send(:create_log, sales_log_xml)
+
+          sales_log = SalesLog.where(old_id: sales_log_id).first
+          expect(sales_log&.created_by&.name).to eq("Unassigned")
+          expect(sales_log&.created_by).not_to eq(other_unassigned_user)
+        end
+      end
     end
 
     context "and the log startdate is before 22/23 collection period" do

From aed7926340ed5fdf41fcf156f232d83a05efe8bc Mon Sep 17 00:00:00 2001
From: natdeanlewissoftwire
 <94526761+natdeanlewissoftwire@users.noreply.github.com>
Date: Mon, 4 Sep 2023 11:07:34 +0100
Subject: [PATCH 08/33] feat: set timeout to 3 mins when pushing to staging
 (#1886)

* feat: set timeout to 10 mins when pushing to staging

* feat: set timeout to 3 mins when pushing to staging (the max)
---
 .github/workflows/staging_pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/staging_pipeline.yml b/.github/workflows/staging_pipeline.yml
index 06f22632b..338d09cec 100644
--- a/.github/workflows/staging_pipeline.yml
+++ b/.github/workflows/staging_pipeline.yml
@@ -229,4 +229,4 @@ jobs:
           cf set-env $APP_NAME S3_CONFIG $S3_CONFIG
           cf set-env $APP_NAME CSV_DOWNLOAD_PAAS_INSTANCE $CSV_DOWNLOAD_PAAS_INSTANCE
           cf set-env $APP_NAME SENTRY_DSN $SENTRY_DSN
-          cf push $APP_NAME --strategy rolling
+          cf push $APP_NAME --strategy rolling -t 180

From d43e9546aec1af66ae8059656db49c3e68c36da3 Mon Sep 17 00:00:00 2001
From: kosiakkatrina <54268893+kosiakkatrina@users.noreply.github.com>
Date: Mon, 4 Sep 2023 12:55:53 +0200
Subject: [PATCH 09/33] Remove rolling strategy on staging (#1887)

---
 .github/workflows/staging_pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/staging_pipeline.yml b/.github/workflows/staging_pipeline.yml
index 338d09cec..f92ba0118 100644
--- a/.github/workflows/staging_pipeline.yml
+++ b/.github/workflows/staging_pipeline.yml
@@ -229,4 +229,4 @@ jobs:
           cf set-env $APP_NAME S3_CONFIG $S3_CONFIG
           cf set-env $APP_NAME CSV_DOWNLOAD_PAAS_INSTANCE $CSV_DOWNLOAD_PAAS_INSTANCE
           cf set-env $APP_NAME SENTRY_DSN $SENTRY_DSN
-          cf push $APP_NAME --strategy rolling -t 180
+          cf push $APP_NAME -t 180

From 0d61ffb804c86537c776c68ec90023a9d52a0bc4 Mon Sep 17 00:00:00 2001
From: kosiakkatrina <54268893+kosiakkatrina@users.noreply.github.com>
Date: Mon, 4 Sep 2023 14:27:18 +0200
Subject: [PATCH 10/33] Add rolling strategy back (#1888)

---
 .github/workflows/staging_pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/staging_pipeline.yml b/.github/workflows/staging_pipeline.yml
index f92ba0118..338d09cec 100644
--- a/.github/workflows/staging_pipeline.yml
+++ b/.github/workflows/staging_pipeline.yml
@@ -229,4 +229,4 @@ jobs:
           cf set-env $APP_NAME S3_CONFIG $S3_CONFIG
           cf set-env $APP_NAME CSV_DOWNLOAD_PAAS_INSTANCE $CSV_DOWNLOAD_PAAS_INSTANCE
           cf set-env $APP_NAME SENTRY_DSN $SENTRY_DSN
-          cf push $APP_NAME -t 180
+          cf push $APP_NAME --strategy rolling -t 180

From c4c93dd4cdd3cdf57c7a7d4daaac39afb522c173 Mon Sep 17 00:00:00 2001
From: kosiakkatrina <54268893+kosiakkatrina@users.noreply.github.com>
Date: Wed, 6 Sep 2023 14:13:46 +0100
Subject: [PATCH 11/33] CLDC-2713 Add a rake task to reimport addresses from
 csv (#1890)

* Add a rake task to reimport addresses from csv

* Add empty line

* Update postcode known

* Update values updated at column
---
 lib/tasks/import_address_from_csv.rake        |  42 +++++
 spec/fixtures/files/addresses_reimport.csv    |   6 +
 .../tasks/correct_address_from_csv_spec.rb    | 154 ++++++++++++++++++
 3 files changed, 202 insertions(+)
 create mode 100644 lib/tasks/import_address_from_csv.rake
 create mode 100644 spec/fixtures/files/addresses_reimport.csv
 create mode 100644 spec/lib/tasks/correct_address_from_csv_spec.rb

diff --git a/lib/tasks/import_address_from_csv.rake b/lib/tasks/import_address_from_csv.rake
new file mode 100644
index 000000000..e910f3a77
--- /dev/null
+++ b/lib/tasks/import_address_from_csv.rake
@@ -0,0 +1,42 @@
+namespace :data_import do
+  desc "Import address data from a csv file"
+  task :import_address_from_csv, %i[file_name] => :environment do |_task, args|
+    file_name = args[:file_name]
+
+    raise "Usage: rake data_import:import_address_from_csv['csv_file_name']" if file_name.blank?
+
+    s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
+    addresses_csv = CSV.parse(s3_service.get_file_io(file_name), headers: true)
+
+    addresses_csv.each do |row|
+      lettings_log_id = row[0]
+
+      if lettings_log_id.blank?
+        Rails.logger.info("Lettings log ID not provided for address: #{[row[1], row[2], row[3], row[4]].join(', ')}")
+        next
+      end
+
+      lettings_log = LettingsLog.find_by(id: lettings_log_id)
+      if lettings_log.blank?
+        Rails.logger.info("Could not find a lettings log with id #{lettings_log_id}")
+        next
+      end
+
+      lettings_log.uprn_known = 0
+      lettings_log.uprn = nil
+      lettings_log.uprn_confirmed = nil
+      lettings_log.address_line1 = row[1]
+      lettings_log.address_line2 = row[2]
+      lettings_log.town_or_city = row[3]
+      lettings_log.postcode_full = row[4]
+      lettings_log.postcode_known = lettings_log.postcode_full.present? ? 1 : nil
+      lettings_log.county = nil
+      lettings_log.is_la_inferred = nil
+      lettings_log.la = nil
+      lettings_log.values_updated_at = Time.zone.now
+
+      lettings_log.save!
+      Rails.logger.info("Updated lettings log #{lettings_log_id}, with address: #{[lettings_log.address_line1, lettings_log.address_line2, lettings_log.town_or_city, lettings_log.postcode_full].join(', ')}")
+    end
+  end
+end
diff --git a/spec/fixtures/files/addresses_reimport.csv b/spec/fixtures/files/addresses_reimport.csv
new file mode 100644
index 000000000..998d2fbfa
--- /dev/null
+++ b/spec/fixtures/files/addresses_reimport.csv
@@ -0,0 +1,6 @@
+lettings_log_id,address_line1,address_line2,town_or_city,postcode_full
+{id},address 1,address 2,town,B1 1BB
+{id2},address 3,address 4,city,B1 1BB
+{id3},,,,C1 1CC
+,address 5,address 6,city,D1 1DD
+fake_id,address 7,address 8,city,D1 1DD
diff --git a/spec/lib/tasks/correct_address_from_csv_spec.rb b/spec/lib/tasks/correct_address_from_csv_spec.rb
new file mode 100644
index 000000000..0a3d1c563
--- /dev/null
+++ b/spec/lib/tasks/correct_address_from_csv_spec.rb
@@ -0,0 +1,154 @@
+require "rails_helper"
+require "rake"
+
+RSpec.describe "data_import" do
+  def replace_entity_ids(lettings_log, second_lettings_log, third_lettings_log, export_template)
+    export_template.sub!(/\{id\}/, lettings_log.id.to_s)
+    export_template.sub!(/\{id2\}/, second_lettings_log.id.to_s)
+    export_template.sub!(/\{id3\}/, third_lettings_log.id.to_s)
+  end
+
+  describe ":import_address_from_csv", type: :task do
+    subject(:task) { Rake::Task["data_import:import_address_from_csv"] }
+
+    let(:instance_name) { "paas_import_instance" }
+    let(:storage_service) { instance_double(Storage::S3Service) }
+    let(:env_config_service) { instance_double(Configuration::EnvConfigurationService) }
+    let(:paas_config_service) { instance_double(Configuration::PaasConfigurationService) }
+
+    before do
+      allow(Storage::S3Service).to receive(:new).and_return(storage_service)
+      allow(Configuration::EnvConfigurationService).to receive(:new).and_return(env_config_service)
+      allow(Configuration::PaasConfigurationService).to receive(:new).and_return(paas_config_service)
+      allow(ENV).to receive(:[])
+      allow(ENV).to receive(:[]).with("IMPORT_PAAS_INSTANCE").and_return(instance_name)
+      allow(ENV).to receive(:[]).with("VCAP_SERVICES").and_return("dummy")
+
+      Rake.application.rake_require("tasks/import_address_from_csv")
+      Rake::Task.define_task(:environment)
+      task.reenable
+
+      WebMock.stub_request(:get, /api.postcodes.io/)
+        .to_return(status: 200, body: "{\"status\":404,\"error\":\"Postcode not found\"}", headers: {})
+      WebMock.stub_request(:get, /api.postcodes.io\/postcodes\/B11BB/)
+        .to_return(status: 200, body: '{"status":200,"result":{"admin_district":"Westminster","codes":{"admin_district":"E08000035"}}}', headers: {})
+    end
+
+    context "when the rake task is run" do
+      let(:addresses_csv_path) { "addresses_reimport_123.csv" }
+      let(:wrong_file_path) { "/test/no_csv_here.csv" }
+      let!(:lettings_log) do
+        create(:lettings_log,
+               uprn_known: nil,
+               uprn: nil,
+               uprn_confirmed: nil,
+               address_line1: nil,
+               address_line2: nil,
+               town_or_city: nil,
+               county: nil,
+               postcode_known: 1,
+               postcode_full: "A1 1AA",
+               la: "E06000064",
+               is_la_inferred: true)
+      end
+
+      let!(:second_lettings_log) do
+        create(:lettings_log,
+               uprn_known: 1,
+               uprn: "1",
+               uprn_confirmed: nil,
+               address_line1: "wrong address line1",
+               address_line2: "wrong address 2",
+               town_or_city: "wrong town",
+               county: "wrong city",
+               postcode_known: 1,
+               postcode_full: "A1 1AA",
+               la: "E06000064",
+               is_la_inferred: true)
+      end
+
+      let!(:third_lettings_log) do
+        create(:lettings_log,
+               uprn_known: 1,
+               uprn: "1",
+               uprn_confirmed: nil,
+               address_line1: "wrong address line1",
+               address_line2: "wrong address 2",
+               town_or_city: "wrong town",
+               county: "wrong city",
+               postcode_known: 1,
+               postcode_full: "A1 1AA",
+               la: "E06000064",
+               is_la_inferred: true)
+      end
+
+      before do
+        allow(storage_service).to receive(:get_file_io)
+        .with("addresses_reimport_123.csv")
+        .and_return(replace_entity_ids(lettings_log, second_lettings_log, third_lettings_log, File.open("./spec/fixtures/files/addresses_reimport.csv").read))
+      end
+
+      it "updates the log address when old address was not given" do
+        task.invoke(addresses_csv_path)
+        lettings_log.reload
+        expect(lettings_log.uprn_known).to eq(0)
+        expect(lettings_log.uprn).to eq(nil)
+        expect(lettings_log.uprn_confirmed).to eq(nil)
+        expect(lettings_log.address_line1).to eq("address 1")
+        expect(lettings_log.address_line2).to eq("address 2")
+        expect(lettings_log.town_or_city).to eq("town")
+        expect(lettings_log.county).to eq(nil)
+        expect(lettings_log.postcode_known).to eq(1)
+        expect(lettings_log.postcode_full).to eq("B1 1BB")
+        expect(lettings_log.la).to eq("E08000035")
+        expect(lettings_log.is_la_inferred).to eq(true)
+      end
+
+      it "updates the log address when old address was given" do
+        task.invoke(addresses_csv_path)
+        second_lettings_log.reload
+        expect(second_lettings_log.uprn_known).to eq(0)
+        expect(second_lettings_log.uprn).to eq(nil)
+        expect(second_lettings_log.uprn_confirmed).to eq(nil)
+        expect(second_lettings_log.address_line1).to eq("address 3")
+        expect(second_lettings_log.address_line2).to eq("address 4")
+        expect(second_lettings_log.town_or_city).to eq("city")
+        expect(second_lettings_log.county).to eq(nil)
+        expect(second_lettings_log.postcode_known).to eq(1)
+        expect(second_lettings_log.postcode_full).to eq("B1 1BB")
+        expect(second_lettings_log.la).to eq("E08000035")
+        expect(second_lettings_log.is_la_inferred).to eq(true)
+      end
+
+      it "updates the log address when new address is not given" do
+        task.invoke(addresses_csv_path)
+        third_lettings_log.reload
+        expect(third_lettings_log.uprn_known).to eq(0)
+        expect(third_lettings_log.uprn).to eq(nil)
+        expect(third_lettings_log.uprn_confirmed).to eq(nil)
+        expect(third_lettings_log.address_line1).to eq(nil)
+        expect(third_lettings_log.address_line2).to eq(nil)
+        expect(third_lettings_log.town_or_city).to eq(nil)
+        expect(third_lettings_log.county).to eq(nil)
+        expect(third_lettings_log.postcode_known).to eq(1)
+        expect(third_lettings_log.postcode_full).to eq("C11CC")
+        expect(third_lettings_log.la).to eq(nil)
+        expect(third_lettings_log.is_la_inferred).to eq(false)
+      end
+
+      it "logs the progress of the update" do
+        expect(Rails.logger).to receive(:info).with("Updated lettings log #{lettings_log.id}, with address: address 1, address 2, town, B1 1BB")
+        expect(Rails.logger).to receive(:info).with("Updated lettings log #{second_lettings_log.id}, with address: address 3, address 4, city, B1 1BB")
+        expect(Rails.logger).to receive(:info).with("Updated lettings log #{third_lettings_log.id}, with address: , , , C11CC")
+        expect(Rails.logger).to receive(:info).with("Lettings log ID not provided for address: address 5, address 6, city, D1 1DD")
+        expect(Rails.logger).to receive(:info).with("Could not find a lettings log with id fake_id")
+
+        task.invoke(addresses_csv_path)
+      end
+
+      it "raises an error when no path is given" do
+        expect { task.invoke(nil) }.to raise_error(RuntimeError, "Usage: rake data_import:import_address_from_csv['csv_file_name']")
+      end
+    end
+  end
+end

From 719321945b3e9e0eef7643a991a0ac079545374f Mon Sep 17 00:00:00 2001
From: kosiakkatrina <54268893+kosiakkatrina@users.noreply.github.com>
Date: Thu, 7 Sep 2023 11:48:41 +0100
Subject: [PATCH 12/33] Set unassigned user if legacy user belongs to a
 different organisation (#1894)

* Set user to unassigned in lettings logs if the legacy user exists but belongs to a different organisation

* Set user to unassigned in sales logs if the legacy user exists but belongs to a different organisation

* Lint
---
 .../imports/lettings_logs_import_service.rb   |  8 +++-
 .../imports/sales_logs_import_service.rb      |  8 +++-
 .../lettings_logs_import_service_spec.rb      | 40 ++++++++++++++++++
 .../imports/sales_logs_import_service_spec.rb | 42 +++++++++++++++++++
 4 files changed, 94 insertions(+), 4 deletions(-)

diff --git a/app/services/imports/lettings_logs_import_service.rb b/app/services/imports/lettings_logs_import_service.rb
index 138126cbb..a53048d83 100644
--- a/app/services/imports/lettings_logs_import_service.rb
+++ b/app/services/imports/lettings_logs_import_service.rb
@@ -251,8 +251,12 @@ module Imports
       owner_id = meta_field_value(xml_doc, "owner-user-id").strip
       if owner_id.present?
         user = LegacyUser.find_by(old_user_id: owner_id)&.user
-        if user.blank?
-          @logger.error("Lettings log '#{attributes['old_id']}' belongs to legacy user with owner-user-id: '#{owner_id}' which cannot be found. Assigning log to 'Unassigned' user.")
+        if user.blank? || (user.organisation_id != attributes["managing_organisation_id"] && user.organisation_id != attributes["owning_organisation_id"])
+          if user.blank?
+            @logger.error("Lettings log '#{attributes['old_id']}' belongs to legacy user with owner-user-id: '#{owner_id}' which cannot be found. Assigning log to 'Unassigned' user.")
+          else
+            @logger.error("Lettings log '#{attributes['old_id']}' belongs to legacy user with owner-user-id: '#{owner_id}' which belongs to a different organisation. Assigning log to 'Unassigned' user.")
+          end
           if User.find_by(name: "Unassigned", organisation_id: attributes["managing_organisation_id"])
             user = User.find_by(name: "Unassigned", organisation_id: attributes["managing_organisation_id"])
           else
diff --git a/app/services/imports/sales_logs_import_service.rb b/app/services/imports/sales_logs_import_service.rb
index ecee55c22..157e797b4 100644
--- a/app/services/imports/sales_logs_import_service.rb
+++ b/app/services/imports/sales_logs_import_service.rb
@@ -179,8 +179,12 @@ module Imports
       if owner_id.present?
         user = LegacyUser.find_by(old_user_id: owner_id)&.user
 
-        if user.blank?
-          @logger.error("Sales log '#{attributes['old_id']}' belongs to legacy user with owner-user-id: '#{owner_id}' which cannot be found. Assigning log to 'Unassigned' user.")
+        if user.blank? || user.organisation_id != attributes["owning_organisation_id"]
+          if user.blank?
+            @logger.error("Sales log '#{attributes['old_id']}' belongs to legacy user with owner-user-id: '#{owner_id}' which cannot be found. Assigning log to 'Unassigned' user.")
+          else
+            @logger.error("Sales log '#{attributes['old_id']}' belongs to legacy user with owner-user-id: '#{owner_id}' which belongs to a different organisation. Assigning log to 'Unassigned' user.")
+          end
           if User.find_by(name: "Unassigned", organisation_id: attributes["owning_organisation_id"])
             user = User.find_by(name: "Unassigned", organisation_id: attributes["owning_organisation_id"])
           else
diff --git a/spec/services/imports/lettings_logs_import_service_spec.rb b/spec/services/imports/lettings_logs_import_service_spec.rb
index 1025fcd3b..7dc3d578d 100644
--- a/spec/services/imports/lettings_logs_import_service_spec.rb
+++ b/spec/services/imports/lettings_logs_import_service_spec.rb
@@ -172,6 +172,46 @@ RSpec.describe Imports::LettingsLogsImportService do
         end
       end
 
+      context "and the user exists on a different organisation" do
+        before do
+          create(:legacy_user, old_user_id: "fake_id")
+          lettings_log_xml.at_xpath("//meta:owner-user-id").content = "fake_id"
+        end
+
+        it "creates a new unassigned user" do
+          expect(logger).to receive(:error).with("Lettings log '0ead17cb-1668-442d-898c-0d52879ff592' belongs to legacy user with owner-user-id: 'fake_id' which belongs to a different organisation. Assigning log to 'Unassigned' user.")
+          lettings_log_service.send(:create_log, lettings_log_xml)
+
+          lettings_log = LettingsLog.where(old_id: lettings_log_id).first
+          expect(lettings_log&.created_by&.name).to eq("Unassigned")
+        end
+
+        it "only creates one unassigned user" do
+          expect(logger).to receive(:error).with("Lettings log '0ead17cb-1668-442d-898c-0d52879ff592' belongs to legacy user with owner-user-id: 'fake_id' which belongs to a different organisation. Assigning log to 'Unassigned' user.")
+          expect(logger).to receive(:error).with("Lettings log 'fake_id' belongs to legacy user with owner-user-id: 'fake_id' which belongs to a different organisation. Assigning log to 'Unassigned' user.")
+          lettings_log_service.send(:create_log, lettings_log_xml)
+          lettings_log_xml.at_xpath("//meta:document-id").content = "fake_id"
+          lettings_log_service.send(:create_log, lettings_log_xml)
+
+          lettings_log = LettingsLog.where(old_id: lettings_log_id).first
+          second_lettings_log = LettingsLog.where(old_id: "fake_id").first
+          expect(lettings_log&.created_by).to eq(second_lettings_log&.created_by)
+        end
+
+        context "when unassigned user exist for a different organisation" do
+          let!(:other_unassigned_user) { create(:user, name: "Unassigned") }
+
+          it "creates a new unassigned user for current organisation" do
+            expect(logger).to receive(:error).with("Lettings log '0ead17cb-1668-442d-898c-0d52879ff592' belongs to legacy user with owner-user-id: 'fake_id' which belongs to a different organisation. Assigning log to 'Unassigned' user.")
+            lettings_log_service.send(:create_log, lettings_log_xml)
+
+            lettings_log = LettingsLog.where(old_id: lettings_log_id).first
+            expect(lettings_log&.created_by&.name).to eq("Unassigned")
+            expect(lettings_log&.created_by).not_to eq(other_unassigned_user)
+          end
+        end
+      end
+
       it "correctly sets imported at date" do
         lettings_log_service.send(:create_log, lettings_log_xml)
 
diff --git a/spec/services/imports/sales_logs_import_service_spec.rb b/spec/services/imports/sales_logs_import_service_spec.rb
index bbabc5c1a..df279e5d5 100644
--- a/spec/services/imports/sales_logs_import_service_spec.rb
+++ b/spec/services/imports/sales_logs_import_service_spec.rb
@@ -163,6 +163,48 @@ RSpec.describe Imports::SalesLogsImportService do
       end
     end
 
+    context "and the user exists on a different organisation" do
+      let(:sales_log_id) { "shared_ownership_sales_log" }
+
+      before do
+        create(:legacy_user, old_user_id: "fake_id")
+        sales_log_xml.at_xpath("//meta:owner-user-id").content = "fake_id"
+      end
+
+      it "creates a new unassigned user" do
+        expect(logger).to receive(:error).with("Sales log 'shared_ownership_sales_log' belongs to legacy user with owner-user-id: 'fake_id' which belongs to a different organisation. Assigning log to 'Unassigned' user.")
+        sales_log_service.send(:create_log, sales_log_xml)
+
+        sales_log = SalesLog.where(old_id: sales_log_id).first
+        expect(sales_log&.created_by&.name).to eq("Unassigned")
+      end
+
+      it "only creates one unassigned user" do
+        expect(logger).to receive(:error).with("Sales log 'shared_ownership_sales_log' belongs to legacy user with owner-user-id: 'fake_id' which belongs to a different organisation. Assigning log to 'Unassigned' user.")
+        expect(logger).to receive(:error).with("Sales log 'fake_id' belongs to legacy user with owner-user-id: 'fake_id' which belongs to a different organisation. Assigning log to 'Unassigned' user.")
+        sales_log_service.send(:create_log, sales_log_xml)
+        sales_log_xml.at_xpath("//meta:document-id").content = "fake_id"
+        sales_log_service.send(:create_log, sales_log_xml)
+
+        sales_log = SalesLog.where(old_id: sales_log_id).first
+        second_sales_log = SalesLog.where(old_id: "fake_id").first
+        expect(sales_log&.created_by).to eq(second_sales_log&.created_by)
+      end
+
+      context "when unassigned user exist for a different organisation" do
+        let!(:other_unassigned_user) { create(:user, name: "Unassigned") }
+
+        it "creates a new unassigned user for current organisation" do
+          expect(logger).to receive(:error).with("Sales log 'shared_ownership_sales_log' belongs to legacy user with owner-user-id: 'fake_id' which belongs to a different organisation. Assigning log to 'Unassigned' user.")
+          sales_log_service.send(:create_log, sales_log_xml)
+
+          sales_log = SalesLog.where(old_id: sales_log_id).first
+          expect(sales_log&.created_by&.name).to eq("Unassigned")
+          expect(sales_log&.created_by).not_to eq(other_unassigned_user)
+        end
+      end
+    end
+
     context "and the log startdate is before 22/23 collection period" do
       let(:sales_log_id) { "shared_ownership_sales_log" }
 

From c3636c12864f6c3d5d04765e52f169266c99e349 Mon Sep 17 00:00:00 2001
From: kosiakkatrina <54268893+kosiakkatrina@users.noreply.github.com>
Date: Thu, 7 Sep 2023 12:03:38 +0100
Subject: [PATCH 13/33] CLDC-2696 Add reimport addresses task (#1891)

* Add update address service for lettings

* Only process collections 2023 onwards

* Update app/services/imports/lettings_logs_field_import_service.rb

Co-authored-by: natdeanlewissoftwire <94526761+natdeanlewissoftwire@users.noreply.github.com>

* Refactor postcode_known

* test

---------

Co-authored-by: natdeanlewissoftwire <94526761+natdeanlewissoftwire@users.noreply.github.com>
---
 .../lettings_logs_field_import_service.rb     |  44 ++++++
 lib/tasks/data_import_field.rake              |   2 +-
 spec/lib/tasks/data_import_field_spec.rb      |  12 ++
 ...lettings_logs_field_import_service_spec.rb | 148 ++++++++++++++++++
 4 files changed, 205 insertions(+), 1 deletion(-)

diff --git a/app/services/imports/lettings_logs_field_import_service.rb b/app/services/imports/lettings_logs_field_import_service.rb
index 4e10c7e1c..38111a214 100644
--- a/app/services/imports/lettings_logs_field_import_service.rb
+++ b/app/services/imports/lettings_logs_field_import_service.rb
@@ -12,6 +12,8 @@ module Imports
         import_from(folder, :update_offered)
       when "creation_method"
         import_from(folder, :update_creation_method)
+      when "address"
+        import_from(folder, :update_address)
       else
         raise "Updating #{field} is not supported by the field import service"
       end
@@ -132,5 +134,47 @@ module Imports
         @logger.warn("Could not find record matching legacy ID #{old_id}")
       end
     end
+
+    def update_address(xml_doc)
+      return if meta_field_value(xml_doc, "form-name").include?("Sales")
+
+      old_id = meta_field_value(xml_doc, "document-id")
+      record = LettingsLog.find_by(old_id:)
+      return @logger.info("lettings log #{record.id} is from previous collection year, skipping") if record.collection_start_year < 2023
+
+      if record.present?
+        if string_or_nil(xml_doc, "AddressLine1").present? && string_or_nil(xml_doc, "TownCity").present?
+          record.la = string_or_nil(xml_doc, "Q28ONS")
+          record.postcode_full = compose_postcode(xml_doc, "POSTCODE", "POSTCOD2")
+          record.postcode_known = postcode_known(record)
+          record.address_line1 = string_or_nil(xml_doc, "AddressLine1")
+          record.address_line2 = string_or_nil(xml_doc, "AddressLine2")
+          record.town_or_city = string_or_nil(xml_doc, "TownCity")
+          record.county = string_or_nil(xml_doc, "County")
+          record.uprn = nil
+          record.uprn_known = 0
+          record.uprn_confirmed = 0
+          record.values_updated_at = Time.zone.now
+          record.save!
+          @logger.info("lettings log #{record.id} address_line1 value has been set to #{record.address_line1}")
+          @logger.info("lettings log #{record.id} address_line2 value has been set to #{record.address_line2}")
+          @logger.info("lettings log #{record.id} town_or_city value has been set to #{record.town_or_city}")
+          @logger.info("lettings log #{record.id} county value has been set to #{record.county}")
+          @logger.info("lettings log #{record.id} postcode_full value has been set to #{record.postcode_full}")
+        else
+          @logger.info("lettings log #{record.id} is missing either or both of address_line1 and town or city, skipping")
+        end
+      else
+        @logger.warn("Could not find record matching legacy ID #{old_id}")
+      end
+    end
+
+    def postcode_known(record)
+      if record.postcode_full.nil?
+        record.la.nil? ? nil : 0 # Assumes we selected No in the form since the LA is present
+      else
+        1
+      end
+    end
   end
 end
diff --git a/lib/tasks/data_import_field.rake b/lib/tasks/data_import_field.rake
index 80e42399e..7eb253277 100644
--- a/lib/tasks/data_import_field.rake
+++ b/lib/tasks/data_import_field.rake
@@ -7,7 +7,7 @@ namespace :core do
 
     # We only allow a reduced list of known fields to be updatable
     case field
-    when "tenancycode", "major_repairs", "lettings_allocation", "offered"
+    when "tenancycode", "major_repairs", "lettings_allocation", "offered", "address"
       s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
       archive_io = s3_service.get_file_io(path)
       archive_service = Storage::ArchiveService.new(archive_io)
diff --git a/spec/lib/tasks/data_import_field_spec.rb b/spec/lib/tasks/data_import_field_spec.rb
index 84de7cf87..4b5f6085f 100644
--- a/spec/lib/tasks/data_import_field_spec.rb
+++ b/spec/lib/tasks/data_import_field_spec.rb
@@ -92,6 +92,18 @@ describe "data_import_field imports" do
         end
       end
 
+      context "and we update the address fields" do
+        let(:field) { "address" }
+
+        it "updates the 2023 logs from the given XML file" do
+          expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name)
+          expect(storage_service).to receive(:get_file_io).with("spec/fixtures/imports/logs")
+          expect(Imports::LettingsLogsFieldImportService).to receive(:new).with(archive_service)
+          expect(import_service).to receive(:update_field).with(field, "logs")
+          task.invoke(field, fixture_path)
+        end
+      end
+
       it "raises an exception if no parameters are provided" do
         expect { task.invoke }.to raise_error(/Usage/)
       end
diff --git a/spec/services/imports/lettings_logs_field_import_service_spec.rb b/spec/services/imports/lettings_logs_field_import_service_spec.rb
index 7a9767aa2..d6222d355 100644
--- a/spec/services/imports/lettings_logs_field_import_service_spec.rb
+++ b/spec/services/imports/lettings_logs_field_import_service_spec.rb
@@ -323,4 +323,152 @@ RSpec.describe Imports::LettingsLogsFieldImportService do
       end
     end
   end
+
+  context "when updating address" do
+    let(:field) { "address" }
+
+    before do
+      WebMock.stub_request(:get, /api.postcodes.io\/postcodes\/B11BB/)
+      .to_return(status: 200, body: '{"status":200,"result":{"admin_district":"Westminster","codes":{"admin_district":"E08000035"}}}', headers: {})
+
+      stub_request(:get, "https://api.os.uk/search/places/v1/uprn?key=OS_DATA_KEY&uprn=123")
+        .to_return(status: 500, body: "{}", headers: {})
+
+      Timecop.freeze(2023, 5, 5)
+      Singleton.__init__(FormHandler)
+      Imports::LettingsLogsImportService.new(storage_service, logger).create_logs(fixture_directory)
+      lettings_log_file.rewind
+    end
+
+    after do
+      Timecop.unfreeze
+      Singleton.__init__(FormHandler)
+    end
+
+    context "when the lettings log has no address values" do
+      let(:lettings_log_id) { "00d2343e-d5fa-4c89-8400-ec3854b0f2b4" }
+      let(:lettings_log) { LettingsLog.find_by(old_id: lettings_log_id) }
+
+      before do
+        lettings_log.update!(uprn_known: nil,
+                             startdate: Time.zone.local(2023, 5, 5),
+                             uprn: nil,
+                             uprn_confirmed: nil,
+                             address_line1: nil,
+                             address_line2: nil,
+                             town_or_city: nil,
+                             county: nil,
+                             postcode_known: nil,
+                             postcode_full: nil,
+                             la: nil,
+                             is_la_inferred: nil)
+      end
+
+      context "and new address values include address" do
+        before do
+          lettings_log_xml.at_xpath("//xmlns:UPRN").content = "123456781234"
+          lettings_log_xml.at_xpath("//xmlns:AddressLine1").content = "address 1"
+          lettings_log_xml.at_xpath("//xmlns:AddressLine2").content = "address 2"
+          lettings_log_xml.at_xpath("//xmlns:TownCity").content = "towncity"
+          lettings_log_xml.at_xpath("//xmlns:County").content = "county"
+          lettings_log_xml.at_xpath("//xmlns:POSTCODE").content = "B1"
+          lettings_log_xml.at_xpath("//xmlns:POSTCOD2").content = "1BB"
+          lettings_log_xml.at_xpath("//xmlns:Q28ONS").content = nil
+        end
+
+        it "updates the lettings_log prioritising address values" do
+          expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} address_line1 value has been set to address 1/)
+          expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} address_line2 value has been set to address 2/)
+          expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} town_or_city value has been set to towncity/)
+          expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} county value has been set to county/)
+          expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} postcode_full value has been set to B1 1BB/)
+          import_service.send(:update_address, lettings_log_xml)
+          lettings_log.reload
+
+          expect(lettings_log.uprn_known).to eq(0)
+          expect(lettings_log.uprn).to eq(nil)
+          expect(lettings_log.uprn_confirmed).to eq(nil)
+          expect(lettings_log.address_line1).to eq("address 1")
+          expect(lettings_log.address_line2).to eq("address 2")
+          expect(lettings_log.town_or_city).to eq("towncity")
+          expect(lettings_log.county).to eq("county")
+          expect(lettings_log.postcode_known).to eq(1)
+          expect(lettings_log.postcode_full).to eq("B1 1BB")
+          expect(lettings_log.la).to eq("E08000035")
+          expect(lettings_log.is_la_inferred).to eq(true)
+        end
+      end
+
+      context "and new address values don't include address" do
+        it "skips the update" do
+          expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} is missing either or both of address_line1 and town or city, skipping/)
+          import_service.send(:update_address, lettings_log_xml)
+        end
+      end
+    end
+
+    context "when the lettings log has address values" do
+      let(:lettings_log_id) { "00d2343e-d5fa-4c89-8400-ec3854b0f2b4" }
+      let(:lettings_log) { LettingsLog.find_by(old_id: lettings_log_id) }
+
+      before do
+        lettings_log_xml.at_xpath("//xmlns:UPRN").content = "123456781234"
+        lettings_log_xml.at_xpath("//xmlns:AddressLine1").content = "address 1"
+        lettings_log_xml.at_xpath("//xmlns:AddressLine2").content = "address 2"
+        lettings_log_xml.at_xpath("//xmlns:TownCity").content = "towncity"
+        lettings_log_xml.at_xpath("//xmlns:County").content = "county"
+        lettings_log_xml.at_xpath("//xmlns:POSTCODE").content = "B1"
+        lettings_log_xml.at_xpath("//xmlns:POSTCOD2").content = "1BC"
+        lettings_log_xml.at_xpath("//xmlns:Q28ONS").content = nil
+        lettings_log.update!(uprn_known: 1,
+                             startdate: Time.zone.local(2023, 5, 5),
+                             uprn: "123",
+                             uprn_confirmed: 0,
+                             address_line1: "wrong address line1",
+                             address_line2: "wrong address 2",
+                             town_or_city: "wrong town",
+                             county: "wrong city",
+                             postcode_known: 1,
+                             postcode_full: "A11AA",
+                             la: "E06000064",
+                             is_la_inferred: true)
+      end
+
+      it "replaces the lettings_log address values" do
+        expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} address_line1 value has been set to address 1/)
+        expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} address_line2 value has been set to address 2/)
+        expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} town_or_city value has been set to towncity/)
+        expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} county value has been set to county/)
+        expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} postcode_full value has been set to B11BC/)
+        import_service.send(:update_address, lettings_log_xml)
+        lettings_log.reload
+
+        expect(lettings_log.uprn_known).to eq(0)
+        expect(lettings_log.uprn).to eq(nil)
+        expect(lettings_log.uprn_confirmed).to eq(nil)
+        expect(lettings_log.address_line1).to eq("address 1")
+        expect(lettings_log.address_line2).to eq("address 2")
+        expect(lettings_log.town_or_city).to eq("towncity")
+        expect(lettings_log.county).to eq("county")
+        expect(lettings_log.postcode_known).to eq(1)
+        expect(lettings_log.postcode_full).to eq("B11BC")
+        expect(lettings_log.la).to eq(nil)
+        expect(lettings_log.is_la_inferred).to eq(false)
+      end
+    end
+
+    context "when the lettings log is from before collection 23/24" do
+      let(:lettings_log_id) { "00d2343e-d5fa-4c89-8400-ec3854b0f2b4" }
+      let(:lettings_log) { LettingsLog.find_by(old_id: lettings_log_id) }
+
+      before do
+        lettings_log.update!(startdate: Time.zone.local(2022, 5, 5))
+      end
+
+      it "skips the update" do
+        expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} is from previous collection year, skipping/)
+        import_service.send(:update_address, lettings_log_xml)
+      end
+    end
+  end
 end

From e5e6e1e21a02076092f0eaf6c91460ead2bdb57e Mon Sep 17 00:00:00 2001
From: natdeanlewissoftwire
 <94526761+natdeanlewissoftwire@users.noreply.github.com>
Date: Thu, 7 Sep 2023 16:09:46 +0100
Subject: [PATCH 14/33] Enforce offered as integer (#1897)

* feat: ensure offered is an integer (decimal 0 was triggering validation errors)

* feat: allow decimals but clear when not integer equivalent as per new requirements
---
 app/models/validations/shared_validations.rb  |  2 +-
 .../imports/lettings_logs_import_service.rb   |  1 +
 .../lettings_logs_import_service_spec.rb      | 21 +++++++++++++++++++
 3 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/app/models/validations/shared_validations.rb b/app/models/validations/shared_validations.rb
index 8cf2e53fe..81bda79de 100644
--- a/app/models/validations/shared_validations.rb
+++ b/app/models/validations/shared_validations.rb
@@ -49,7 +49,7 @@ module Validations::SharedValidations
       elsif incorrect_accuracy || value.to_d != value.to_i    # if the user enters a value in exponent notation (eg '4e1') the to_i method does not convert this to the correct value
         field = question.check_answer_label || question.id
         case question.step
-        when 1 then record.errors.add question.id.to_sym, I18n.t("validations.numeric.whole_number", field:)
+        when 1 then record.errors.add question.id.to_sym, :not_integer, message: I18n.t("validations.numeric.whole_number", field:)
         when 10 then record.errors.add question.id.to_sym, I18n.t("validations.numeric.nearest_ten", field:)
         end
       end
diff --git a/app/services/imports/lettings_logs_import_service.rb b/app/services/imports/lettings_logs_import_service.rb
index a53048d83..cbaeee01a 100644
--- a/app/services/imports/lettings_logs_import_service.rb
+++ b/app/services/imports/lettings_logs_import_service.rb
@@ -326,6 +326,7 @@ module Imports
         %i[prevten non_temp_accommodation] => %w[prevten rsnvac],
         %i[joint not_joint_tenancy] => %w[joint],
         %i[offered outside_the_range] => %w[offered],
+        %i[offered not_integer] => %w[offered],
         %i[earnings over_hard_max] => %w[ecstat1],
         %i[tshortfall no_outstanding_charges] => %w[tshortfall hbrentshortfall],
         %i[beds outside_the_range] => %w[beds],
diff --git a/spec/services/imports/lettings_logs_import_service_spec.rb b/spec/services/imports/lettings_logs_import_service_spec.rb
index 7dc3d578d..8367c7d71 100644
--- a/spec/services/imports/lettings_logs_import_service_spec.rb
+++ b/spec/services/imports/lettings_logs_import_service_spec.rb
@@ -545,6 +545,27 @@ RSpec.describe Imports::LettingsLogsImportService do
         end
       end
 
+      context "and the number of times the property was relet is 0.01" do
+        before do
+          lettings_log_xml.at_xpath("//xmlns:Q20").content = "0.01"
+        end
+
+        it "intercepts the relevant validation error" do
+          expect { lettings_log_service.send(:create_log, lettings_log_xml) }
+            .not_to raise_error
+        end
+
+        it "clears out the number offered answer" do
+          allow(logger).to receive(:warn)
+
+          lettings_log_service.send(:create_log, lettings_log_xml)
+          lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
+
+          expect(lettings_log).not_to be_nil
+          expect(lettings_log.offered).to be_nil
+        end
+      end
+
       context "and the gender identity is refused" do
         before do
           lettings_log_xml.at_xpath("//xmlns:P1Sex").content = "Person prefers not to say"

From 41228e22958dcb2ab5585398409a7d3726553642 Mon Sep 17 00:00:00 2001
From: natdeanlewissoftwire
 <94526761+natdeanlewissoftwire@users.noreply.github.com>
Date: Fri, 8 Sep 2023 16:02:03 +0100
Subject: [PATCH 15/33] Fix hundredth error on migration of very small scharge
 values (#1898)

* feat: round all decimal values to 2dp in safe_string_as_decimal

* refactor: lint
---
 app/services/imports/logs_import_service.rb   |  2 +-
 .../lettings_logs_import_service_spec.rb      | 23 +++++++++++++++++++
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/app/services/imports/logs_import_service.rb b/app/services/imports/logs_import_service.rb
index 930956f16..a82208bbe 100644
--- a/app/services/imports/logs_import_service.rb
+++ b/app/services/imports/logs_import_service.rb
@@ -72,7 +72,7 @@ module Imports
       if str.nil?
         nil
       else
-        BigDecimal(str, exception: false)
+        BigDecimal(str, exception: false).round(2)
       end
     end
 
diff --git a/spec/services/imports/lettings_logs_import_service_spec.rb b/spec/services/imports/lettings_logs_import_service_spec.rb
index 8367c7d71..6bfe53571 100644
--- a/spec/services/imports/lettings_logs_import_service_spec.rb
+++ b/spec/services/imports/lettings_logs_import_service_spec.rb
@@ -796,6 +796,29 @@ RSpec.describe Imports::LettingsLogsImportService do
         end
       end
 
+      context "and scharge is ever so slightly positive" do
+        let(:lettings_log_id) { "0b4a68df-30cc-474a-93c0-a56ce8fdad3b" }
+
+        before do
+          lettings_log_xml.at_xpath("//xmlns:Q18aii").content = "1.66533E-16"
+        end
+
+        it "does not raise an error" do
+          expect { lettings_log_service.send(:create_log, lettings_log_xml) }
+            .not_to raise_error
+        end
+
+        it "sets scharge to 0" do
+          allow(logger).to receive(:warn)
+
+          lettings_log_service.send(:create_log, lettings_log_xml)
+          lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
+
+          expect(lettings_log).not_to be_nil
+          expect(lettings_log.scharge).to eq(0)
+        end
+      end
+
       context "and tshortfall is not positive" do
         let(:lettings_log_id) { "0b4a68df-30cc-474a-93c0-a56ce8fdad3b" }
 

From 19ad92c21d28cf03ca8255d00d4103aae0f047f2 Mon Sep 17 00:00:00 2001
From: natdeanlewissoftwire
 <94526761+natdeanlewissoftwire@users.noreply.github.com>
Date: Fri, 8 Sep 2023 16:26:45 +0100
Subject: [PATCH 16/33] feat: don't raise "no matching location" error (#1900)

---
 .../imports/lettings_logs_import_service.rb   |  3 ---
 .../lettings_logs_import_service_spec.rb      | 19 +++++++++++++++++++
 2 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/app/services/imports/lettings_logs_import_service.rb b/app/services/imports/lettings_logs_import_service.rb
index cbaeee01a..49f1e4603 100644
--- a/app/services/imports/lettings_logs_import_service.rb
+++ b/app/services/imports/lettings_logs_import_service.rb
@@ -217,9 +217,6 @@ module Imports
 
         schemes = Scheme.where(old_visible_id: scheme_old_visible_id, owning_organisation_id: attributes["owning_organisation_id"])
         location = Location.find_by(old_visible_id: location_old_visible_id, scheme: schemes)
-        if location.nil? && [location_old_visible_id, scheme_old_visible_id].all?(&:present?) && previous_status != "saved"
-          raise "No matching location for scheme #{scheme_old_visible_id} and location #{location_old_visible_id} (visible IDs)"
-        end
 
         if location.present?
           # Set the scheme via location, because the scheme old visible ID can be duplicated at import
diff --git a/spec/services/imports/lettings_logs_import_service_spec.rb b/spec/services/imports/lettings_logs_import_service_spec.rb
index 6bfe53571..c1eac29bb 100644
--- a/spec/services/imports/lettings_logs_import_service_spec.rb
+++ b/spec/services/imports/lettings_logs_import_service_spec.rb
@@ -1248,6 +1248,25 @@ RSpec.describe Imports::LettingsLogsImportService do
         end
       end
 
+      context "and the scheme and location are not valid" do
+        let(:lettings_log_id) { "0b4a68df-30cc-474a-93c0-a56ce8fdad3b" }
+
+        before do
+          lettings_log_xml.at_xpath("//xmlns:_1cmangroupcode").content = "999"
+          lettings_log_xml.at_xpath("//xmlns:_1cschemecode").content = "999"
+        end
+
+        it "saves log without location and scheme" do
+          expect(logger).not_to receive(:warn)
+          lettings_log_service.send(:create_log, lettings_log_xml)
+          lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
+
+          expect(lettings_log.scheme_id).to be_nil
+          expect(lettings_log.location_id).to be_nil
+          expect(lettings_log.status).to eq("in_progress")
+        end
+      end
+
       context "and this is a supported housing log with a single location under a scheme" do
         let(:lettings_log_id) { "0b4a68df-30cc-474a-93c0-a56ce8fdad3b" }
 

From 11ca1bbcff7db9670a7b05d8239f610e7497f96b Mon Sep 17 00:00:00 2001
From: kosiakkatrina <54268893+kosiakkatrina@users.noreply.github.com>
Date: Tue, 12 Sep 2023 08:21:04 +0100
Subject: [PATCH 17/33] CLDC-2724 Add missing data reporting for imported
 lettings logs (#1892)

* Add generate_missing_answers_report task and method

* Create example report

* change task description

* refactor

* Report old form id instead of old id
---
 app/services/imports/import_report_service.rb | 47 +++++++++++++++++++
 lib/tasks/full_import.rake                    |  9 ++++
 ...lettings_logs_missing_answers_examples.csv | 16 +++++++
 ...d_lettings_logs_missing_answers_report.csv |  4 ++
 spec/lib/tasks/full_import_spec.rb            | 26 ++++++++++
 .../imports/import_report_service_spec.rb     | 38 +++++++++++++++
 6 files changed, 140 insertions(+)
 create mode 100644 spec/fixtures/files/imported_lettings_logs_missing_answers_examples.csv
 create mode 100644 spec/fixtures/files/imported_lettings_logs_missing_answers_report.csv

diff --git a/app/services/imports/import_report_service.rb b/app/services/imports/import_report_service.rb
index 4e387c4c4..7dcde699c 100644
--- a/app/services/imports/import_report_service.rb
+++ b/app/services/imports/import_report_service.rb
@@ -84,5 +84,52 @@ module Imports
 
       @logger.info("Unassigned logs report available in s3 import bucket at #{report_name}")
     end
+
+    def generate_missing_answers_report(report_suffix)
+      Rails.logger.info("Generating missing imported lettings logs answers report")
+      unanswered_question_counts = {}
+      missing_answers_example_sets = {}
+
+      LettingsLog.where.not(old_id: nil).where(status: "in_progress").each do |lettings_log|
+        applicable_questions = lettings_log.form.subsections.map { |s| s.applicable_questions(lettings_log).select { |q| q.enabled?(lettings_log) } }.flatten
+        unanswered_questions = (applicable_questions.filter { |q| q.unanswered?(lettings_log) }.map(&:id) - lettings_log.optional_fields).join(", ")
+
+        if unanswered_question_counts[unanswered_questions].present?
+          unanswered_question_counts[unanswered_questions] += 1
+          missing_answers_example_sets[unanswered_questions] << { id: lettings_log.id, old_form_id: lettings_log.old_form_id, owning_organisation_id: lettings_log.owning_organisation_id } unless unanswered_question_counts[unanswered_questions] > 10
+        else
+          unanswered_question_counts[unanswered_questions] = 1
+          missing_answers_example_sets[unanswered_questions] = [{ id: lettings_log.id, old_form_id: lettings_log.old_form_id, owning_organisation_id: lettings_log.owning_organisation_id }]
+        end
+      end
+
+      rep = CSV.generate do |report|
+        headers = ["Missing answers", "Total number of affected logs"]
+        report << headers
+
+        unanswered_question_counts.each do |missing_answers, count|
+          report << [missing_answers, count]
+        end
+      end
+
+      missing_answers_examples = CSV.generate do |report|
+        headers = ["Missing answers", "Organisation ID", "Log ID", "Old Form ID"]
+        report << headers
+
+        missing_answers_example_sets.each do |missing_answers, examples|
+          examples.each do |example|
+            report << [missing_answers, example[:owning_organisation_id], example[:id], example[:old_form_id]]
+          end
+        end
+      end
+
+      report_name = "MissingAnswersReport_#{report_suffix}.csv"
+      @storage_service.write_file(report_name, BYTE_ORDER_MARK + rep)
+
+      examples_report_name = "MissingAnswersExamples_#{report_suffix}.csv"
+      @storage_service.write_file(examples_report_name, BYTE_ORDER_MARK + missing_answers_examples)
+
+      @logger.info("Missing answers report available in s3 import bucket at #{report_name}")
+    end
   end
 end
diff --git a/lib/tasks/full_import.rake b/lib/tasks/full_import.rake
index 3b74b458a..4c5fcf380 100644
--- a/lib/tasks/full_import.rake
+++ b/lib/tasks/full_import.rake
@@ -117,6 +117,15 @@ namespace :import do
     Imports::ImportReportService.new(s3_service, institutions_csv).create_reports(institutions_csv_name)
   end
 
+  desc "Generate missing answers report"
+  task :generate_missing_answers_report, %i[file_suffix] => :environment do |_task, args|
+    file_suffix = args[:file_suffix]
+    raise "Usage: rake import:generate_reports['file_suffix']" if file_suffix.blank?
+
+    s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
+    Imports::ImportReportService.new(s3_service, nil).generate_missing_answers_report(file_suffix)
+  end
+
   desc "Run import from logs step to end"
   task :logs_onwards, %i[institutions_csv_name] => %i[environment logs trigger_invites generate_reports]
 
diff --git a/spec/fixtures/files/imported_lettings_logs_missing_answers_examples.csv b/spec/fixtures/files/imported_lettings_logs_missing_answers_examples.csv
new file mode 100644
index 000000000..0c35bae0b
--- /dev/null
+++ b/spec/fixtures/files/imported_lettings_logs_missing_answers_examples.csv
@@ -0,0 +1,16 @@
+Missing answers,Organisation ID,Log ID,Old Form ID
+age1_known,{org_id0},{id0},1000
+age1_known,{org_id1},{id1},1001
+age1_known,{org_id2},{id2},1002
+age1_known,{org_id3},{id3},1003
+age1_known,{org_id4},{id4},1004
+age1_known,{org_id5},{id5},1005
+age1_known,{org_id6},{id6},1006
+age1_known,{org_id7},{id7},1007
+age1_known,{org_id8},{id8},1008
+age1_known,{org_id9},{id9},1009
+beds,{org_id2_0},{id2_0},2000
+beds,{org_id2_1},{id2_1},2001
+beds,{org_id2_2},{id2_2},2002
+beds,{org_id2_3},{id2_3},2003
+"beds, age1_known",{org_id},{id},300
diff --git a/spec/fixtures/files/imported_lettings_logs_missing_answers_report.csv b/spec/fixtures/files/imported_lettings_logs_missing_answers_report.csv
new file mode 100644
index 000000000..936dc82e0
--- /dev/null
+++ b/spec/fixtures/files/imported_lettings_logs_missing_answers_report.csv
@@ -0,0 +1,4 @@
+Missing answers,Total number of affected logs
+age1_known,11
+beds,4
+"beds, age1_known",1
diff --git a/spec/lib/tasks/full_import_spec.rb b/spec/lib/tasks/full_import_spec.rb
index b95b370cd..2bb85e9b6 100644
--- a/spec/lib/tasks/full_import_spec.rb
+++ b/spec/lib/tasks/full_import_spec.rb
@@ -54,6 +54,32 @@ describe "full import", type: :task do
     end
   end
 
+  describe "import:generate_missing_answers_report" do
+    subject(:task) { Rake::Task["import:generate_missing_answers_report"] }
+
+    before do
+      Rake.application.rake_require("tasks/full_import")
+      Rake::Task.define_task(:environment)
+      task.reenable
+    end
+
+    context "when generating a missing answers report" do
+      let(:import_report_service) { instance_double(Imports::ImportReportService) }
+
+      before do
+        allow(Imports::ImportReportService).to receive(:new).and_return(import_report_service)
+        allow(ENV).to receive(:[]).with("VCAP_SERVICES").and_return("dummy")
+      end
+
+      it "creates a missing answers report" do
+        expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name)
+        expect(Imports::ImportReportService).to receive(:new).with(storage_service, nil)
+        expect(import_report_service).to receive(:generate_missing_answers_report).with("some_name")
+        task.invoke("some_name")
+      end
+    end
+  end
+
   describe "import:initial" do
     subject(:task) { Rake::Task["import:initial"] }
 
diff --git a/spec/services/imports/import_report_service_spec.rb b/spec/services/imports/import_report_service_spec.rb
index abd318a8d..ca6e81bc2 100644
--- a/spec/services/imports/import_report_service_spec.rb
+++ b/spec/services/imports/import_report_service_spec.rb
@@ -98,4 +98,42 @@ RSpec.describe Imports::ImportReportService do
       end
     end
   end
+
+  describe "#generate_missing_answers_report" do
+    context "when there are in progress imported logs" do
+      let(:institutions_csv) { nil }
+      let(:expected_content) { File.read("spec/fixtures/files/imported_lettings_logs_missing_answers_report.csv") }
+      let(:expected__answers_examples_content) { File.read("spec/fixtures/files/imported_lettings_logs_missing_answers_examples.csv") }
+
+      before do
+        create_list(:lettings_log, 11, :completed, age1_known: nil) do |log, i|
+          log.old_form_id = "100#{i}"
+          log.old_id = "old_id_age1_known_#{i}"
+          log.save!
+          expected__answers_examples_content.sub!("{id#{i}}", log.id.to_s)
+          expected__answers_examples_content.sub!("{org_id#{i}}", log.owning_organisation_id.to_s)
+        end
+        create_list(:lettings_log, 4, :completed, beds: nil) do |log, i|
+          log.old_form_id = "200#{i}"
+          log.old_id = "old_id_beds_#{i}"
+          expected__answers_examples_content.sub!("{id2_#{i}}", log.id.to_s)
+          expected__answers_examples_content.sub!("{org_id2_#{i}}", log.owning_organisation_id.to_s)
+          log.save!
+        end
+        create(:lettings_log, :completed, age1_known: nil, beds: nil, old_form_id: "300", old_id: "123") do |log|
+          expected__answers_examples_content.sub!("{id}", log.id.to_s)
+          expected__answers_examples_content.sub!("{org_id}", log.owning_organisation_id.to_s)
+        end
+
+        create_list(:lettings_log, 2, :completed, age1_known: nil)
+      end
+
+      it "generates a csv with expected missing fields" do
+        expect(storage_service).to receive(:write_file).with("MissingAnswersReport_report_suffix.csv", "#{expected_content}")
+        expect(storage_service).to receive(:write_file).with("MissingAnswersExamples_report_suffix.csv", "#{expected__answers_examples_content}")
+
+        report_service.generate_missing_answers_report("report_suffix")
+      end
+    end
+  end
 end

From f59c67b651894e336921fed135991ef8788eb5d2 Mon Sep 17 00:00:00 2001
From: kosiakkatrina <54268893+kosiakkatrina@users.noreply.github.com>
Date: Tue, 12 Sep 2023 08:42:40 +0100
Subject: [PATCH 18/33] Add reimport reason update_field case (#1896)

---
 .../lettings_logs_field_import_service.rb     | 27 +++++++
 lib/tasks/data_import_field.rake              |  2 +-
 spec/lib/tasks/data_import_field_spec.rb      | 12 ++++
 ...lettings_logs_field_import_service_spec.rb | 70 +++++++++++++++++++
 4 files changed, 110 insertions(+), 1 deletion(-)

diff --git a/app/services/imports/lettings_logs_field_import_service.rb b/app/services/imports/lettings_logs_field_import_service.rb
index 38111a214..68c3ba5f4 100644
--- a/app/services/imports/lettings_logs_field_import_service.rb
+++ b/app/services/imports/lettings_logs_field_import_service.rb
@@ -14,6 +14,8 @@ module Imports
         import_from(folder, :update_creation_method)
       when "address"
         import_from(folder, :update_address)
+      when "reason"
+        import_from(folder, :update_reason)
       else
         raise "Updating #{field} is not supported by the field import service"
       end
@@ -176,5 +178,30 @@ module Imports
         1
       end
     end
+
+    def update_reason(xml_doc)
+      return if meta_field_value(xml_doc, "form-name").include?("Sales")
+
+      old_id = meta_field_value(xml_doc, "document-id")
+      record = LettingsLog.find_by(old_id:)
+
+      if record.present?
+        if record.reason.present?
+          @logger.info("lettings log #{record.id} has a value for reason, skipping update")
+        else
+          reason = unsafe_string_as_integer(xml_doc, "Q9a")
+          reasonother = string_or_nil(xml_doc, "Q9aa")
+          if reason == 20 && reasonother.blank?
+            @logger.info("lettings log #{record.id}'s reason is other but other reason is not provided, skipping update")
+          else
+            record.update!(reason:, reasonother:)
+            @logger.info("lettings log #{record.id}'s reason value has been set to #{reason}")
+            @logger.info("lettings log #{record.id}'s reasonother value has been set to #{reasonother}") if record.reasonother.present?
+          end
+        end
+      else
+        @logger.warn("lettings log with old id #{old_id} not found")
+      end
+    end
   end
 end
diff --git a/lib/tasks/data_import_field.rake b/lib/tasks/data_import_field.rake
index 7eb253277..c52d69d99 100644
--- a/lib/tasks/data_import_field.rake
+++ b/lib/tasks/data_import_field.rake
@@ -7,7 +7,7 @@ namespace :core do
 
     # We only allow a reduced list of known fields to be updatable
     case field
-    when "tenancycode", "major_repairs", "lettings_allocation", "offered", "address"
+    when "tenancycode", "major_repairs", "lettings_allocation", "offered", "address", "reason"
       s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
       archive_io = s3_service.get_file_io(path)
       archive_service = Storage::ArchiveService.new(archive_io)
diff --git a/spec/lib/tasks/data_import_field_spec.rb b/spec/lib/tasks/data_import_field_spec.rb
index 4b5f6085f..18eb656bb 100644
--- a/spec/lib/tasks/data_import_field_spec.rb
+++ b/spec/lib/tasks/data_import_field_spec.rb
@@ -104,6 +104,18 @@ describe "data_import_field imports" do
         end
       end
 
+      context "and we update the reason field" do
+        let(:field) { "reason" }
+
+        it "updates the 2023 logs from the given XML file" do
+          expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name)
+          expect(storage_service).to receive(:get_file_io).with("spec/fixtures/imports/logs")
+          expect(Imports::LettingsLogsFieldImportService).to receive(:new).with(archive_service)
+          expect(import_service).to receive(:update_field).with(field, "logs")
+          task.invoke(field, fixture_path)
+        end
+      end
+
       it "raises an exception if no parameters are provided" do
         expect { task.invoke }.to raise_error(/Usage/)
       end
diff --git a/spec/services/imports/lettings_logs_field_import_service_spec.rb b/spec/services/imports/lettings_logs_field_import_service_spec.rb
index d6222d355..4bed2dc63 100644
--- a/spec/services/imports/lettings_logs_field_import_service_spec.rb
+++ b/spec/services/imports/lettings_logs_field_import_service_spec.rb
@@ -471,4 +471,74 @@ RSpec.describe Imports::LettingsLogsFieldImportService do
       end
     end
   end
+
+  context "when updating reason" do
+    let(:field) { "reason" }
+
+    context "when the lettings log has no reason value" do
+      let(:lettings_log) { LettingsLog.find_by(old_id: lettings_log_id) }
+
+      before do
+        Imports::LettingsLogsImportService.new(storage_service, logger).create_logs(fixture_directory)
+        lettings_log_file.rewind
+        lettings_log.update!(reason: nil)
+        lettings_log_xml.at_xpath("//xmlns:Q9a").content = "47"
+      end
+
+      it "updates the lettings_log reason value" do
+        expect(logger).to receive(:info).with(/lettings log \d+'s reason value has been set to 47/)
+        expect { import_service.send(:update_reason, lettings_log_xml) }
+          .to(change { lettings_log.reload.reason }.from(nil).to(47))
+      end
+    end
+
+    context "when the lettings log has a different reason value" do
+      let(:lettings_log) { LettingsLog.find_by(old_id: lettings_log_id) }
+
+      before do
+        Imports::LettingsLogsImportService.new(storage_service, logger).create_logs(fixture_directory)
+        lettings_log_file.rewind
+        lettings_log.update!(reason: 18)
+        lettings_log_xml.at_xpath("//xmlns:Q9a").content = "47"
+      end
+
+      it "does not update the lettings_log reason value" do
+        expect(logger).to receive(:info).with(/lettings log \d+ has a value for reason, skipping update/)
+        expect { import_service.send(:update_reason, lettings_log_xml) }
+          .not_to(change { lettings_log.reload.reason })
+      end
+    end
+
+    context "when the new value is 'other'" do
+      let(:lettings_log) { LettingsLog.find_by(old_id: lettings_log_id) }
+
+      before do
+        Imports::LettingsLogsImportService.new(storage_service, logger).create_logs(fixture_directory)
+        lettings_log_file.rewind
+        lettings_log.update!(reason: nil)
+        lettings_log_xml.at_xpath("//xmlns:Q9a").content = "20"
+      end
+
+      context "and other value is given" do
+        before do
+          lettings_log_xml.at_xpath("//xmlns:Q9aa").content = "other"
+        end
+
+        it "updates the lettings_log reason value" do
+          expect(logger).to receive(:info).with(/lettings log \d+'s reason value has been set to 20/)
+          expect(logger).to receive(:info).with(/lettings log \d+'s reasonother value has been set to other/)
+          expect { import_service.send(:update_reason, lettings_log_xml) }
+            .to(change { lettings_log.reload.reason }.from(nil).to(20))
+        end
+      end
+
+      context "and other value is not given" do
+        it "does not update the lettings_log reason value" do
+          expect(logger).to receive(:info).with(/lettings log \d+'s reason is other but other reason is not provided, skipping update/)
+          expect { import_service.send(:update_reason, lettings_log_xml) }
+            .not_to(change { lettings_log.reload.reason })
+        end
+      end
+    end
+  end
 end

From 8726d231bd7334c6a2cd001c11c50e2607786133 Mon Sep 17 00:00:00 2001
From: kosiakkatrina <54268893+kosiakkatrina@users.noreply.github.com>
Date: Tue, 12 Sep 2023 09:41:01 +0100
Subject: [PATCH 19/33] Remove homelessness validation from the import (#1901)

---
 app/services/imports/lettings_logs_import_service.rb | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/app/services/imports/lettings_logs_import_service.rb b/app/services/imports/lettings_logs_import_service.rb
index 49f1e4603..f400612fe 100644
--- a/app/services/imports/lettings_logs_import_service.rb
+++ b/app/services/imports/lettings_logs_import_service.rb
@@ -142,12 +142,6 @@ module Imports
       attributes["rp_hardship"] = unsafe_string_as_integer(xml_doc, "Q14b4").present? ? 1 : nil
       attributes["rp_dontknow"] = unsafe_string_as_integer(xml_doc, "Q14b5").present? ? 1 : nil
 
-      # Trips validation
-      if attributes["homeless"] == 1 && attributes["rp_homeless"] == 1
-        attributes["homeless"] = nil
-        attributes["rp_homeless"] = nil
-      end
-
       attributes["cbl"] = allocation_system(unsafe_string_as_integer(xml_doc, "Q15CBL"))
       attributes["chr"] = allocation_system(unsafe_string_as_integer(xml_doc, "Q15CHR"))
       attributes["cap"] = allocation_system(unsafe_string_as_integer(xml_doc, "Q15CAP"))

From bda7bc7d08eb46158b14428ea097d58dd32f81b8 Mon Sep 17 00:00:00 2001
From: kosiakkatrina <54268893+kosiakkatrina@users.noreply.github.com>
Date: Tue, 12 Sep 2023 10:48:27 +0100
Subject: [PATCH 20/33] Only blank tenancy length when it is invalid for
 tenancy type (#1902)

* Only blank tenancy lenght when it is invalid for tenancy type

* Update spec/services/imports/lettings_logs_import_service_spec.rb

Co-authored-by: natdeanlewissoftwire <94526761+natdeanlewissoftwire@users.noreply.github.com>

---------

Co-authored-by: natdeanlewissoftwire <94526761+natdeanlewissoftwire@users.noreply.github.com>
---
 app/services/imports/lettings_logs_import_service.rb       | 2 +-
 spec/services/imports/lettings_logs_import_service_spec.rb | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/services/imports/lettings_logs_import_service.rb b/app/services/imports/lettings_logs_import_service.rb
index f400612fe..ca3cde434 100644
--- a/app/services/imports/lettings_logs_import_service.rb
+++ b/app/services/imports/lettings_logs_import_service.rb
@@ -312,7 +312,7 @@ module Imports
         %i[chcharge out_of_range] => %w[chcharge],
         %i[referral internal_transfer_non_social_housing] => %w[referral],
         %i[referral internal_transfer_fixed_or_lifetime] => %w[referral],
-        %i[tenancylength tenancylength_invalid] => %w[tenancylength tenancy],
+        %i[tenancylength tenancylength_invalid] => %w[tenancylength],
         %i[prevten over_25_foster_care] => %w[prevten age1],
         %i[prevten non_temp_accommodation] => %w[prevten rsnvac],
         %i[joint not_joint_tenancy] => %w[joint],
diff --git a/spec/services/imports/lettings_logs_import_service_spec.rb b/spec/services/imports/lettings_logs_import_service_spec.rb
index c1eac29bb..800955cfc 100644
--- a/spec/services/imports/lettings_logs_import_service_spec.rb
+++ b/spec/services/imports/lettings_logs_import_service_spec.rb
@@ -325,7 +325,7 @@ RSpec.describe Imports::LettingsLogsImportService do
             .not_to raise_error
         end
 
-        it "clears out the invalid answers" do
+        it "clears out the invalid tenancy length" do
           allow(logger).to receive(:warn)
 
           lettings_log_service.send(:create_log, lettings_log_xml)
@@ -333,7 +333,7 @@ RSpec.describe Imports::LettingsLogsImportService do
 
           expect(lettings_log).not_to be_nil
           expect(lettings_log.tenancylength).to be_nil
-          expect(lettings_log.tenancy).to be_nil
+          expect(lettings_log.tenancy).to eq(4)
         end
       end
 

From c049089684af25425730d7843c7cf0c14fe588e2 Mon Sep 17 00:00:00 2001
From: kosiakkatrina <54268893+kosiakkatrina@users.noreply.github.com>
Date: Tue, 12 Sep 2023 10:49:41 +0100
Subject: [PATCH 21/33] CLDC-2725 Add missing sales data report (#1893)

* Add missing sales answers reporting

* Fix log name
---
 app/services/imports/import_report_service.rb | 49 ++++++++++-----
 .../imports/import_report_service_spec.rb     | 60 +++++++++++++++----
 2 files changed, 83 insertions(+), 26 deletions(-)

diff --git a/app/services/imports/import_report_service.rb b/app/services/imports/import_report_service.rb
index 7dcde699c..ffcd78fc5 100644
--- a/app/services/imports/import_report_service.rb
+++ b/app/services/imports/import_report_service.rb
@@ -86,24 +86,47 @@ module Imports
     end
 
     def generate_missing_answers_report(report_suffix)
-      Rails.logger.info("Generating missing imported lettings logs answers report")
+      create_missing_answers_report(report_suffix, LettingsLog)
+      create_missing_answers_report(report_suffix, SalesLog)
+    end
+
+    def create_missing_answers_report(report_suffix, log_class)
+      class_name = log_class.name.underscore.humanize
+      Rails.logger.info("Generating missing imported #{class_name}s answers report")
+
+      unanswered_question_counts, missing_answers_example_sets = process_missing_answers(log_class)
+
+      report_name = "MissingAnswersReport#{log_class.name}_#{report_suffix}.csv"
+      @storage_service.write_file(report_name, BYTE_ORDER_MARK + missing_answers_report(unanswered_question_counts))
+
+      examples_report_name = "MissingAnswersExamples#{log_class.name}_#{report_suffix}.csv"
+      @storage_service.write_file(examples_report_name, BYTE_ORDER_MARK + missing_answers_examples(missing_answers_example_sets))
+
+      @logger.info("Missing #{class_name}s answers report available in s3 import bucket at #{report_name}")
+    end
+
+    def process_missing_answers(log_class)
       unanswered_question_counts = {}
       missing_answers_example_sets = {}
 
-      LettingsLog.where.not(old_id: nil).where(status: "in_progress").each do |lettings_log|
-        applicable_questions = lettings_log.form.subsections.map { |s| s.applicable_questions(lettings_log).select { |q| q.enabled?(lettings_log) } }.flatten
-        unanswered_questions = (applicable_questions.filter { |q| q.unanswered?(lettings_log) }.map(&:id) - lettings_log.optional_fields).join(", ")
+      log_class.where.not(old_id: nil).where(status: "in_progress").each do |log|
+        applicable_questions = log.form.subsections.map { |s| s.applicable_questions(log).select { |q| q.enabled?(log) } }.flatten
+        unanswered_questions = (applicable_questions.filter { |q| q.unanswered?(log) }.map(&:id) - log.optional_fields).join(", ")
 
         if unanswered_question_counts[unanswered_questions].present?
           unanswered_question_counts[unanswered_questions] += 1
-          missing_answers_example_sets[unanswered_questions] << { id: lettings_log.id, old_form_id: lettings_log.old_form_id, owning_organisation_id: lettings_log.owning_organisation_id } unless unanswered_question_counts[unanswered_questions] > 10
+          missing_answers_example_sets[unanswered_questions] << { id: log.id, old_form_id: log.old_form_id, owning_organisation_id: log.owning_organisation_id } unless unanswered_question_counts[unanswered_questions] > 10
         else
           unanswered_question_counts[unanswered_questions] = 1
-          missing_answers_example_sets[unanswered_questions] = [{ id: lettings_log.id, old_form_id: lettings_log.old_form_id, owning_organisation_id: lettings_log.owning_organisation_id }]
+          missing_answers_example_sets[unanswered_questions] = [{ id: log.id, old_form_id: log.old_form_id, owning_organisation_id: log.owning_organisation_id }]
         end
       end
 
-      rep = CSV.generate do |report|
+      [unanswered_question_counts, missing_answers_example_sets]
+    end
+
+    def missing_answers_report(unanswered_question_counts)
+      CSV.generate do |report|
         headers = ["Missing answers", "Total number of affected logs"]
         report << headers
 
@@ -111,8 +134,10 @@ module Imports
           report << [missing_answers, count]
         end
       end
+    end
 
-      missing_answers_examples = CSV.generate do |report|
+    def missing_answers_examples(missing_answers_example_sets)
+      CSV.generate do |report|
         headers = ["Missing answers", "Organisation ID", "Log ID", "Old Form ID"]
         report << headers
 
@@ -122,14 +147,6 @@ module Imports
           end
         end
       end
-
-      report_name = "MissingAnswersReport_#{report_suffix}.csv"
-      @storage_service.write_file(report_name, BYTE_ORDER_MARK + rep)
-
-      examples_report_name = "MissingAnswersExamples_#{report_suffix}.csv"
-      @storage_service.write_file(examples_report_name, BYTE_ORDER_MARK + missing_answers_examples)
-
-      @logger.info("Missing answers report available in s3 import bucket at #{report_name}")
     end
   end
 end
diff --git a/spec/services/imports/import_report_service_spec.rb b/spec/services/imports/import_report_service_spec.rb
index ca6e81bc2..938ab9f1b 100644
--- a/spec/services/imports/import_report_service_spec.rb
+++ b/spec/services/imports/import_report_service_spec.rb
@@ -100,37 +100,77 @@ RSpec.describe Imports::ImportReportService do
   end
 
   describe "#generate_missing_answers_report" do
-    context "when there are in progress imported logs" do
+    context "when there are in progress imported lettings logs" do
       let(:institutions_csv) { nil }
       let(:expected_content) { File.read("spec/fixtures/files/imported_lettings_logs_missing_answers_report.csv") }
-      let(:expected__answers_examples_content) { File.read("spec/fixtures/files/imported_lettings_logs_missing_answers_examples.csv") }
+      let(:expected_answers_examples_content) { File.read("spec/fixtures/files/imported_lettings_logs_missing_answers_examples.csv") }
 
       before do
         create_list(:lettings_log, 11, :completed, age1_known: nil) do |log, i|
           log.old_form_id = "100#{i}"
           log.old_id = "old_id_age1_known_#{i}"
           log.save!
-          expected__answers_examples_content.sub!("{id#{i}}", log.id.to_s)
-          expected__answers_examples_content.sub!("{org_id#{i}}", log.owning_organisation_id.to_s)
+          expected_answers_examples_content.sub!("{id#{i}}", log.id.to_s)
+          expected_answers_examples_content.sub!("{org_id#{i}}", log.owning_organisation_id.to_s)
         end
         create_list(:lettings_log, 4, :completed, beds: nil) do |log, i|
           log.old_form_id = "200#{i}"
           log.old_id = "old_id_beds_#{i}"
-          expected__answers_examples_content.sub!("{id2_#{i}}", log.id.to_s)
-          expected__answers_examples_content.sub!("{org_id2_#{i}}", log.owning_organisation_id.to_s)
+          expected_answers_examples_content.sub!("{id2_#{i}}", log.id.to_s)
+          expected_answers_examples_content.sub!("{org_id2_#{i}}", log.owning_organisation_id.to_s)
           log.save!
         end
         create(:lettings_log, :completed, age1_known: nil, beds: nil, old_form_id: "300", old_id: "123") do |log|
-          expected__answers_examples_content.sub!("{id}", log.id.to_s)
-          expected__answers_examples_content.sub!("{org_id}", log.owning_organisation_id.to_s)
+          expected_answers_examples_content.sub!("{id}", log.id.to_s)
+          expected_answers_examples_content.sub!("{org_id}", log.owning_organisation_id.to_s)
         end
 
         create_list(:lettings_log, 2, :completed, age1_known: nil)
       end
 
       it "generates a csv with expected missing fields" do
-        expect(storage_service).to receive(:write_file).with("MissingAnswersReport_report_suffix.csv", "#{expected_content}")
-        expect(storage_service).to receive(:write_file).with("MissingAnswersExamples_report_suffix.csv", "#{expected__answers_examples_content}")
+        expect(storage_service).to receive(:write_file).with("MissingAnswersReportLettingsLog_report_suffix.csv", "#{expected_content}")
+        expect(storage_service).to receive(:write_file).with("MissingAnswersExamplesLettingsLog_report_suffix.csv", "#{expected_answers_examples_content}")
+        expect(storage_service).to receive(:write_file).with("MissingAnswersReportSalesLog_report_suffix.csv", "\uFEFFMissing answers,Total number of affected logs\n")
+        expect(storage_service).to receive(:write_file).with("MissingAnswersExamplesSalesLog_report_suffix.csv", "\uFEFFMissing answers,Organisation ID,Log ID,Old Form ID\n")
+
+        report_service.generate_missing_answers_report("report_suffix")
+      end
+    end
+
+    context "when there are in progress imported sales logs" do
+      let(:institutions_csv) { nil }
+      let(:expected_content) { File.read("spec/fixtures/files/imported_lettings_logs_missing_answers_report.csv") }
+      let(:expected_answers_examples_content) { File.read("spec/fixtures/files/imported_lettings_logs_missing_answers_examples.csv") }
+
+      before do
+        create_list(:sales_log, 11, :completed, age1_known: nil) do |log, i|
+          log.old_id = "age1_known_#{i}"
+          log.old_form_id = "100#{i}"
+          log.save!
+          expected_answers_examples_content.sub!("{id#{i}}", log.id.to_s)
+          expected_answers_examples_content.sub!("{org_id#{i}}", log.owning_organisation_id.to_s)
+        end
+        create_list(:sales_log, 4, :completed, beds: nil) do |log, i|
+          log.old_id = "beds_#{i}"
+          log.old_form_id = "200#{i}"
+          expected_answers_examples_content.sub!("{id2_#{i}}", log.id.to_s)
+          expected_answers_examples_content.sub!("{org_id2_#{i}}", log.owning_organisation_id.to_s)
+          log.save!
+        end
+        create(:sales_log, :completed, age1_known: nil, beds: nil, old_id: "beds_and_age", old_form_id: "300") do |log|
+          expected_answers_examples_content.sub!("{id}", log.id.to_s)
+          expected_answers_examples_content.sub!("{org_id}", log.owning_organisation_id.to_s)
+        end
+
+        create_list(:sales_log, 2, :completed, age1_known: nil)
+      end
+
+      it "generates a csv with expected missing fields" do
+        expect(storage_service).to receive(:write_file).with("MissingAnswersReportLettingsLog_report_suffix.csv", "\uFEFFMissing answers,Total number of affected logs\n")
+        expect(storage_service).to receive(:write_file).with("MissingAnswersExamplesLettingsLog_report_suffix.csv", "\uFEFFMissing answers,Organisation ID,Log ID,Old Form ID\n")
+        expect(storage_service).to receive(:write_file).with("MissingAnswersReportSalesLog_report_suffix.csv", "#{expected_content}")
+        expect(storage_service).to receive(:write_file).with("MissingAnswersExamplesSalesLog_report_suffix.csv", "#{expected_answers_examples_content}")
 
         report_service.generate_missing_answers_report("report_suffix")
       end

From 961aefa0956f7907430ada8f63e523b91b3f6c3c Mon Sep 17 00:00:00 2001
From: kosiakkatrina <54268893+kosiakkatrina@users.noreply.github.com>
Date: Tue, 12 Sep 2023 10:54:11 +0100
Subject: [PATCH 22/33] =?UTF-8?q?Add=20Children=E2=80=99s=20Social=20Care?=
 =?UTF-8?q?=20to=20all=20referral=20questions=20(#1905)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 app/models/form/lettings/questions/referral_prp.rb       | 3 +++
 .../lettings/questions/referral_supported_housing.rb     | 3 +++
 .../lettings/questions/referral_supported_housing_prp.rb | 1 +
 config/forms/2022_2023.json                              | 9 +++++++++
 4 files changed, 16 insertions(+)

diff --git a/app/models/form/lettings/questions/referral_prp.rb b/app/models/form/lettings/questions/referral_prp.rb
index 36dc247b7..69a42bed5 100644
--- a/app/models/form/lettings/questions/referral_prp.rb
+++ b/app/models/form/lettings/questions/referral_prp.rb
@@ -45,6 +45,9 @@ class Form::Lettings::Questions::ReferralPrp < ::Form::Question
     "13" => {
       "value" => "Youth offending team",
     },
+    "17" => {
+      "value" => "Children’s Social Care",
+    },
     "16" => {
       "value" => "Other",
     },
diff --git a/app/models/form/lettings/questions/referral_supported_housing.rb b/app/models/form/lettings/questions/referral_supported_housing.rb
index 5ec9de70e..68b9fd8e7 100644
--- a/app/models/form/lettings/questions/referral_supported_housing.rb
+++ b/app/models/form/lettings/questions/referral_supported_housing.rb
@@ -45,6 +45,9 @@ class Form::Lettings::Questions::ReferralSupportedHousing < ::Form::Question
     "13" => {
       "value" => "Youth offending team",
     },
+    "17" => {
+      "value" => "Children’s Social Care",
+    },
     "16" => {
       "value" => "Other",
     },
diff --git a/app/models/form/lettings/questions/referral_supported_housing_prp.rb b/app/models/form/lettings/questions/referral_supported_housing_prp.rb
index 4824dd6c5..32fb474ed 100644
--- a/app/models/form/lettings/questions/referral_supported_housing_prp.rb
+++ b/app/models/form/lettings/questions/referral_supported_housing_prp.rb
@@ -24,6 +24,7 @@ class Form::Lettings::Questions::ReferralSupportedHousingPrp < ::Form::Question
     "12" => { "value" => "Police, probation or prison" },
     "7" => { "value" => "Voluntary agency" },
     "13" => { "value" => "Youth offending team" },
+    "17" => { "value" => "Children’s Social Care" },
     "16" => { "value" => "Other" },
   }.freeze
 end
diff --git a/config/forms/2022_2023.json b/config/forms/2022_2023.json
index db11b0950..66b975d9c 100644
--- a/config/forms/2022_2023.json
+++ b/config/forms/2022_2023.json
@@ -7081,6 +7081,9 @@
                     "13": {
                       "value": "Youth offending team"
                     },
+                    "17": {
+                      "value": "Children’s Social Care"
+                    },
                     "16": {
                       "value": "Other"
                     }
@@ -7138,6 +7141,9 @@
                     "13": {
                       "value": "Youth offending team"
                     },
+                    "17": {
+                      "value": "Children’s Social Care"
+                    },
                     "16": {
                       "value": "Other"
                     }
@@ -7198,6 +7204,9 @@
                     "13": {
                       "value": "Youth offending team"
                     },
+                    "17": {
+                      "value": "Children’s Social Care"
+                    },
                     "16": {
                       "value": "Other"
                     }

From 564fa9ebb50f627a77a5db97f49c494f71b791ee Mon Sep 17 00:00:00 2001
From: kosiakkatrina <54268893+kosiakkatrina@users.noreply.github.com>
Date: Tue, 12 Sep 2023 11:15:07 +0100
Subject: [PATCH 23/33] Mark hodate value check confirmed (#1906)

* Mark hodate value check confirmed

* Add other missing value_checks for sales
---
 .../imports/sales_logs_import_service.rb      |  6 +++
 .../imports/sales_logs_import_service_spec.rb | 50 +++++++++++++++++++
 2 files changed, 56 insertions(+)

diff --git a/app/services/imports/sales_logs_import_service.rb b/app/services/imports/sales_logs_import_service.rb
index 157e797b4..68d8e95ce 100644
--- a/app/services/imports/sales_logs_import_service.rb
+++ b/app/services/imports/sales_logs_import_service.rb
@@ -152,6 +152,10 @@ module Imports
       attributes["discounted_sale_value_check"] = 0
       attributes["buyer_livein_value_check"] = 0
       attributes["percentage_discount_value_check"] = 0
+      attributes["hodate_check"] = 0
+      attributes["saledate_check"] = 0
+      attributes["combined_income_value_check"] = 0
+      attributes["stairowned_value_check"] = 0
 
       # 2023/34 attributes
       attributes["address_line1"] = string_or_nil(xml_doc, "AddressLine1")
@@ -330,6 +334,8 @@ module Imports
          discounted_sale_value_check
          buyer_livein_value_check
          percentage_discount_value_check
+         combined_income_value_check
+         stairowned_value_check
          uprn_known
          uprn_confirmed]
     end
diff --git a/spec/services/imports/sales_logs_import_service_spec.rb b/spec/services/imports/sales_logs_import_service_spec.rb
index df279e5d5..2357618d0 100644
--- a/spec/services/imports/sales_logs_import_service_spec.rb
+++ b/spec/services/imports/sales_logs_import_service_spec.rb
@@ -595,6 +595,56 @@ RSpec.describe Imports::SalesLogsImportService do
       end
     end
 
+    context "and the hodate soft validation is triggered (hodate_value_check)" do
+      let(:sales_log_id) { "shared_ownership_sales_log" }
+
+      before do
+        sales_log_xml.at_xpath("//xmlns:HODAY").content = "1"
+        sales_log_xml.at_xpath("//xmlns:HOMONTH").content = "1"
+        sales_log_xml.at_xpath("//xmlns:HOYEAR").content = "2018"
+      end
+
+      it "completes the log" do
+        sales_log_service.send(:create_log, sales_log_xml)
+        sales_log = SalesLog.find_by(old_id: sales_log_id)
+        expect(sales_log.status).to eq("completed")
+      end
+    end
+
+    context "and the combined income soft validation is triggered (combined_income_value_check)" do
+      let(:sales_log_id) { "shared_ownership_sales_log" }
+
+      before do
+        sales_log_xml.at_xpath("//xmlns:joint").content = "1 Yes"
+        sales_log_xml.at_xpath("//xmlns:Q2Person1Income").content = "45000"
+        sales_log_xml.at_xpath("//xmlns:P2IncKnown").content = "1 Yes"
+        sales_log_xml.at_xpath("//xmlns:Q2Person2Income").content = "45000"
+      end
+
+      it "completes the log" do
+        sales_log_service.send(:create_log, sales_log_xml)
+        sales_log = SalesLog.find_by(old_id: sales_log_id)
+        expect(sales_log.status).to eq("completed")
+      end
+    end
+
+    context "and the stairowned soft validation is triggered (stairowned_value_check)" do
+      let(:sales_log_id) { "shared_ownership_sales_log" }
+
+      before do
+        sales_log_xml.at_xpath("//xmlns:DerSaleType").content = "24"
+        sales_log_xml.at_xpath("//xmlns:PercentOwns").content = "77"
+        sales_log_xml.at_xpath("//xmlns:Q17aStaircase").content = "1"
+        sales_log_xml.at_xpath("//xmlns:PercentBought").content = "10"
+      end
+
+      it "completes the log" do
+        sales_log_service.send(:create_log, sales_log_xml)
+        sales_log = SalesLog.find_by(old_id: sales_log_id)
+        expect(sales_log.status).to eq("completed")
+      end
+    end
+
     context "and it has an invalid record with invalid child age" do
       let(:sales_log_id) { "discounted_ownership_sales_log" }
 

From 213a4bf2290a6ec2d0cdd587e16d4f6ba816ec6e Mon Sep 17 00:00:00 2001
From: natdeanlewissoftwire
 <94526761+natdeanlewissoftwire@users.noreply.github.com>
Date: Tue, 12 Sep 2023 13:45:01 +0100
Subject: [PATCH 24/33] Blank invalid combined income and proplen values on
 migration (#1907)

* feat: clear invalid combine incomes

* feat: clear out of range proplen values
---
 .../sales/financial_validations.rb            |  4 +-
 .../imports/sales_logs_import_service.rb      |  5 ++
 .../imports/sales_logs_import_service_spec.rb | 81 ++++++++++++++++++-
 3 files changed, 87 insertions(+), 3 deletions(-)

diff --git a/app/models/validations/sales/financial_validations.rb b/app/models/validations/sales/financial_validations.rb
index ffdee561a..82c57e75d 100644
--- a/app/models/validations/sales/financial_validations.rb
+++ b/app/models/validations/sales/financial_validations.rb
@@ -30,9 +30,9 @@ module Validations::Sales::FinancialValidations
     combined_income = record.income1 + record.income2
     relevant_fields = %i[income1 income2 ownershipsch uprn la postcode_full]
     if record.london_property? && combined_income > 90_000
-      relevant_fields.each { |field| record.errors.add field, I18n.t("validations.financial.income.combined_over_hard_max_for_london") }
+      relevant_fields.each { |field| record.errors.add field, :over_combined_hard_max_for_london, message: I18n.t("validations.financial.income.combined_over_hard_max_for_london") }
     elsif record.property_not_in_london? && combined_income > 80_000
-      relevant_fields.each { |field| record.errors.add field, I18n.t("validations.financial.income.combined_over_hard_max_for_outside_london") }
+      relevant_fields.each { |field| record.errors.add field, :over_combined_hard_max_for_outside_london, message: I18n.t("validations.financial.income.combined_over_hard_max_for_outside_london") }
     end
   end
 
diff --git a/app/services/imports/sales_logs_import_service.rb b/app/services/imports/sales_logs_import_service.rb
index 68d8e95ce..1a9fe54a0 100644
--- a/app/services/imports/sales_logs_import_service.rb
+++ b/app/services/imports/sales_logs_import_service.rb
@@ -250,15 +250,20 @@ module Imports
         %i[postcode_full postcodes_not_matching] => %w[ppcodenk ppostcode_full],
         %i[exdate over_a_year_from_saledate] => %w[exdate],
         %i[income1 over_hard_max_for_outside_london] => %w[income1],
+        %i[income1 over_combined_hard_max_for_outside_london] => %w[income1 income2],
         %i[income1 over_hard_max_for_london] => %w[income1],
+        %i[income1 over_combined_hard_max_for_london] => %w[income1 income2],
         %i[income2 over_hard_max_for_outside_london] => %w[income2],
+        %i[income2 over_combined_hard_max_for_outside_london] => %w[income1 income2],
         %i[income2 over_hard_max_for_london] => %w[income2],
+        %i[income2 over_combined_hard_max_for_london] => %w[income1 income2],
         %i[equity over_max] => %w[equity],
         %i[equity under_min] => %w[equity],
         %i[mscharge under_min] => %w[mscharge has_mscharge],
         %i[mortgage cannot_be_0] => %w[mortgage],
         %i[frombeds outside_the_range] => %w[frombeds],
         %i[age1 outside_the_range] => %w[age1 age1_known],
+        %i[proplen outside_the_range] => %w[proplen],
       }
 
       errors.each do |(error, fields)|
diff --git a/spec/services/imports/sales_logs_import_service_spec.rb b/spec/services/imports/sales_logs_import_service_spec.rb
index 2357618d0..e4d5b9473 100644
--- a/spec/services/imports/sales_logs_import_service_spec.rb
+++ b/spec/services/imports/sales_logs_import_service_spec.rb
@@ -900,7 +900,7 @@ RSpec.describe Imports::SalesLogsImportService do
           .not_to raise_error
       end
 
-      it "clears out the referral answer" do
+      it "clears out the mortgage answer" do
         allow(logger).to receive(:warn)
 
         sales_log_service.send(:create_log, sales_log_xml)
@@ -911,6 +911,29 @@ RSpec.describe Imports::SalesLogsImportService do
       end
     end
 
+    context "when proplen is out of range" do
+      let(:sales_log_id) { "shared_ownership_sales_log" }
+
+      before do
+        sales_log_xml.at_xpath("//xmlns:Q16aProplen2").content = "2000"
+      end
+
+      it "intercepts the relevant validation error" do
+        expect { sales_log_service.send(:create_log, sales_log_xml) }
+          .not_to raise_error
+      end
+
+      it "clears out the proplen answer" do
+        allow(logger).to receive(:warn)
+
+        sales_log_service.send(:create_log, sales_log_xml)
+        sales_log = SalesLog.find_by(old_id: sales_log_id)
+
+        expect(sales_log).not_to be_nil
+        expect(sales_log.proplen).to be_nil
+      end
+    end
+
     context "and it has an invalid income 1" do
       let(:sales_log_id) { "shared_ownership_sales_log" }
 
@@ -961,6 +984,34 @@ RSpec.describe Imports::SalesLogsImportService do
       end
     end
 
+    context "and it has an invalid combined income outside london" do
+      let(:sales_log_id) { "shared_ownership_sales_log" }
+
+      before do
+        sales_log_xml.at_xpath("//xmlns:joint").content = "1 Yes"
+        sales_log_xml.at_xpath("//xmlns:Q2Person1Income").content = "45000"
+        sales_log_xml.at_xpath("//xmlns:P2IncKnown").content = "1 Yes"
+        sales_log_xml.at_xpath("//xmlns:Q2Person2Income").content = "40000"
+        sales_log_xml.at_xpath("//xmlns:Q14ONSLACode").content = "E07000223"
+      end
+
+      it "intercepts the relevant validation error" do
+        expect { sales_log_service.send(:create_log, sales_log_xml) }
+          .not_to raise_error
+      end
+
+      it "clears out the invalid answers" do
+        allow(logger).to receive(:warn)
+
+        sales_log_service.send(:create_log, sales_log_xml)
+        sales_log = SalesLog.find_by(old_id: sales_log_id)
+
+        expect(sales_log).not_to be_nil
+        expect(sales_log.income1).to be_nil
+        expect(sales_log.income2).to be_nil
+      end
+    end
+
     context "and it has an invalid income 1 for london" do
       let(:sales_log_id) { "shared_ownership_sales_log" }
 
@@ -1011,6 +1062,34 @@ RSpec.describe Imports::SalesLogsImportService do
       end
     end
 
+    context "and it has an invalid combined income for london" do
+      let(:sales_log_id) { "shared_ownership_sales_log" }
+
+      before do
+        sales_log_xml.at_xpath("//xmlns:joint").content = "1 Yes"
+        sales_log_xml.at_xpath("//xmlns:Q2Person1Income").content = "50000"
+        sales_log_xml.at_xpath("//xmlns:P2IncKnown").content = "1 Yes"
+        sales_log_xml.at_xpath("//xmlns:Q2Person2Income").content = "45000"
+        sales_log_xml.at_xpath("//xmlns:Q14ONSLACode").content = "E09000012"
+      end
+
+      it "intercepts the relevant validation error" do
+        expect { sales_log_service.send(:create_log, sales_log_xml) }
+          .not_to raise_error
+      end
+
+      it "clears out the invalid answers" do
+        allow(logger).to receive(:warn)
+
+        sales_log_service.send(:create_log, sales_log_xml)
+        sales_log = SalesLog.find_by(old_id: sales_log_id)
+
+        expect(sales_log).not_to be_nil
+        expect(sales_log.income1).to be_nil
+        expect(sales_log.income2).to be_nil
+      end
+    end
+
     context "and it has an invalid mscharge" do
       let(:sales_log_id) { "shared_ownership_sales_log" }
 

From c80565a54d7baf7c0d251f85b63319020cf06c56 Mon Sep 17 00:00:00 2001
From: natdeanlewissoftwire
 <94526761+natdeanlewissoftwire@users.noreply.github.com>
Date: Tue, 12 Sep 2023 13:45:20 +0100
Subject: [PATCH 25/33] Make saledate nil safe(r) (#1904)

* feat: allow missing day/month/year values if CompletionDate exists as we don't always receive these from old core

* feat: make safe string as decimal nil safer too

* feat: add test
---
 app/services/imports/logs_import_service.rb   |  2 +-
 .../imports/sales_logs_import_service.rb      |  4 +--
 .../lettings_logs_import_service_spec.rb      | 21 ++++++++++++++++
 .../imports/sales_logs_import_service_spec.rb | 25 +++++++++++++++++++
 4 files changed, 49 insertions(+), 3 deletions(-)

diff --git a/app/services/imports/logs_import_service.rb b/app/services/imports/logs_import_service.rb
index a82208bbe..f7456a3fb 100644
--- a/app/services/imports/logs_import_service.rb
+++ b/app/services/imports/logs_import_service.rb
@@ -72,7 +72,7 @@ module Imports
       if str.nil?
         nil
       else
-        BigDecimal(str, exception: false).round(2)
+        BigDecimal(str, exception: false)&.round(2)
       end
     end
 
diff --git a/app/services/imports/sales_logs_import_service.rb b/app/services/imports/sales_logs_import_service.rb
index 1a9fe54a0..7d10f52ca 100644
--- a/app/services/imports/sales_logs_import_service.rb
+++ b/app/services/imports/sales_logs_import_service.rb
@@ -15,7 +15,7 @@ module Imports
     def create_log(xml_doc)
       # only import sales logs from 22/23 collection period onwards
       return unless meta_field_value(xml_doc, "form-name").include?("Sales")
-      return unless compose_date(xml_doc, "DAY", "MONTH", "YEAR") >= Time.zone.local(2022, 4, 1)
+      return unless (compose_date(xml_doc, "DAY", "MONTH", "YEAR") || Time.zone.parse(field_value(xml_doc, "xmlns", "CompletionDate"))) >= Time.zone.local(2022, 4, 1)
 
       attributes = {}
 
@@ -24,7 +24,7 @@ module Imports
       # Required fields for status complete or logic to work
       # Note: order matters when we derive from previous values (attributes parameter)
 
-      attributes["saledate"] = compose_date(xml_doc, "DAY", "MONTH", "YEAR")
+      attributes["saledate"] = compose_date(xml_doc, "DAY", "MONTH", "YEAR") || Time.zone.parse(field_value(xml_doc, "xmlns", "CompletionDate"))
       attributes["owning_organisation_id"] = find_organisation_id(xml_doc, "OWNINGORGID")
       attributes["type"] = unsafe_string_as_integer(xml_doc, "DerSaleType")
       attributes["old_id"] = meta_field_value(xml_doc, "document-id")
diff --git a/spec/services/imports/lettings_logs_import_service_spec.rb b/spec/services/imports/lettings_logs_import_service_spec.rb
index 800955cfc..dad2e68ed 100644
--- a/spec/services/imports/lettings_logs_import_service_spec.rb
+++ b/spec/services/imports/lettings_logs_import_service_spec.rb
@@ -566,6 +566,27 @@ RSpec.describe Imports::LettingsLogsImportService do
         end
       end
 
+      context "and the number of times the property was relet is a non nil string that is nil as a BigDecimal" do
+        before do
+          lettings_log_xml.at_xpath("//xmlns:Q20").content = "a"
+        end
+
+        it "doesn't throw an error" do
+          expect { lettings_log_service.send(:create_log, lettings_log_xml) }
+            .not_to raise_error
+        end
+
+        it "clears out the number offered answer" do
+          allow(logger).to receive(:warn)
+
+          lettings_log_service.send(:create_log, lettings_log_xml)
+          lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
+
+          expect(lettings_log).not_to be_nil
+          expect(lettings_log.offered).to be_nil
+        end
+      end
+
       context "and the gender identity is refused" do
         before do
           lettings_log_xml.at_xpath("//xmlns:P1Sex").content = "Person prefers not to say"
diff --git a/spec/services/imports/sales_logs_import_service_spec.rb b/spec/services/imports/sales_logs_import_service_spec.rb
index e4d5b9473..6d067dbd4 100644
--- a/spec/services/imports/sales_logs_import_service_spec.rb
+++ b/spec/services/imports/sales_logs_import_service_spec.rb
@@ -228,6 +228,31 @@ RSpec.describe Imports::SalesLogsImportService do
       end
     end
 
+    context "and the log startdate is only present in the CompletionDate field" do
+      let(:sales_log_id) { "shared_ownership_sales_log" }
+
+      before do
+        sales_log_xml.at_xpath("//xmlns:DAY").content = nil
+        sales_log_xml.at_xpath("//xmlns:MONTH").content = nil
+        sales_log_xml.at_xpath("//xmlns:YEAR").content = nil
+        sales_log_xml.at_xpath("//xmlns:CompletionDate").content = "2022-10-9"
+        sales_log_xml.at_xpath("//xmlns:HODAY").content = 9
+        sales_log_xml.at_xpath("//xmlns:HOMONTH").content = 10
+        sales_log_xml.at_xpath("//xmlns:HOYEAR").content = 2022
+        sales_log_xml.at_xpath("//xmlns:EXDAY").content = 9
+        sales_log_xml.at_xpath("//xmlns:EXMONTH").content = 10
+        sales_log_xml.at_xpath("//xmlns:EXYEAR").content = 2022
+      end
+
+      it "creates the log with the correct saledate" do
+        expect(logger).not_to receive(:error)
+        expect(logger).not_to receive(:warn)
+        expect { sales_log_service.send(:create_log, sales_log_xml) }
+          .to change(SalesLog, :count).by(1)
+        expect(SalesLog.last.saledate).to eq(Time.zone.local(2022, 10, 9))
+      end
+    end
+
     context "when the log is valid" do
       let(:sales_log_id) { "shared_ownership_sales_log" }
 

From 077390bb5b5001ab09d6c949b37ed9f99506b144 Mon Sep 17 00:00:00 2001
From: kosiakkatrina <54268893+kosiakkatrina@users.noreply.github.com>
Date: Tue, 12 Sep 2023 15:15:56 +0100
Subject: [PATCH 26/33] CLDC-2699 Update values updated at when updating reason
 field (#1909)

* Update values updated at

* Update tests
---
 .../imports/lettings_logs_field_import_service.rb      |  2 +-
 .../imports/lettings_logs_field_import_service_spec.rb | 10 +++++++---
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/app/services/imports/lettings_logs_field_import_service.rb b/app/services/imports/lettings_logs_field_import_service.rb
index 68c3ba5f4..8b4124796 100644
--- a/app/services/imports/lettings_logs_field_import_service.rb
+++ b/app/services/imports/lettings_logs_field_import_service.rb
@@ -194,7 +194,7 @@ module Imports
           if reason == 20 && reasonother.blank?
             @logger.info("lettings log #{record.id}'s reason is other but other reason is not provided, skipping update")
           else
-            record.update!(reason:, reasonother:)
+            record.update!(reason:, reasonother:, values_updated_at: Time.zone.now)
             @logger.info("lettings log #{record.id}'s reason value has been set to #{reason}")
             @logger.info("lettings log #{record.id}'s reasonother value has been set to #{reasonother}") if record.reasonother.present?
           end
diff --git a/spec/services/imports/lettings_logs_field_import_service_spec.rb b/spec/services/imports/lettings_logs_field_import_service_spec.rb
index 4bed2dc63..ac35169a5 100644
--- a/spec/services/imports/lettings_logs_field_import_service_spec.rb
+++ b/spec/services/imports/lettings_logs_field_import_service_spec.rb
@@ -481,7 +481,7 @@ RSpec.describe Imports::LettingsLogsFieldImportService do
       before do
         Imports::LettingsLogsImportService.new(storage_service, logger).create_logs(fixture_directory)
         lettings_log_file.rewind
-        lettings_log.update!(reason: nil)
+        lettings_log.update!(reason: nil, values_updated_at: nil)
         lettings_log_xml.at_xpath("//xmlns:Q9a").content = "47"
       end
 
@@ -489,6 +489,7 @@ RSpec.describe Imports::LettingsLogsFieldImportService do
         expect(logger).to receive(:info).with(/lettings log \d+'s reason value has been set to 47/)
         expect { import_service.send(:update_reason, lettings_log_xml) }
           .to(change { lettings_log.reload.reason }.from(nil).to(47))
+        expect(lettings_log.values_updated_at).not_to be_nil
       end
     end
 
@@ -498,7 +499,7 @@ RSpec.describe Imports::LettingsLogsFieldImportService do
       before do
         Imports::LettingsLogsImportService.new(storage_service, logger).create_logs(fixture_directory)
         lettings_log_file.rewind
-        lettings_log.update!(reason: 18)
+        lettings_log.update!(reason: 18, values_updated_at: nil)
         lettings_log_xml.at_xpath("//xmlns:Q9a").content = "47"
       end
 
@@ -506,6 +507,7 @@ RSpec.describe Imports::LettingsLogsFieldImportService do
         expect(logger).to receive(:info).with(/lettings log \d+ has a value for reason, skipping update/)
         expect { import_service.send(:update_reason, lettings_log_xml) }
           .not_to(change { lettings_log.reload.reason })
+        expect(lettings_log.values_updated_at).to be_nil
       end
     end
 
@@ -515,7 +517,7 @@ RSpec.describe Imports::LettingsLogsFieldImportService do
       before do
         Imports::LettingsLogsImportService.new(storage_service, logger).create_logs(fixture_directory)
         lettings_log_file.rewind
-        lettings_log.update!(reason: nil)
+        lettings_log.update!(reason: nil, values_updated_at: nil)
         lettings_log_xml.at_xpath("//xmlns:Q9a").content = "20"
       end
 
@@ -529,6 +531,7 @@ RSpec.describe Imports::LettingsLogsFieldImportService do
           expect(logger).to receive(:info).with(/lettings log \d+'s reasonother value has been set to other/)
           expect { import_service.send(:update_reason, lettings_log_xml) }
             .to(change { lettings_log.reload.reason }.from(nil).to(20))
+          expect(lettings_log.values_updated_at).not_to be_nil
         end
       end
 
@@ -537,6 +540,7 @@ RSpec.describe Imports::LettingsLogsFieldImportService do
           expect(logger).to receive(:info).with(/lettings log \d+'s reason is other but other reason is not provided, skipping update/)
           expect { import_service.send(:update_reason, lettings_log_xml) }
             .not_to(change { lettings_log.reload.reason })
+          expect(lettings_log.values_updated_at).to be_nil
         end
       end
     end

From a1cc9d78636707c7f1fc5571bbf084946a918fcf Mon Sep 17 00:00:00 2001
From: kosiakkatrina <54268893+kosiakkatrina@users.noreply.github.com>
Date: Tue, 12 Sep 2023 16:29:03 +0100
Subject: [PATCH 27/33] Move person details if some persons are skipped (#1903)

* Move person details if some persons are skipped

* Check that undesired person details don't get saved

* Make sure we don't repeat people details
---
 .../imports/lettings_logs_import_service.rb   |  28 ++-
 .../lettings_logs_import_service_spec.rb      | 220 ++++++++++++++++++
 2 files changed, 241 insertions(+), 7 deletions(-)

diff --git a/app/services/imports/lettings_logs_import_service.rb b/app/services/imports/lettings_logs_import_service.rb
index ca3cde434..d40df0fc1 100644
--- a/app/services/imports/lettings_logs_import_service.rb
+++ b/app/services/imports/lettings_logs_import_service.rb
@@ -76,14 +76,19 @@ module Imports
       attributes["irproduct_other"] = string_or_nil(xml_doc, "IRProductOther")
       attributes["rent_type"] = rent_type(xml_doc, attributes["lar"], attributes["irproduct"])
       attributes["hhmemb"] = household_members(xml_doc, previous_status)
+
+      people_indexes = people_with_details_ids(xml_doc)
+      available_people_indexes = people_indexes + (people_indexes.max + 1..8).to_a
       (1..8).each do |index|
-        attributes["age#{index}"] = safe_string_as_integer(xml_doc, "P#{index}Age")
-        attributes["age#{index}_known"] = age_known(xml_doc, index, attributes["hhmemb"])
-        attributes["sex#{index}"] = sex(xml_doc, index)
-        attributes["ecstat#{index}"] = unsafe_string_as_integer(xml_doc, "P#{index}Eco")
+        person_index = available_people_indexes[index - 1]
+        attributes["age#{index}"] = safe_string_as_integer(xml_doc, "P#{person_index}Age")
+        attributes["age#{index}_known"] = age_known(xml_doc, index, person_index, attributes["hhmemb"])
+        attributes["sex#{index}"] = sex(xml_doc, person_index)
+        attributes["ecstat#{index}"] = unsafe_string_as_integer(xml_doc, "P#{person_index}Eco")
       end
       (2..8).each do |index|
-        attributes["relat#{index}"] = relat(xml_doc, index)
+        person_index = available_people_indexes[index - 1]
+        attributes["relat#{index}"] = relat(xml_doc, person_index)
         attributes["details_known_#{index}"] = details_known(index, attributes)
 
         # Trips validation
@@ -449,10 +454,10 @@ module Imports
       end
     end
 
-    def age_known(xml_doc, index, hhmemb)
+    def age_known(xml_doc, index, person_index, hhmemb)
       return nil if hhmemb.present? && index > hhmemb
 
-      age_refused = string_or_nil(xml_doc, "P#{index}AR")
+      age_refused = string_or_nil(xml_doc, "P#{person_index}AR")
       if age_refused.present?
         if age_refused.casecmp?("AGE_REFUSED") || age_refused.casecmp?("No")
           return 1 # No
@@ -577,6 +582,15 @@ module Imports
       ((2..8).map { |x| string_or_nil(xml_doc, "P#{x}Rel") } + [string_or_nil(xml_doc, "P1Sex")]).compact
     end
 
+    def people_with_details_ids(xml_doc)
+      [1] + (2..8).select do |x|
+        string_or_nil(xml_doc, "P#{x}Rel").present? ||
+          string_or_nil(xml_doc, "P#{x}Sex").present? ||
+          string_or_nil(xml_doc, "P#{x}Age").present? ||
+          string_or_nil(xml_doc, "P#{x}Eco").present?
+      end
+    end
+
     def tshortfall_known?(xml_doc, attributes)
       if attributes["tshortfall"].blank? && attributes["hbrentshortfall"] == 1 && overridden?(xml_doc, "xmlns", "Q18dyes")
         1
diff --git a/spec/services/imports/lettings_logs_import_service_spec.rb b/spec/services/imports/lettings_logs_import_service_spec.rb
index dad2e68ed..efef0997a 100644
--- a/spec/services/imports/lettings_logs_import_service_spec.rb
+++ b/spec/services/imports/lettings_logs_import_service_spec.rb
@@ -1303,6 +1303,226 @@ RSpec.describe Imports::LettingsLogsImportService do
           expect(lettings_log.status).to eq("completed")
         end
       end
+
+      context "and there are several household members" do
+        context "and one person details are skipped" do
+          before do
+            lettings_log_xml.at_xpath("//xmlns:HHMEMB").content = 3
+            lettings_log_xml.at_xpath("//xmlns:P2AR").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Age").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Sex").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Rel").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Eco").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P3Age").content = 7
+            lettings_log_xml.at_xpath("//xmlns:P3Sex").content = "Male"
+            lettings_log_xml.at_xpath("//xmlns:P3Rel").content = "Child"
+            lettings_log_xml.at_xpath("//xmlns:P3Eco").content = "9) Child under 16"
+          end
+
+          it "correctly moves person details" do
+            lettings_log_service.send(:create_log, lettings_log_xml)
+
+            lettings_log = LettingsLog.where(old_id: lettings_log_id).first
+            expect(lettings_log&.hhmemb).to eq(3)
+            expect(lettings_log&.details_known_2).to eq(0)
+            expect(lettings_log&.age2_known).to eq(0)
+            expect(lettings_log&.age2).to eq(7)
+            expect(lettings_log&.sex2).to eq("M")
+            expect(lettings_log&.relat2).to eq("C")
+
+            expect(lettings_log&.details_known_3).to eq(0)
+            expect(lettings_log&.age3_known).to eq(0)
+            expect(lettings_log&.age3).to eq(nil)
+            expect(lettings_log&.sex3).to eq(nil)
+            expect(lettings_log&.relat3).to eq(nil)
+
+            [4, 5].each do |i|
+              expect(lettings_log&.send("details_known_#{i}")).to eq(nil)
+              expect(lettings_log&.send("age#{i}_known")).to eq(nil)
+              expect(lettings_log&.send("age#{i}")).to eq(nil)
+              expect(lettings_log&.send("sex#{i}")).to eq(nil)
+              expect(lettings_log&.send("relat#{i}")).to eq(nil)
+              expect(lettings_log&.send("ecstat#{i}")).to eq(nil)
+            end
+          end
+        end
+
+        context "and several consecutive person details are skipped" do
+          before do
+            lettings_log_xml.at_xpath("//xmlns:HHMEMB").content = 4
+
+            lettings_log_xml.at_xpath("//xmlns:P2AR").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Age").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Sex").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Rel").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Eco").content = nil
+
+            lettings_log_xml.at_xpath("//xmlns:P3AR").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P3Age").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P3Sex").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P3Rel").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P3Eco").content = nil
+
+            lettings_log_xml.at_xpath("//xmlns:P4Age").content = 7
+            lettings_log_xml.at_xpath("//xmlns:P4Sex").content = "Male"
+            lettings_log_xml.at_xpath("//xmlns:P4Rel").content = "Child"
+            lettings_log_xml.at_xpath("//xmlns:P4Eco").content = "9) Child under 16"
+
+            lettings_log_xml.at_xpath("//xmlns:P5Age").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P5Sex").content = "Male"
+            lettings_log_xml.at_xpath("//xmlns:P5Rel").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P5Eco").content = nil
+
+            lettings_log_xml.at_xpath("//xmlns:P6Age").content = 12
+            lettings_log_xml.at_xpath("//xmlns:P6Sex").content = "Female"
+            lettings_log_xml.at_xpath("//xmlns:P6Rel").content = "Child"
+            lettings_log_xml.at_xpath("//xmlns:P6Eco").content = "9) Child under 16"
+          end
+
+          it "correctly moves person details" do
+            lettings_log_service.send(:create_log, lettings_log_xml)
+
+            lettings_log = LettingsLog.where(old_id: lettings_log_id).first
+            expect(lettings_log&.hhmemb).to eq(4)
+            expect(lettings_log&.details_known_2).to eq(0)
+            expect(lettings_log&.age2_known).to eq(0)
+            expect(lettings_log&.age2).to eq(7)
+            expect(lettings_log&.sex2).to eq("M")
+            expect(lettings_log&.relat2).to eq("C")
+
+            expect(lettings_log&.details_known_3).to eq(0)
+            expect(lettings_log&.age3_known).to eq(0)
+            expect(lettings_log&.age3).to eq(nil)
+            expect(lettings_log&.sex3).to eq("M")
+            expect(lettings_log&.relat3).to eq(nil)
+
+            expect(lettings_log&.details_known_4).to eq(0)
+            expect(lettings_log&.age4_known).to eq(0)
+            expect(lettings_log&.age4).to eq(12)
+            expect(lettings_log&.sex4).to eq("F")
+            expect(lettings_log&.relat4).to eq("C")
+
+            [5, 6, 7, 8].each do |i|
+              expect(lettings_log&.send("details_known_#{i}")).to eq(nil)
+              expect(lettings_log&.send("age#{i}_known")).to eq(nil)
+              expect(lettings_log&.send("age#{i}")).to eq(nil)
+              expect(lettings_log&.send("sex#{i}")).to eq(nil)
+              expect(lettings_log&.send("relat#{i}")).to eq(nil)
+              expect(lettings_log&.send("ecstat#{i}")).to eq(nil)
+            end
+          end
+        end
+
+        context "and several non consecutive person details are skipped" do
+          before do
+            lettings_log_xml.at_xpath("//xmlns:HHMEMB").content = 4
+            lettings_log_xml.at_xpath("//xmlns:P2AR").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Age").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Sex").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Rel").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Eco").content = nil
+
+            lettings_log_xml.at_xpath("//xmlns:P3Age").content = 7
+            lettings_log_xml.at_xpath("//xmlns:P3Sex").content = "Male"
+            lettings_log_xml.at_xpath("//xmlns:P3Rel").content = "Child"
+            lettings_log_xml.at_xpath("//xmlns:P3Eco").content = "9) Child under 16"
+
+            lettings_log_xml.at_xpath("//xmlns:P4AR").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P4Age").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P4Sex").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P4Rel").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P4Eco").content = nil
+
+            lettings_log_xml.at_xpath("//xmlns:P5Age").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P5Sex").content = "Male"
+            lettings_log_xml.at_xpath("//xmlns:P5Rel").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P5Eco").content = nil
+
+            lettings_log_xml.at_xpath("//xmlns:P6Age").content = 12
+            lettings_log_xml.at_xpath("//xmlns:P6Sex").content = "Female"
+            lettings_log_xml.at_xpath("//xmlns:P6Rel").content = "Child"
+            lettings_log_xml.at_xpath("//xmlns:P6Eco").content = "9) Child under 16"
+          end
+
+          it "correctly moves person details" do
+            lettings_log_service.send(:create_log, lettings_log_xml)
+
+            lettings_log = LettingsLog.where(old_id: lettings_log_id).first
+            expect(lettings_log&.hhmemb).to eq(4)
+            expect(lettings_log&.details_known_2).to eq(0)
+            expect(lettings_log&.age2_known).to eq(0)
+            expect(lettings_log&.age2).to eq(7)
+            expect(lettings_log&.sex2).to eq("M")
+            expect(lettings_log&.relat2).to eq("C")
+
+            expect(lettings_log&.details_known_3).to eq(0)
+            expect(lettings_log&.age3_known).to eq(0)
+            expect(lettings_log&.age3).to eq(nil)
+            expect(lettings_log&.sex3).to eq("M")
+            expect(lettings_log&.relat3).to eq(nil)
+
+            expect(lettings_log&.details_known_4).to eq(0)
+            expect(lettings_log&.age4_known).to eq(0)
+            expect(lettings_log&.age4).to eq(12)
+            expect(lettings_log&.sex4).to eq("F")
+            expect(lettings_log&.relat4).to eq("C")
+
+            [5, 6, 7, 8].each do |i|
+              expect(lettings_log&.send("details_known_#{i}")).to eq(nil)
+              expect(lettings_log&.send("age#{i}_known")).to eq(nil)
+              expect(lettings_log&.send("age#{i}")).to eq(nil)
+              expect(lettings_log&.send("sex#{i}")).to eq(nil)
+              expect(lettings_log&.send("relat#{i}")).to eq(nil)
+              expect(lettings_log&.send("ecstat#{i}")).to eq(nil)
+            end
+          end
+        end
+
+        context "with 3 houusehold members without any person data" do
+          before do
+            lettings_log_xml.at_xpath("//xmlns:HHMEMB").content = 3
+            lettings_log_xml.at_xpath("//xmlns:P2AR").content = "No"
+            lettings_log_xml.at_xpath("//xmlns:P2Age").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Sex").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Rel").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P2Eco").content = nil
+
+            lettings_log_xml.at_xpath("//xmlns:P3AR").content = "No"
+            lettings_log_xml.at_xpath("//xmlns:P3Age").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P3Sex").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P3Rel").content = nil
+            lettings_log_xml.at_xpath("//xmlns:P3Eco").content = nil
+          end
+
+          it "correctly sets person details" do
+            lettings_log_service.send(:create_log, lettings_log_xml)
+
+            lettings_log = LettingsLog.where(old_id: lettings_log_id).first
+            expect(lettings_log&.hhmemb).to eq(3)
+
+            expect(lettings_log&.details_known_2).to eq(0)
+            expect(lettings_log&.age2_known).to eq(1)
+            expect(lettings_log&.age2).to eq(nil)
+            expect(lettings_log&.sex2).to eq(nil)
+            expect(lettings_log&.relat2).to eq(nil)
+
+            expect(lettings_log&.details_known_3).to eq(0)
+            expect(lettings_log&.age3_known).to eq(1)
+            expect(lettings_log&.age3).to eq(nil)
+            expect(lettings_log&.sex3).to eq(nil)
+            expect(lettings_log&.relat3).to eq(nil)
+
+            [4, 5, 6, 7, 8].each do |i|
+              expect(lettings_log&.send("details_known_#{i}")).to eq(nil)
+              expect(lettings_log&.send("age#{i}_known")).to eq(nil)
+              expect(lettings_log&.send("age#{i}")).to eq(nil)
+              expect(lettings_log&.send("sex#{i}")).to eq(nil)
+              expect(lettings_log&.send("relat#{i}")).to eq(nil)
+              expect(lettings_log&.send("ecstat#{i}")).to eq(nil)
+            end
+          end
+        end
+      end
     end
   end
 

From d3dec60f3969d6e02df3892d21ba36d452dacd06 Mon Sep 17 00:00:00 2001
From: kosiakkatrina <54268893+kosiakkatrina@users.noreply.github.com>
Date: Tue, 12 Sep 2023 16:41:00 +0100
Subject: [PATCH 28/33] Only set soc tenant fields for 2022 or if they're given
 (#1908)

* Only set soc tenant fields for 2022 or if they're given

* Refactor
---
 .../imports/sales_logs_import_service.rb      | 14 +++++--
 .../imports/sales_logs_import_service_spec.rb | 38 +++++++++++++++++++
 2 files changed, 49 insertions(+), 3 deletions(-)

diff --git a/app/services/imports/sales_logs_import_service.rb b/app/services/imports/sales_logs_import_service.rb
index 7d10f52ca..c57f2d16a 100644
--- a/app/services/imports/sales_logs_import_service.rb
+++ b/app/services/imports/sales_logs_import_service.rb
@@ -1,5 +1,7 @@
 module Imports
   class SalesLogsImportService < LogsImportService
+    include CollectionTimeHelper
+
     def initialize(storage_service, logger = Rails.logger, allow_updates: false)
       @logs_with_discrepancies = Set.new
       @logs_overridden = Set.new
@@ -124,7 +126,7 @@ module Imports
       attributes["mortgagelenderother"] = mortgage_lender_other(xml_doc, attributes)
       attributes["postcode_full"] = parse_postcode(string_or_nil(xml_doc, "Q14Postcode"))
       attributes["pcodenk"] = 0 if attributes["postcode_full"].present? # known if given
-      attributes["soctenant"] = 0 if attributes["ownershipsch"] == 1
+      attributes["soctenant"] = 0 if set_soctenant_fields?(attributes)
 
       attributes["previous_la_known"] = 1 if attributes["prevloc"].present?
       if attributes["la"].present?
@@ -584,6 +586,12 @@ module Imports
       end
     end
 
+    def set_soctenant_fields?(attributes)
+      return false if attributes["ownershipsch"] != 1
+
+      %w[socprevten frombeds fromprop].any? { |field| attributes[field].present? } || collection_start_year_for_date(attributes["saledate"]) < 2023
+    end
+
     def set_default_values(attributes)
       attributes["armedforcesspouse"] ||= 7
       attributes["hhregres"] ||= 8
@@ -600,8 +608,8 @@ module Imports
       attributes["pcodenk"] ||= 1
       attributes["prevten"] ||= 0
       attributes["extrabor"] ||= 3 if attributes["mortgageused"] == 1
-      attributes["socprevten"] ||= 10 if attributes["ownershipsch"] == 1
-      attributes["fromprop"] ||= 0 if attributes["ownershipsch"] == 1
+      attributes["socprevten"] ||= 10 if set_soctenant_fields?(attributes)
+      attributes["fromprop"] ||= 0 if set_soctenant_fields?(attributes)
       attributes["mortgagelender"] ||= 0 if attributes["mortgageused"] == 1
 
       # buyer 1 characteristics
diff --git a/spec/services/imports/sales_logs_import_service_spec.rb b/spec/services/imports/sales_logs_import_service_spec.rb
index 6d067dbd4..615201f3a 100644
--- a/spec/services/imports/sales_logs_import_service_spec.rb
+++ b/spec/services/imports/sales_logs_import_service_spec.rb
@@ -409,6 +409,42 @@ RSpec.describe Imports::SalesLogsImportService do
           sales_log = SalesLog.find_by(old_id: sales_log_id)
           expect(sales_log.proplen_asked).to eq(1)
         end
+
+        context "when setting soctenant fields" do
+          it "does not set soctenant value if none of the soctenant questions are answered" do
+            sales_log_xml.at_xpath("//xmlns:Q20Bedrooms").content = nil
+            sales_log_xml.at_xpath("//xmlns:PrevRentType").content = nil
+            sales_log_xml.at_xpath("//xmlns:Q21PropertyType").content = nil
+
+            expect(logger).not_to receive(:error)
+            expect(logger).not_to receive(:warn)
+            expect(logger).not_to receive(:info)
+            expect { sales_log_service.send(:create_log, sales_log_xml) }
+            .to change(SalesLog, :count).by(1)
+            sales_log = SalesLog.find_by(old_id: sales_log_id)
+            expect(sales_log.soctenant).to eq(nil)
+            expect(sales_log.frombeds).to eq(nil)
+            expect(sales_log.fromprop).to eq(nil)
+            expect(sales_log.socprevten).to eq(nil)
+          end
+
+          it "sets soctenant to don't know if any of the soctenant questions are answered" do
+            sales_log_xml.at_xpath("//xmlns:Q20Bedrooms").content = "2"
+            sales_log_xml.at_xpath("//xmlns:PrevRentType").content = nil
+            sales_log_xml.at_xpath("//xmlns:Q21PropertyType").content = nil
+
+            expect(logger).not_to receive(:error)
+            expect(logger).not_to receive(:warn)
+            expect(logger).not_to receive(:info)
+            expect { sales_log_service.send(:create_log, sales_log_xml) }
+            .to change(SalesLog, :count).by(1)
+            sales_log = SalesLog.find_by(old_id: sales_log_id)
+            expect(sales_log.soctenant).to eq(0)
+            expect(sales_log.frombeds).to eq(2)
+            expect(sales_log.fromprop).to eq(0)
+            expect(sales_log.socprevten).to eq(10)
+          end
+        end
       end
 
       context "with discounted sale type" do
@@ -2091,6 +2127,7 @@ RSpec.describe Imports::SalesLogsImportService do
 
           sales_log = SalesLog.find_by(old_id: sales_log_id)
           expect(sales_log&.fromprop).to be(0)
+          expect(sales_log&.soctenant).to be(0)
         end
       end
 
@@ -2107,6 +2144,7 @@ RSpec.describe Imports::SalesLogsImportService do
 
           sales_log = SalesLog.find_by(old_id: sales_log_id)
           expect(sales_log&.socprevten).to be(10)
+          expect(sales_log&.soctenant).to be(0)
         end
       end
 

From 20337fde59481e6d487612284024d2d274ab5cdd Mon Sep 17 00:00:00 2001
From: natdeanlewissoftwire
 <94526761+natdeanlewissoftwire@users.noreply.github.com>
Date: Wed, 13 Sep 2023 16:07:04 +0100
Subject: [PATCH 29/33] Migration use kebab org fields (#1913)

* feat: use kebab case org fields

* feat: update tests
---
 .../imports/lettings_logs_import_service.rb   |  4 ++--
 app/services/imports/logs_import_service.rb   |  6 ++---
 .../sales_logs_field_import_service.rb        |  2 +-
 .../imports/sales_logs_import_service.rb      |  2 +-
 app/services/imports/scheme_import_service.rb |  2 +-
 .../00d2343e-d5fa-4c89-8400-ec3854b0f2b4.xml  |  2 +-
 .../0b4a68df-30cc-474a-93c0-a56ce8fdad3b.xml  |  2 +-
 .../0ead17cb-1668-442d-898c-0d52879ff592.xml  |  2 +-
 .../discounted_ownership_sales_log.xml        |  4 ++--
 .../imports/sales_logs/lettings_log.xml       |  4 ++--
 .../lettings_logs_import_service_spec.rb      | 22 +++++++++----------
 .../sales_logs_field_import_service_spec.rb   |  2 +-
 .../imports/sales_logs_import_service_spec.rb | 12 +++++-----
 13 files changed, 33 insertions(+), 33 deletions(-)

diff --git a/app/services/imports/lettings_logs_import_service.rb b/app/services/imports/lettings_logs_import_service.rb
index d40df0fc1..1d3a3c3c7 100644
--- a/app/services/imports/lettings_logs_import_service.rb
+++ b/app/services/imports/lettings_logs_import_service.rb
@@ -61,8 +61,8 @@ module Imports
       # Required fields for status complete or logic to work
       # Note: order matters when we derive from previous values (attributes parameter)
       attributes["startdate"] = compose_date(xml_doc, "DAY", "MONTH", "YEAR")
-      attributes["owning_organisation_id"] = find_organisation_id(xml_doc, "OWNINGORGID")
-      attributes["managing_organisation_id"] = find_organisation_id(xml_doc, "MANINGORGID")
+      attributes["owning_organisation_id"] = find_organisation_id(xml_doc, "owner-institution-id")
+      attributes["managing_organisation_id"] = find_organisation_id(xml_doc, "managing-institution-id")
       attributes["creation_method"] = creation_method(xml_doc)
       attributes["joint"] = unsafe_string_as_integer(xml_doc, "joint")
       attributes["startertenancy"] = unsafe_string_as_integer(xml_doc, "_2a")
diff --git a/app/services/imports/logs_import_service.rb b/app/services/imports/logs_import_service.rb
index f7456a3fb..3b08f57ed 100644
--- a/app/services/imports/logs_import_service.rb
+++ b/app/services/imports/logs_import_service.rb
@@ -31,9 +31,9 @@ module Imports
     end
 
     def find_organisation_id(xml_doc, id_field)
-      old_visible_id = string_or_nil(xml_doc, id_field)
-      organisation = Organisation.find_by(old_visible_id:)
-      raise "Organisation not found with legacy ID #{old_visible_id}" if organisation.nil?
+      old_org_id = meta_field_value(xml_doc, id_field)&.strip.presence
+      organisation = Organisation.find_by(old_org_id:)
+      raise "Organisation not found with old org ID #{old_org_id}" if organisation.nil?
 
       organisation.id
     end
diff --git a/app/services/imports/sales_logs_field_import_service.rb b/app/services/imports/sales_logs_field_import_service.rb
index 2a08a7ccf..4cda93d9d 100644
--- a/app/services/imports/sales_logs_field_import_service.rb
+++ b/app/services/imports/sales_logs_field_import_service.rb
@@ -45,7 +45,7 @@ module Imports
         if record.owning_organisation_id.present?
           @logger.info("sales log #{record.id} has a value for owning_organisation_id, skipping update")
         else
-          owning_organisation_id = find_organisation_id(xml_doc, "OWNINGORGID")
+          owning_organisation_id = find_organisation_id(xml_doc, "owner-institution-id")
           record.update!(owning_organisation_id:)
           @logger.info("sales log #{record.id}'s owning_organisation_id value has been set to #{owning_organisation_id}")
         end
diff --git a/app/services/imports/sales_logs_import_service.rb b/app/services/imports/sales_logs_import_service.rb
index c57f2d16a..b406c1c8b 100644
--- a/app/services/imports/sales_logs_import_service.rb
+++ b/app/services/imports/sales_logs_import_service.rb
@@ -27,7 +27,7 @@ module Imports
       # Note: order matters when we derive from previous values (attributes parameter)
 
       attributes["saledate"] = compose_date(xml_doc, "DAY", "MONTH", "YEAR") || Time.zone.parse(field_value(xml_doc, "xmlns", "CompletionDate"))
-      attributes["owning_organisation_id"] = find_organisation_id(xml_doc, "OWNINGORGID")
+      attributes["owning_organisation_id"] = find_organisation_id(xml_doc, "owner-institution-id")
       attributes["type"] = unsafe_string_as_integer(xml_doc, "DerSaleType")
       attributes["old_id"] = meta_field_value(xml_doc, "document-id")
       attributes["old_form_id"] = safe_string_as_integer(xml_doc, "Form")
diff --git a/app/services/imports/scheme_import_service.rb b/app/services/imports/scheme_import_service.rb
index b0b8f3ceb..7000c391f 100644
--- a/app/services/imports/scheme_import_service.rb
+++ b/app/services/imports/scheme_import_service.rb
@@ -54,7 +54,7 @@ module Imports
 
     def find_owning_organisation_id(old_org_id)
       organisation = Organisation.find_by(old_org_id:)
-      raise "Organisation not found with legacy ID #{old_org_id}" if organisation.nil?
+      raise "Organisation not found with old org ID #{old_org_id}" if organisation.nil?
 
       organisation.id
     end
diff --git a/spec/fixtures/imports/logs/00d2343e-d5fa-4c89-8400-ec3854b0f2b4.xml b/spec/fixtures/imports/logs/00d2343e-d5fa-4c89-8400-ec3854b0f2b4.xml
index cd0e03d74..50f4a4d16 100644
--- a/spec/fixtures/imports/logs/00d2343e-d5fa-4c89-8400-ec3854b0f2b4.xml
+++ b/spec/fixtures/imports/logs/00d2343e-d5fa-4c89-8400-ec3854b0f2b4.xml
@@ -4,7 +4,7 @@
     <meta:document-id>00d2343e-d5fa-4c89-8400-ec3854b0f2b4</meta:document-id>
     <meta:owner-user-id>c3061a2e6ea0b702e6f6210d5c52d2a92612d2aa</meta:owner-user-id>
     <meta:owner-institution-id>7c5bd5fb549c09a2c55d7cb90d7ba84927e64618</meta:owner-institution-id>
-    <meta:managing-institution-id>7c5bd5fb549c09a2c55d7cb90d7ba84927e64618</meta:managing-institution-id>
+    <meta:managing-institution-id>7c5bd5fb549c09z2c55d9cb90d7ba84927e64618</meta:managing-institution-id>
     <meta:created-date>2022-04-14T16:01:30.369241Z</meta:created-date>
     <meta:modified-date>2022-04-14T16:01:30.369241Z</meta:modified-date>
     <meta:status>submitted-valid</meta:status>
diff --git a/spec/fixtures/imports/logs/0b4a68df-30cc-474a-93c0-a56ce8fdad3b.xml b/spec/fixtures/imports/logs/0b4a68df-30cc-474a-93c0-a56ce8fdad3b.xml
index 38cef1339..8d9465305 100644
--- a/spec/fixtures/imports/logs/0b4a68df-30cc-474a-93c0-a56ce8fdad3b.xml
+++ b/spec/fixtures/imports/logs/0b4a68df-30cc-474a-93c0-a56ce8fdad3b.xml
@@ -3,7 +3,7 @@
     <meta:form-name>2021-CORE-SR-SH</meta:form-name>
     <meta:document-id>0b4a68df-30cc-474a-93c0-a56ce8fdad3b</meta:document-id>
     <meta:owner-user-id>c3061a2e6ea0b702e6f6210d5c52d2a92612d2aa</meta:owner-user-id>
-    <meta:owner-institution-id>7c5bd5fb549c09z2c55d9cb90d7ba84927e64618</meta:owner-institution-id>
+    <meta:owner-institution-id>7c5bd5fb549c09a2c55d7cb90d7ba84927e64618</meta:owner-institution-id>
     <meta:managing-institution-id>7c5bd5fb549c09z2c55d9cb90d7ba84927e64618</meta:managing-institution-id>
     <meta:created-date>2022-01-05T12:50:20.39153Z</meta:created-date>
     <meta:modified-date>2022-01-05T12:50:20.39153Z</meta:modified-date>
diff --git a/spec/fixtures/imports/logs/0ead17cb-1668-442d-898c-0d52879ff592.xml b/spec/fixtures/imports/logs/0ead17cb-1668-442d-898c-0d52879ff592.xml
index 87797e242..aba2b2256 100644
--- a/spec/fixtures/imports/logs/0ead17cb-1668-442d-898c-0d52879ff592.xml
+++ b/spec/fixtures/imports/logs/0ead17cb-1668-442d-898c-0d52879ff592.xml
@@ -19,7 +19,7 @@
     </meta:owner-user-id>
     <meta:owner-institution-id>7c5bd5fb549c09a2c55d7cb90d7ba84927e64618
     </meta:owner-institution-id>
-    <meta:managing-institution-id>7c5bd5fb549c09a2c55d7cb90d7ba84927e64618
+    <meta:managing-institution-id>7c5bd5fb549c09z2c55d9cb90d7ba84927e64618
     </meta:managing-institution-id>
     <meta:created-date>2021-10-08T14:48:17.096123Z</meta:created-date>
     <meta:modified-date>2021-10-08T14:48:17.096123Z</meta:modified-date>
diff --git a/spec/fixtures/imports/sales_logs/discounted_ownership_sales_log.xml b/spec/fixtures/imports/sales_logs/discounted_ownership_sales_log.xml
index 0c912dd3c..ec39157c3 100644
--- a/spec/fixtures/imports/sales_logs/discounted_ownership_sales_log.xml
+++ b/spec/fixtures/imports/sales_logs/discounted_ownership_sales_log.xml
@@ -3,8 +3,8 @@
     <meta:form-name>2022-CORE-Sales</meta:form-name>
     <meta:document-id>discounted_ownership_sales_log</meta:document-id>
     <meta:owner-user-id>c3061a2e6ea0b702e6f6210d5c52d2a92612d2aa</meta:owner-user-id>
-    <meta:owner-institution-id>7c5bd5fb549c09z2c55d9cb90d7ba84927e64618</meta:owner-institution-id>
-    <meta:managing-institution-id>7c5bd5fb549c09z2c55d9cb90d7ba84927e64618</meta:managing-institution-id>
+    <meta:owner-institution-id>7c5bd5fb549c09a2c55d7cb90d7ba84927e64618</meta:owner-institution-id>
+    <meta:managing-institution-id>7c5bd5fb549c09a2c55d7cb90d7ba84927e64618</meta:managing-institution-id>
     <meta:created-date>2023-02-21T11:54:51.786722Z</meta:created-date>
     <meta:modified-date>2023-02-22T10:59:45.88188Z</meta:modified-date>
     <meta:status>submitted-valid</meta:status>
diff --git a/spec/fixtures/imports/sales_logs/lettings_log.xml b/spec/fixtures/imports/sales_logs/lettings_log.xml
index 1f7f794f6..d02fd6052 100644
--- a/spec/fixtures/imports/sales_logs/lettings_log.xml
+++ b/spec/fixtures/imports/sales_logs/lettings_log.xml
@@ -3,8 +3,8 @@
     <meta:form-name>2021-CORE-SR-SH</meta:form-name>
     <meta:document-id>lettings_log</meta:document-id>
     <meta:owner-user-id>c3061a2e6ea0b702e6f6210d5c52d2a92612d2aa</meta:owner-user-id>
-    <meta:owner-institution-id>7c5bd5fb549c09z2c55d9cb90d7ba84927e64618</meta:owner-institution-id>
-    <meta:managing-institution-id>7c5bd5fb549c09z2c55d9cb90d7ba84927e64618</meta:managing-institution-id>
+    <meta:owner-institution-id>7c5bd5fb549c09a2c55d7cb90d7ba84927e64618</meta:owner-institution-id>
+    <meta:managing-institution-id>7c5bd5fb549c09a2c55d7cb90d7ba84927e64618</meta:managing-institution-id>
     <meta:created-date>2022-01-05T12:50:20.39153Z</meta:created-date>
     <meta:modified-date>2022-01-05T12:50:20.39153Z</meta:modified-date>
     <meta:status>submitted-valid</meta:status>
diff --git a/spec/services/imports/lettings_logs_import_service_spec.rb b/spec/services/imports/lettings_logs_import_service_spec.rb
index efef0997a..ac8dddccd 100644
--- a/spec/services/imports/lettings_logs_import_service_spec.rb
+++ b/spec/services/imports/lettings_logs_import_service_spec.rb
@@ -22,8 +22,8 @@ RSpec.describe Imports::LettingsLogsImportService do
     let(:real_2022_2023_form) { Form.new("config/forms/2022_2023.json") }
     let(:fixture_directory) { "spec/fixtures/imports/logs" }
 
-    let(:organisation) { FactoryBot.create(:organisation, old_visible_id: "1", provider_type: "PRP") }
-    let(:managing_organisation) { FactoryBot.create(:organisation, old_visible_id: "2", provider_type: "PRP") }
+    let(:organisation) { FactoryBot.create(:organisation, old_org_id: "7c5bd5fb549c09a2c55d7cb90d7ba84927e64618", provider_type: "PRP") }
+    let(:managing_organisation) { FactoryBot.create(:organisation, old_org_id: "7c5bd5fb549c09z2c55d9cb90d7ba84927e64618", provider_type: "PRP") }
     let(:scheme1) { FactoryBot.create(:scheme, old_visible_id: "0123", owning_organisation: organisation) }
     let(:scheme2) { FactoryBot.create(:scheme, old_visible_id: "456", owning_organisation: organisation) }
 
@@ -36,8 +36,8 @@ RSpec.describe Imports::LettingsLogsImportService do
              .to_return(status: 200, body: '{"status":200,"result":{"admin_district":"Westminster","codes":{"admin_district":"E08000035"}}}', headers: {})
 
       allow(Organisation).to receive(:find_by).and_return(nil)
-      allow(Organisation).to receive(:find_by).with(old_visible_id: organisation.old_visible_id).and_return(organisation)
-      allow(Organisation).to receive(:find_by).with(old_visible_id: managing_organisation.old_visible_id).and_return(managing_organisation)
+      allow(Organisation).to receive(:find_by).with(old_org_id: organisation.old_org_id).and_return(organisation)
+      allow(Organisation).to receive(:find_by).with(old_org_id: managing_organisation.old_org_id).and_return(managing_organisation)
 
       # Created by users
       FactoryBot.create(:user, old_user_id: "c3061a2e6ea0b702e6f6210d5c52d2a92612d2aa", organisation:)
@@ -231,11 +231,11 @@ RSpec.describe Imports::LettingsLogsImportService do
       end
 
       context "and the organisation legacy ID does not exist" do
-        before { lettings_log_xml.at_xpath("//xmlns:OWNINGORGID").content = 99_999 }
+        before { lettings_log_xml.at_xpath("//meta:owner-institution-id").content = 99_999 }
 
         it "raises an exception" do
           expect { lettings_log_service.send(:create_log, lettings_log_xml) }
-            .to raise_error(RuntimeError, "Organisation not found with legacy ID 99999")
+            .to raise_error(RuntimeError, "Organisation not found with old org ID 99999")
         end
       end
 
@@ -1547,8 +1547,8 @@ RSpec.describe Imports::LettingsLogsImportService do
     let(:real_2022_2023_form) { Form.new("config/forms/2022_2023.json") }
     let(:fixture_directory) { "spec/fixtures/imports/logs" }
 
-    let(:organisation) { FactoryBot.create(:organisation, old_visible_id: "1", provider_type: "PRP") }
-    let(:managing_organisation) { FactoryBot.create(:organisation, old_visible_id: "2", provider_type: "PRP") }
+    let(:organisation) { FactoryBot.create(:organisation, old_org_id: "7c5bd5fb549c09a2c55d7cb90d7ba84927e64618", provider_type: "PRP") }
+    let(:managing_organisation) { FactoryBot.create(:organisation, old_org_id: "7c5bd5fb549c09z2c55d9cb90d7ba84927e64618", provider_type: "PRP") }
     let(:scheme1) { FactoryBot.create(:scheme, old_visible_id: "0123", owning_organisation: organisation) }
     let(:scheme2) { FactoryBot.create(:scheme, old_visible_id: "456", owning_organisation: organisation) }
 
@@ -1561,8 +1561,8 @@ RSpec.describe Imports::LettingsLogsImportService do
              .to_return(status: 200, body: '{"status":200,"result":{"admin_district":"Westminster","codes":{"admin_district":"E08000035"}}}', headers: {})
 
       allow(Organisation).to receive(:find_by).and_return(nil)
-      allow(Organisation).to receive(:find_by).with(old_visible_id: organisation.old_visible_id).and_return(organisation)
-      allow(Organisation).to receive(:find_by).with(old_visible_id: managing_organisation.old_visible_id).and_return(managing_organisation)
+      allow(Organisation).to receive(:find_by).with(old_org_id: organisation.old_org_id).and_return(organisation)
+      allow(Organisation).to receive(:find_by).with(old_org_id: managing_organisation.old_org_id).and_return(managing_organisation)
 
       # Created by users
       FactoryBot.create(:user, old_user_id: "c3061a2e6ea0b702e6f6210d5c52d2a92612d2aa", organisation:)
@@ -1792,7 +1792,7 @@ RSpec.describe Imports::LettingsLogsImportService do
         .not_to raise_error
       end
 
-      it "clears out the referral answer" do
+      it "clears out the period answer" do
         allow(logger).to receive(:warn)
         lettings_log_service.send(:create_log, lettings_log_xml)
         lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
diff --git a/spec/services/imports/sales_logs_field_import_service_spec.rb b/spec/services/imports/sales_logs_field_import_service_spec.rb
index f1ae6680c..6b39b3acb 100644
--- a/spec/services/imports/sales_logs_field_import_service_spec.rb
+++ b/spec/services/imports/sales_logs_field_import_service_spec.rb
@@ -9,7 +9,7 @@ RSpec.describe Imports::SalesLogsFieldImportService do
   let(:fixture_directory) { "spec/fixtures/imports/sales_logs" }
   let(:sales_log_filename) { "shared_ownership_sales_log" }
   let(:sales_log_file) { File.open("#{fixture_directory}/#{sales_log_filename}.xml") }
-  let(:organisation) { create(:organisation, old_visible_id: "1") }
+  let(:organisation) { create(:organisation, old_visible_id: "1", old_org_id: "7c5bd5fb549c09a2c55d7cb90d7ba84927e64618") }
   let(:old_user_id) { "c3061a2e6ea0b702e6f6210d5c52d2a92612d2aa" }
 
   let(:remote_folder) { "sales_logs" }
diff --git a/spec/services/imports/sales_logs_import_service_spec.rb b/spec/services/imports/sales_logs_import_service_spec.rb
index 615201f3a..2306448b8 100644
--- a/spec/services/imports/sales_logs_import_service_spec.rb
+++ b/spec/services/imports/sales_logs_import_service_spec.rb
@@ -8,8 +8,8 @@ RSpec.describe Imports::SalesLogsImportService do
 
   let(:fixture_directory) { "spec/fixtures/imports/sales_logs" }
 
-  let(:organisation) { FactoryBot.create(:organisation, old_visible_id: "1", provider_type: "PRP") }
-  let(:managing_organisation) { FactoryBot.create(:organisation, old_visible_id: "2", provider_type: "PRP") }
+  let(:organisation) { FactoryBot.create(:organisation, old_org_id: "7c5bd5fb549c09a2c55d7cb90d7ba84927e64618", provider_type: "PRP") }
+  let(:managing_organisation) { FactoryBot.create(:organisation, old_org_id: "7c5bd5fb549c09z2c55d9cb90d7ba84927e64618", provider_type: "PRP") }
   let(:remote_folder) { "sales_logs" }
 
   def open_file(directory, filename)
@@ -27,8 +27,8 @@ RSpec.describe Imports::SalesLogsImportService do
 
   before do
     allow(Organisation).to receive(:find_by).and_return(nil)
-    allow(Organisation).to receive(:find_by).with(old_visible_id: organisation.old_visible_id).and_return(organisation)
-    allow(Organisation).to receive(:find_by).with(old_visible_id: managing_organisation.old_visible_id).and_return(managing_organisation)
+    allow(Organisation).to receive(:find_by).with(old_org_id: organisation.old_org_id).and_return(organisation)
+    allow(Organisation).to receive(:find_by).with(old_org_id: managing_organisation.old_org_id).and_return(managing_organisation)
 
     # Created by users
     FactoryBot.create(:user, old_user_id: "c3061a2e6ea0b702e6f6210d5c52d2a92612d2aa", organisation:)
@@ -116,11 +116,11 @@ RSpec.describe Imports::SalesLogsImportService do
     context "and the organisation legacy ID does not exist" do
       let(:sales_log_id) { "shared_ownership_sales_log" }
 
-      before { sales_log_xml.at_xpath("//xmlns:OWNINGORGID").content = 99_999 }
+      before { sales_log_xml.at_xpath("//meta:owner-institution-id").content = 99_999 }
 
       it "raises an exception" do
         expect { sales_log_service.send(:create_log, sales_log_xml) }
-          .to raise_error(RuntimeError, "Organisation not found with legacy ID 99999")
+          .to raise_error(RuntimeError, "Organisation not found with old org ID 99999")
       end
     end
 

From 3da6416950944739438bccf37cc0e92193a80ba7 Mon Sep 17 00:00:00 2001
From: kosiakkatrina <54268893+kosiakkatrina@users.noreply.github.com>
Date: Wed, 13 Sep 2023 16:07:35 +0100
Subject: [PATCH 30/33] Set income and savings not known for in progress logs
 (#1911)

---
 .../imports/sales_logs_import_service.rb      |  3 +
 .../imports/sales_logs_import_service_spec.rb | 67 +++++++++++++++++++
 2 files changed, 70 insertions(+)

diff --git a/app/services/imports/sales_logs_import_service.rb b/app/services/imports/sales_logs_import_service.rb
index b406c1c8b..7848f8628 100644
--- a/app/services/imports/sales_logs_import_service.rb
+++ b/app/services/imports/sales_logs_import_service.rb
@@ -68,11 +68,14 @@ module Imports
       attributes["la"] = string_or_nil(xml_doc, "Q14ONSLACode")
       attributes["income1"] = safe_string_as_integer(xml_doc, "Q2Person1Income")
       attributes["income1nk"] = income_known(unsafe_string_as_integer(xml_doc, "P1IncKnown"))
+      attributes["income1nk"] ||= 0 if attributes["income1"].present?
       attributes["inc1mort"] = unsafe_string_as_integer(xml_doc, "Q2Person1Mortgage")
       attributes["income2"] = safe_string_as_integer(xml_doc, "Q2Person2Income")
       attributes["income2nk"] = income_known(unsafe_string_as_integer(xml_doc, "P2IncKnown"))
+      attributes["income2nk"] ||= 0 if attributes["income2"].present?
       attributes["savings"] = safe_string_as_integer(xml_doc, "Q3Savings")&.round(-1)
       attributes["savingsnk"] = savings_known(xml_doc)
+      attributes["savingsnk"] ||= 0 if attributes["savings"].present?
       attributes["prevown"] = unsafe_string_as_integer(xml_doc, "Q4PrevOwnedProperty")
       attributes["mortgage"] = safe_string_as_decimal(xml_doc, "CALCMORT")
       attributes["inc2mort"] = unsafe_string_as_integer(xml_doc, "Q2Person2MortApplication")
diff --git a/spec/services/imports/sales_logs_import_service_spec.rb b/spec/services/imports/sales_logs_import_service_spec.rb
index 2306448b8..69e82ead8 100644
--- a/spec/services/imports/sales_logs_import_service_spec.rb
+++ b/spec/services/imports/sales_logs_import_service_spec.rb
@@ -2200,5 +2200,72 @@ RSpec.describe Imports::SalesLogsImportService do
         end
       end
     end
+
+    context "when setting income for in progress logs" do
+      let(:sales_log_id) { "shared_ownership_sales_log" }
+
+      before do
+        sales_log_xml.at_xpath("//meta:status").content = "saved"
+      end
+
+      it "sets income to known if not answered but income is given for person 1" do
+        sales_log_xml.at_xpath("//xmlns:joint").content = "2 No"
+        sales_log_xml.at_xpath("//xmlns:P1IncKnown").content = ""
+        sales_log_xml.at_xpath("//xmlns:Q2Person1Income").content = "20000"
+        sales_log_xml.at_xpath("//xmlns:P2IncKnown").content = ""
+        sales_log_xml.at_xpath("//xmlns:Q2Person2Income").content = "30000"
+        sales_log_service.send(:create_log, sales_log_xml)
+
+        sales_log = SalesLog.find_by(old_id: sales_log_id)
+        expect(sales_log&.income1nk).to eq(0)
+        expect(sales_log&.income1).to eq(20_000)
+        expect(sales_log&.income2nk).to eq(nil)
+        expect(sales_log&.income2).to eq(nil)
+      end
+
+      it "sets income to known if not answered but income is given for person 2" do
+        sales_log_xml.at_xpath("//xmlns:joint").content = "1 Yes"
+        sales_log_xml.at_xpath("//xmlns:JointMore").content = "2 No"
+        sales_log_xml.at_xpath("//xmlns:P1IncKnown").content = ""
+        sales_log_xml.at_xpath("//xmlns:Q2Person1Income").content = "20000"
+        sales_log_xml.at_xpath("//xmlns:P2IncKnown").content = ""
+        sales_log_xml.at_xpath("//xmlns:Q2Person2Income").content = "30000"
+        sales_log_service.send(:create_log, sales_log_xml)
+
+        sales_log = SalesLog.find_by(old_id: sales_log_id)
+        expect(sales_log&.income1nk).to eq(0)
+        expect(sales_log&.income1).to eq(20_000)
+        expect(sales_log&.income2nk).to eq(0)
+        expect(sales_log&.income2).to eq(30_000)
+      end
+
+      it "sets savings to known if savings are given" do
+        sales_log_xml.at_xpath("//xmlns:Q3Savings").content = "10000"
+        sales_log_xml.at_xpath("//xmlns:savingsKnown").content = ""
+        sales_log_service.send(:create_log, sales_log_xml)
+
+        sales_log = SalesLog.find_by(old_id: sales_log_id)
+        expect(sales_log&.savingsnk).to eq(0)
+        expect(sales_log&.savings).to eq(10_000)
+      end
+
+      it "does not set savings or income fields when values aren't given" do
+        sales_log_xml.at_xpath("//xmlns:P1IncKnown").content = ""
+        sales_log_xml.at_xpath("//xmlns:Q2Person1Income").content = ""
+        sales_log_xml.at_xpath("//xmlns:P2IncKnown").content = ""
+        sales_log_xml.at_xpath("//xmlns:Q2Person2Income").content = ""
+        sales_log_xml.at_xpath("//xmlns:Q3Savings").content = ""
+        sales_log_xml.at_xpath("//xmlns:savingsKnown").content = ""
+        sales_log_service.send(:create_log, sales_log_xml)
+
+        sales_log = SalesLog.find_by(old_id: sales_log_id)
+        expect(sales_log&.income1nk).to eq(nil)
+        expect(sales_log&.income1).to eq(nil)
+        expect(sales_log&.income2nk).to eq(nil)
+        expect(sales_log&.income2).to eq(nil)
+        expect(sales_log&.savingsnk).to eq(nil)
+        expect(sales_log&.savings).to eq(nil)
+      end
+    end
   end
 end

From 7a64cb7f564092cf33d152f614fac53f8780ec4e Mon Sep 17 00:00:00 2001
From: kosiakkatrina <54268893+kosiakkatrina@users.noreply.github.com>
Date: Wed, 13 Sep 2023 16:34:42 +0100
Subject: [PATCH 31/33] CLDC-2776 Add referral option to general needs logs
 (#1910)

* Add option to referral PRP general needs questions

* Add referral value check

* Add referral value check to 2023

* Update soft validation content

* Add referral value check to 2022

* Confirm soft validation on import

* Tests
---
 .../lettings/pages/referral_value_check.rb    | 22 +++++++++
 .../form/lettings/questions/referral_prp.rb   |  3 ++
 .../questions/referral_value_check.rb         | 14 ++++++
 .../subsections/household_situation.rb        |  1 +
 app/models/lettings_log.rb                    |  4 ++
 .../imports/lettings_logs_import_service.rb   |  3 +-
 config/forms/2022_2023.json                   | 45 ++++++++++++++++++-
 config/locales/en.yml                         |  3 ++
 ...20230912115233_add_referral_value_check.rb |  5 +++
 db/schema.rb                                  |  3 +-
 .../files/lettings_log_csv_export_codes.csv   |  4 +-
 .../files/lettings_log_csv_export_labels.csv  |  4 +-
 ...tings_log_csv_export_non_support_codes.csv |  4 +-
 ...ings_log_csv_export_non_support_labels.csv |  4 +-
 .../subsections/household_situation_spec.rb   |  1 +
 .../validations/soft_validations_spec.rb      | 30 +++++++++++++
 .../lettings_logs_import_service_spec.rb      | 19 ++++++++
 17 files changed, 158 insertions(+), 11 deletions(-)
 create mode 100644 app/models/form/lettings/pages/referral_value_check.rb
 create mode 100644 app/models/form/lettings/questions/referral_value_check.rb
 create mode 100644 db/migrate/20230912115233_add_referral_value_check.rb

diff --git a/app/models/form/lettings/pages/referral_value_check.rb b/app/models/form/lettings/pages/referral_value_check.rb
new file mode 100644
index 000000000..55f6f3e8b
--- /dev/null
+++ b/app/models/form/lettings/pages/referral_value_check.rb
@@ -0,0 +1,22 @@
+class Form::Lettings::Pages::ReferralValueCheck < ::Form::Page
+  def initialize(id, hsh, subsection)
+    super
+    @id = "referral_value_check"
+    @depends_on = [{ "la_referral_for_general_needs?" => true }]
+    @title_text = {
+      "translation" => "soft_validations.referral.title_text",
+    }
+    @informative_text = {
+      "translation" => "soft_validations.referral.hint_text",
+      "arguments" => [],
+    }
+  end
+
+  def questions
+    @questions ||= [Form::Lettings::Questions::ReferralValueCheck.new(nil, nil, self)]
+  end
+
+  def interruption_screen_question_ids
+    %w[needstype referral]
+  end
+end
diff --git a/app/models/form/lettings/questions/referral_prp.rb b/app/models/form/lettings/questions/referral_prp.rb
index 69a42bed5..8a3f03568 100644
--- a/app/models/form/lettings/questions/referral_prp.rb
+++ b/app/models/form/lettings/questions/referral_prp.rb
@@ -21,6 +21,9 @@ class Form::Lettings::Questions::ReferralPrp < ::Form::Question
     "3" => {
       "value" => "Nominated by a local housing authority",
     },
+    "4" => {
+      "value" => "Referred by local authority housing department",
+    },
     "8" => {
       "value" => "Re-located through official housing mobility scheme",
     },
diff --git a/app/models/form/lettings/questions/referral_value_check.rb b/app/models/form/lettings/questions/referral_value_check.rb
new file mode 100644
index 000000000..30aa45f4c
--- /dev/null
+++ b/app/models/form/lettings/questions/referral_value_check.rb
@@ -0,0 +1,14 @@
+class Form::Lettings::Questions::ReferralValueCheck < ::Form::Question
+  def initialize(id, hsh, page)
+    super
+    @id = "referral_value_check"
+    @check_answer_label = "Referral confirmation"
+    @header = "Are you sure?"
+    @type = "interruption_screen"
+    @check_answers_card_number = 0
+    @answer_options = ANSWER_OPTIONS
+    @hidden_in_check_answers = { "depends_on" => [{ "referral_value_check" => 0 }, { "referral_value_check" => 1 }] }
+  end
+
+  ANSWER_OPTIONS = { "0" => { "value" => "Yes" }, "1" => { "value" => "No" } }.freeze
+end
diff --git a/app/models/form/lettings/subsections/household_situation.rb b/app/models/form/lettings/subsections/household_situation.rb
index e09033563..9db7c1f04 100644
--- a/app/models/form/lettings/subsections/household_situation.rb
+++ b/app/models/form/lettings/subsections/household_situation.rb
@@ -24,6 +24,7 @@ class Form::Lettings::Subsections::HouseholdSituation < ::Form::Subsection
       Form::Lettings::Pages::ReferralPrp.new(nil, nil, self),
       Form::Lettings::Pages::ReferralSupportedHousing.new(nil, nil, self),
       Form::Lettings::Pages::ReferralSupportedHousingPrp.new(nil, nil, self),
+      Form::Lettings::Pages::ReferralValueCheck.new(nil, nil, self),
     ].compact
   end
 end
diff --git a/app/models/lettings_log.rb b/app/models/lettings_log.rb
index 6d15699cc..3d01c5284 100644
--- a/app/models/lettings_log.rb
+++ b/app/models/lettings_log.rb
@@ -568,6 +568,10 @@ class LettingsLog < Log
     end
   end
 
+  def la_referral_for_general_needs?
+    is_general_needs? && referral == 4
+  end
+
 private
 
   def reset_invalid_unresolved_log_fields!
diff --git a/app/services/imports/lettings_logs_import_service.rb b/app/services/imports/lettings_logs_import_service.rb
index 1d3a3c3c7..865f8417f 100644
--- a/app/services/imports/lettings_logs_import_service.rb
+++ b/app/services/imports/lettings_logs_import_service.rb
@@ -242,6 +242,7 @@ module Imports
       attributes["rent_value_check"] = 0
       attributes["net_income_value_check"] = 0
       attributes["carehome_charges_value_check"] = 0
+      attributes["referral_value_check"] = 0
 
       # Sets the log creator
       owner_id = meta_field_value(xml_doc, "owner-user-id").strip
@@ -401,7 +402,7 @@ module Imports
     end
 
     def fields_not_present_in_softwire_data
-      %w[majorrepairs illness_type_0 tshortfall_known pregnancy_value_check retirement_value_check rent_value_check net_income_value_check major_repairs_date_value_check void_date_value_check carehome_charges_value_check housingneeds_type housingneeds_other created_by uprn_known uprn_confirmed]
+      %w[majorrepairs illness_type_0 tshortfall_known pregnancy_value_check retirement_value_check rent_value_check net_income_value_check major_repairs_date_value_check void_date_value_check carehome_charges_value_check referral_value_check housingneeds_type housingneeds_other created_by uprn_known uprn_confirmed]
     end
 
     def check_status_completed(lettings_log, previous_status)
diff --git a/config/forms/2022_2023.json b/config/forms/2022_2023.json
index 66b975d9c..0fea72f3b 100644
--- a/config/forms/2022_2023.json
+++ b/config/forms/2022_2023.json
@@ -7057,6 +7057,9 @@
                     "3": {
                       "value": "Nominated by a local housing authority"
                     },
+                    "4" : { 
+                      "value" : "Referred by local authority housing department" 
+                    },
                     "8": {
                       "value": "Re-located through official housing mobility scheme"
                     },
@@ -7115,7 +7118,7 @@
                       "value": "Tenant applied directly (no referral)"
                     },
                     "3": {
-                      "value": "Referred by local authority housing department"
+                      "value": "Nominated by a local housing authority"
                     },
                     "8": {
                       "value": "Re-located through official housing mobility scheme"
@@ -7220,6 +7223,46 @@
                   "renewal": 0
                 }
               ]
+            },
+            "referral_value_check": {
+              "depends_on": [
+                {
+                  "la_referral_for_general_needs?": true
+                }
+              ],
+              "title_text": {
+                "translation": "soft_validations.referral.title_text"
+              },
+              "informative_text": {
+                "translation": "soft_validations.referral.hint_text",
+                "arguments": []
+              },
+              "questions": {
+                "referral_value_check": {
+                  "check_answer_label": "Referral confirmation",
+                  "hidden_in_check_answers": {
+                    "depends_on": [
+                      {
+                        "referral_value_check": 0
+                      },
+                      {
+                        "referral_value_check": 1
+                      }
+                    ]
+                  },
+                  "header": "Are you sure?",
+                  "type": "interruption_screen",
+                  "answer_options": {
+                    "0": {
+                      "value": "Yes"
+                    },
+                    "1": {
+                      "value": "No"
+                    }
+                  }
+                }
+              },
+              "interruption_screen_question_ids": ["needstype", "referral"]
             }
           }
         }
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 5d0f435f9..8f53b4590 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -706,6 +706,9 @@ Make sure these answers are correct."
     mortgage:
       title_text: "You told us that the mortgage amount is %{mortgage}"
       hint_text: "This is more than 5 times the income, which is higher than we would expect."
+    referral:
+      title_text: "Are you sure?"
+      hint_text: "This is a general needs log, and this referral type is for supported housing."
 
   devise:
     email:
diff --git a/db/migrate/20230912115233_add_referral_value_check.rb b/db/migrate/20230912115233_add_referral_value_check.rb
new file mode 100644
index 000000000..580a41563
--- /dev/null
+++ b/db/migrate/20230912115233_add_referral_value_check.rb
@@ -0,0 +1,5 @@
+class AddReferralValueCheck < ActiveRecord::Migration[7.0]
+  def change
+    add_column :lettings_logs, :referral_value_check, :integer
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 7218e75a5..26aef31ac 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2023_08_28_145454) do
+ActiveRecord::Schema[7.0].define(version: 2023_09_12_115233) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
@@ -298,6 +298,7 @@ ActiveRecord::Schema[7.0].define(version: 2023_08_28_145454) do
     t.datetime "discarded_at"
     t.integer "creation_method", default: 1
     t.datetime "values_updated_at"
+    t.integer "referral_value_check"
     t.index ["bulk_upload_id"], name: "index_lettings_logs_on_bulk_upload_id"
     t.index ["created_by_id"], name: "index_lettings_logs_on_created_by_id"
     t.index ["location_id"], name: "index_lettings_logs_on_location_id"
diff --git a/spec/fixtures/files/lettings_log_csv_export_codes.csv b/spec/fixtures/files/lettings_log_csv_export_codes.csv
index 2fc1624af..6076d253b 100644
--- a/spec/fixtures/files/lettings_log_csv_export_codes.csv
+++ b/spec/fixtures/files/lettings_log_csv_export_codes.csv
@@ -1,2 +1,2 @@
-id,status,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,old_id,old_form_id,collection_start_year,owning_organisation_name,managing_organisation_name,needstype,lettype,renewal,startdate,renttype,rent_type_detail,irproduct,irproduct_other,lar,tenancycode,propcode,uprn_known,uprn,uprn_confirmed,address_line1,address_line2,town_or_city,county,postcode_full,is_la_inferred,la_label,la,first_time_property_let_as_social_housing,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,vacdays,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,hhmemb,pregnancy_value_check,age1_known,refused,hhtype,totchild,totelder,totadult,age1,retirement_value_check,sex1,ethnic_group,ethnic,national,ecstat1,details_known_2,relat2,age2_known,age2,sex2,ecstat2,details_known_3,relat3,age3_known,age3,sex3,ecstat3,details_known_4,relat4,age4_known,age4,sex4,ecstat4,details_known_5,relat5,age5_known,age5,sex5,ecstat5,details_known_6,relat6,age6_known,age6,sex6,ecstat6,details_known_7,relat7,age7_known,age7,sex7,ecstat7,details_known_8,relat8,age8_known,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,new_old,homeless,ppcodenk,ppostcode_full,previous_la_known,is_previous_la_inferred,prevloc_label,prevloc,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,letting_allocation_unknown,referral,net_income_known,incref,earnings,incfreq,net_income_value_check,hb,has_benefits,benefits,household_charge,nocharge,period,is_carehome,chcharge,wchchrg,carehome_charges_value_check,brent,wrent,rent_value_check,scharge,wscharge,pscharge,wpschrge,supcharg,wsupchrg,tcharge,wtcharge,hbrentshortfall,tshortfall_known,tshortfall,wtshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_admin_district,location_startdate
-,completed,s.port@jeemayle.com,false,2023-06-26T00:00:00+01:00,,2023-06-26T00:00:00+01:00,1,,,2023,DLUHC,DLUHC,1,7,0,2023-06-26T00:00:00+01:00,2,1,,,,HIJKLMN,ABCDEFG,0,,,fake address,,London,,NW9 5LL,false,Barnet,E09000003,0,2,6,2,2,7,1,1,3,2023-06-24T00:00:00+01:00,,,1,2023-06-25T00:00:00+01:00,,3,1,4,,2,,1,2,,0,0,4,0,0,2,35,,F,0,2,13,0,0,P,0,32,M,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,4,1,2,1,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,2,7,4,,6,2,1,0,TN23 6LZ,1,false,Ashford,E07000105,1,0,1,0,0,0,0,0,1,,2,0,0,68,1,,6,1,1,,0,2,,,,,200.0,100.0,,50.0,25.0,40.0,20.0,35.0,17.5,325.0,162.5,1,0,12.0,6.0,,,,,,,,,,,,,,,,,,,,
+id,status,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,old_id,old_form_id,collection_start_year,owning_organisation_name,managing_organisation_name,needstype,lettype,renewal,startdate,renttype,rent_type_detail,irproduct,irproduct_other,lar,tenancycode,propcode,uprn_known,uprn,uprn_confirmed,address_line1,address_line2,town_or_city,county,postcode_full,is_la_inferred,la_label,la,first_time_property_let_as_social_housing,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,vacdays,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,hhmemb,pregnancy_value_check,age1_known,refused,hhtype,totchild,totelder,totadult,age1,retirement_value_check,sex1,ethnic_group,ethnic,national,ecstat1,details_known_2,relat2,age2_known,age2,sex2,ecstat2,details_known_3,relat3,age3_known,age3,sex3,ecstat3,details_known_4,relat4,age4_known,age4,sex4,ecstat4,details_known_5,relat5,age5_known,age5,sex5,ecstat5,details_known_6,relat6,age6_known,age6,sex6,ecstat6,details_known_7,relat7,age7_known,age7,sex7,ecstat7,details_known_8,relat8,age8_known,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,new_old,homeless,ppcodenk,ppostcode_full,previous_la_known,is_previous_la_inferred,prevloc_label,prevloc,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,letting_allocation_unknown,referral,referral_value_check,net_income_known,incref,earnings,incfreq,net_income_value_check,hb,has_benefits,benefits,household_charge,nocharge,period,is_carehome,chcharge,wchchrg,carehome_charges_value_check,brent,wrent,rent_value_check,scharge,wscharge,pscharge,wpschrge,supcharg,wsupchrg,tcharge,wtcharge,hbrentshortfall,tshortfall_known,tshortfall,wtshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_admin_district,location_startdate
+,completed,s.port@jeemayle.com,false,2023-06-26T00:00:00+01:00,,2023-06-26T00:00:00+01:00,1,,,2023,DLUHC,DLUHC,1,7,0,2023-06-26T00:00:00+01:00,2,1,,,,HIJKLMN,ABCDEFG,0,,,fake address,,London,,NW9 5LL,false,Barnet,E09000003,0,2,6,2,2,7,1,1,3,2023-06-24T00:00:00+01:00,,,1,2023-06-25T00:00:00+01:00,,3,1,4,,2,,1,2,,0,0,4,0,0,2,35,,F,0,2,13,0,0,P,0,32,M,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,4,1,2,1,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,2,7,4,,6,2,1,0,TN23 6LZ,1,false,Ashford,E07000105,1,0,1,0,0,0,0,0,1,,2,,0,0,68,1,,6,1,1,,0,2,,,,,200.0,100.0,,50.0,25.0,40.0,20.0,35.0,17.5,325.0,162.5,1,0,12.0,6.0,,,,,,,,,,,,,,,,,,,,
diff --git a/spec/fixtures/files/lettings_log_csv_export_labels.csv b/spec/fixtures/files/lettings_log_csv_export_labels.csv
index 3c50ecff0..e553083c1 100644
--- a/spec/fixtures/files/lettings_log_csv_export_labels.csv
+++ b/spec/fixtures/files/lettings_log_csv_export_labels.csv
@@ -1,2 +1,2 @@
-id,status,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,old_id,old_form_id,collection_start_year,owning_organisation_name,managing_organisation_name,needstype,lettype,renewal,startdate,renttype,rent_type_detail,irproduct,irproduct_other,lar,tenancycode,propcode,uprn_known,uprn,uprn_confirmed,address_line1,address_line2,town_or_city,county,postcode_full,is_la_inferred,la_label,la,first_time_property_let_as_social_housing,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,vacdays,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,hhmemb,pregnancy_value_check,age1_known,refused,hhtype,totchild,totelder,totadult,age1,retirement_value_check,sex1,ethnic_group,ethnic,national,ecstat1,details_known_2,relat2,age2_known,age2,sex2,ecstat2,details_known_3,relat3,age3_known,age3,sex3,ecstat3,details_known_4,relat4,age4_known,age4,sex4,ecstat4,details_known_5,relat5,age5_known,age5,sex5,ecstat5,details_known_6,relat6,age6_known,age6,sex6,ecstat6,details_known_7,relat7,age7_known,age7,sex7,ecstat7,details_known_8,relat8,age8_known,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,new_old,homeless,ppcodenk,ppostcode_full,previous_la_known,is_previous_la_inferred,prevloc_label,prevloc,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,letting_allocation_unknown,referral,net_income_known,incref,earnings,incfreq,net_income_value_check,hb,has_benefits,benefits,household_charge,nocharge,period,is_carehome,chcharge,wchchrg,carehome_charges_value_check,brent,wrent,rent_value_check,scharge,wscharge,pscharge,wpschrge,supcharg,wsupchrg,tcharge,wtcharge,hbrentshortfall,tshortfall_known,tshortfall,wtshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_admin_district,location_startdate
-,completed,s.port@jeemayle.com,false,2023-06-26T00:00:00+01:00,,2023-06-26T00:00:00+01:00,single log,,,2023,DLUHC,DLUHC,General needs,7,No,2023-06-26T00:00:00+01:00,2,Affordable Rent,,,,HIJKLMN,ABCDEFG,No,,,fake address,,London,,NW9 5LL,No,Barnet,E09000003,No,Affordable rent basis,Tenant abandoned property,2,2,House,Purpose built,Yes,3,2023-06-24T00:00:00+01:00,,,Yes,2023-06-25T00:00:00+01:00,,Don’t know,Yes,Assured Shorthold Tenancy (AST) – Fixed term,,2,,1,2,,Yes,0,4,0,0,2,35,,Female,White,Irish,Tenant prefers not to say,Other,Yes,Partner,Yes,32,Male,Not seeking work,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,Yes – the person is a current or former regular,No – they left up to and including 5 years ago,Yes,No,Yes,Fully wheelchair accessible housing,1,0,0,0,0,0,No,Yes,0,0,1,0,0,0,0,0,0,0,Less than 1 year,1 year but under 2 years,Loss of tied accommodation,,Other supported housing,2,No,Yes,TN23 6LZ,Yes,No,Ashford,E07000105,Yes,0,1,0,0,0,0,0,1,,Tenant applied directly (no referral or nomination),Yes,0,68,Weekly,,Universal Credit housing element,1,All,,0,Every 2 weeks,,,,,200.0,100.0,,50.0,25.0,40.0,20.0,35.0,17.5,325.0,162.5,Yes,Yes,12.0,6.0,,,,,,,,,,,,,,,,,,,,
+id,status,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,old_id,old_form_id,collection_start_year,owning_organisation_name,managing_organisation_name,needstype,lettype,renewal,startdate,renttype,rent_type_detail,irproduct,irproduct_other,lar,tenancycode,propcode,uprn_known,uprn,uprn_confirmed,address_line1,address_line2,town_or_city,county,postcode_full,is_la_inferred,la_label,la,first_time_property_let_as_social_housing,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,vacdays,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,hhmemb,pregnancy_value_check,age1_known,refused,hhtype,totchild,totelder,totadult,age1,retirement_value_check,sex1,ethnic_group,ethnic,national,ecstat1,details_known_2,relat2,age2_known,age2,sex2,ecstat2,details_known_3,relat3,age3_known,age3,sex3,ecstat3,details_known_4,relat4,age4_known,age4,sex4,ecstat4,details_known_5,relat5,age5_known,age5,sex5,ecstat5,details_known_6,relat6,age6_known,age6,sex6,ecstat6,details_known_7,relat7,age7_known,age7,sex7,ecstat7,details_known_8,relat8,age8_known,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,new_old,homeless,ppcodenk,ppostcode_full,previous_la_known,is_previous_la_inferred,prevloc_label,prevloc,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,letting_allocation_unknown,referral,referral_value_check,net_income_known,incref,earnings,incfreq,net_income_value_check,hb,has_benefits,benefits,household_charge,nocharge,period,is_carehome,chcharge,wchchrg,carehome_charges_value_check,brent,wrent,rent_value_check,scharge,wscharge,pscharge,wpschrge,supcharg,wsupchrg,tcharge,wtcharge,hbrentshortfall,tshortfall_known,tshortfall,wtshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_admin_district,location_startdate
+,completed,s.port@jeemayle.com,false,2023-06-26T00:00:00+01:00,,2023-06-26T00:00:00+01:00,single log,,,2023,DLUHC,DLUHC,General needs,7,No,2023-06-26T00:00:00+01:00,2,Affordable Rent,,,,HIJKLMN,ABCDEFG,No,,,fake address,,London,,NW9 5LL,No,Barnet,E09000003,No,Affordable rent basis,Tenant abandoned property,2,2,House,Purpose built,Yes,3,2023-06-24T00:00:00+01:00,,,Yes,2023-06-25T00:00:00+01:00,,Don’t know,Yes,Assured Shorthold Tenancy (AST) – Fixed term,,2,,1,2,,Yes,0,4,0,0,2,35,,Female,White,Irish,Tenant prefers not to say,Other,Yes,Partner,Yes,32,Male,Not seeking work,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,Yes – the person is a current or former regular,No – they left up to and including 5 years ago,Yes,No,Yes,Fully wheelchair accessible housing,1,0,0,0,0,0,No,Yes,0,0,1,0,0,0,0,0,0,0,Less than 1 year,1 year but under 2 years,Loss of tied accommodation,,Other supported housing,2,No,Yes,TN23 6LZ,Yes,No,Ashford,E07000105,Yes,0,1,0,0,0,0,0,1,,Tenant applied directly (no referral or nomination),,Yes,0,68,Weekly,,Universal Credit housing element,1,All,,0,Every 2 weeks,,,,,200.0,100.0,,50.0,25.0,40.0,20.0,35.0,17.5,325.0,162.5,Yes,Yes,12.0,6.0,,,,,,,,,,,,,,,,,,,,
diff --git a/spec/fixtures/files/lettings_log_csv_export_non_support_codes.csv b/spec/fixtures/files/lettings_log_csv_export_non_support_codes.csv
index 8847034ee..6908e829b 100644
--- a/spec/fixtures/files/lettings_log_csv_export_non_support_codes.csv
+++ b/spec/fixtures/files/lettings_log_csv_export_non_support_codes.csv
@@ -1,2 +1,2 @@
-id,status,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,collection_start_year,owning_organisation_name,managing_organisation_name,lettype,renewal,startdate,irproduct,irproduct_other,lar,tenancycode,propcode,uprn_known,uprn,address_line1,address_line2,town_or_city,county,postcode_full,la_label,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,refused,age1,sex1,ethnic_group,ethnic,national,ecstat1,relat2,age2,sex2,ecstat2,relat3,age3,sex3,ecstat3,relat4,age4,sex4,ecstat4,relat5,age5,sex5,ecstat5,relat6,age6,sex6,ecstat6,relat7,age7,sex7,ecstat7,relat8,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,homeless,ppcodenk,ppostcode_full,prevloc_label,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,referral,incref,earnings,incfreq,hb,has_benefits,benefits,household_charge,nocharge,period,chcharge,wchchrg,carehome_charges_value_check,brent,scharge,pscharge,supcharg,tcharge,hbrentshortfall,tshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_admin_district,location_startdate
-,completed,choreographer@owtluk.com,false,2023-06-26T00:00:00+01:00,,2023-06-26T00:00:00+01:00,1,2023,DLUHC,DLUHC,7,0,2023-06-26T00:00:00+01:00,,,,HIJKLMN,ABCDEFG,0,,fake address,,London,,NW9 5LL,Barnet,2,6,2,2,7,1,1,3,2023-06-24T00:00:00+01:00,,1,2023-06-25T00:00:00+01:00,,3,1,4,,2,,1,0,35,F,0,2,13,0,P,32,M,6,,,,,,,,,,,,,,,,,,,,,,,,,1,4,1,2,1,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,2,7,4,,6,1,0,TN23 6LZ,Ashford,1,0,1,0,0,0,0,0,1,2,0,68,1,6,1,1,,0,2,,,,200.0,50.0,40.0,35.0,325.0,1,12.0,,,,,,,,,,,,,,,,,,,,
+id,status,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,collection_start_year,owning_organisation_name,managing_organisation_name,lettype,renewal,startdate,irproduct,irproduct_other,lar,tenancycode,propcode,uprn_known,uprn,address_line1,address_line2,town_or_city,county,postcode_full,la_label,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,refused,age1,sex1,ethnic_group,ethnic,national,ecstat1,relat2,age2,sex2,ecstat2,relat3,age3,sex3,ecstat3,relat4,age4,sex4,ecstat4,relat5,age5,sex5,ecstat5,relat6,age6,sex6,ecstat6,relat7,age7,sex7,ecstat7,relat8,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,homeless,ppcodenk,ppostcode_full,prevloc_label,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,referral,referral_value_check,incref,earnings,incfreq,hb,has_benefits,benefits,household_charge,nocharge,period,chcharge,wchchrg,carehome_charges_value_check,brent,scharge,pscharge,supcharg,tcharge,hbrentshortfall,tshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_admin_district,location_startdate
+,completed,choreographer@owtluk.com,false,2023-06-26T00:00:00+01:00,,2023-06-26T00:00:00+01:00,1,2023,DLUHC,DLUHC,7,0,2023-06-26T00:00:00+01:00,,,,HIJKLMN,ABCDEFG,0,,fake address,,London,,NW9 5LL,Barnet,2,6,2,2,7,1,1,3,2023-06-24T00:00:00+01:00,,1,2023-06-25T00:00:00+01:00,,3,1,4,,2,,1,0,35,F,0,2,13,0,P,32,M,6,,,,,,,,,,,,,,,,,,,,,,,,,1,4,1,2,1,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,2,7,4,,6,1,0,TN23 6LZ,Ashford,1,0,1,0,0,0,0,0,1,2,,0,68,1,6,1,1,,0,2,,,,200.0,50.0,40.0,35.0,325.0,1,12.0,,,,,,,,,,,,,,,,,,,,
diff --git a/spec/fixtures/files/lettings_log_csv_export_non_support_labels.csv b/spec/fixtures/files/lettings_log_csv_export_non_support_labels.csv
index bb2230668..fd70049dc 100644
--- a/spec/fixtures/files/lettings_log_csv_export_non_support_labels.csv
+++ b/spec/fixtures/files/lettings_log_csv_export_non_support_labels.csv
@@ -1,2 +1,2 @@
-id,status,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,collection_start_year,owning_organisation_name,managing_organisation_name,lettype,renewal,startdate,irproduct,irproduct_other,lar,tenancycode,propcode,uprn_known,uprn,address_line1,address_line2,town_or_city,county,postcode_full,la_label,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,refused,age1,sex1,ethnic_group,ethnic,national,ecstat1,relat2,age2,sex2,ecstat2,relat3,age3,sex3,ecstat3,relat4,age4,sex4,ecstat4,relat5,age5,sex5,ecstat5,relat6,age6,sex6,ecstat6,relat7,age7,sex7,ecstat7,relat8,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,homeless,ppcodenk,ppostcode_full,prevloc_label,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,referral,incref,earnings,incfreq,hb,has_benefits,benefits,household_charge,nocharge,period,chcharge,wchchrg,carehome_charges_value_check,brent,scharge,pscharge,supcharg,tcharge,hbrentshortfall,tshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_admin_district,location_startdate
-,completed,choreographer@owtluk.com,false,2023-06-26T00:00:00+01:00,,2023-06-26T00:00:00+01:00,single log,2023,DLUHC,DLUHC,7,No,2023-06-26T00:00:00+01:00,,,,HIJKLMN,ABCDEFG,No,,fake address,,London,,NW9 5LL,Barnet,Affordable rent basis,Tenant abandoned property,2,2,House,Purpose built,Yes,3,2023-06-24T00:00:00+01:00,,Yes,2023-06-25T00:00:00+01:00,,Don’t know,Yes,Assured Shorthold Tenancy (AST) – Fixed term,,2,,1,0,35,Female,White,Irish,Tenant prefers not to say,Other,Partner,32,Male,Not seeking work,,,,,,,,,,,,,,,,,,,,,,,,,Yes – the person is a current or former regular,No – they left up to and including 5 years ago,Yes,No,Yes,Fully wheelchair accessible housing,1,0,0,0,0,0,No,Yes,0,0,1,0,0,0,0,0,0,0,Less than 1 year,1 year but under 2 years,Loss of tied accommodation,,Other supported housing,No,Yes,TN23 6LZ,Ashford,Yes,0,1,0,0,0,0,0,1,Tenant applied directly (no referral or nomination),0,68,Weekly,Universal Credit housing element,1,All,,0,Every 2 weeks,,,,200.0,50.0,40.0,35.0,325.0,Yes,12.0,,,,,,,,,,,,,,,,,,,,
+id,status,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,collection_start_year,owning_organisation_name,managing_organisation_name,lettype,renewal,startdate,irproduct,irproduct_other,lar,tenancycode,propcode,uprn_known,uprn,address_line1,address_line2,town_or_city,county,postcode_full,la_label,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,refused,age1,sex1,ethnic_group,ethnic,national,ecstat1,relat2,age2,sex2,ecstat2,relat3,age3,sex3,ecstat3,relat4,age4,sex4,ecstat4,relat5,age5,sex5,ecstat5,relat6,age6,sex6,ecstat6,relat7,age7,sex7,ecstat7,relat8,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,homeless,ppcodenk,ppostcode_full,prevloc_label,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,referral,referral_value_check,incref,earnings,incfreq,hb,has_benefits,benefits,household_charge,nocharge,period,chcharge,wchchrg,carehome_charges_value_check,brent,scharge,pscharge,supcharg,tcharge,hbrentshortfall,tshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_admin_district,location_startdate
+,completed,choreographer@owtluk.com,false,2023-06-26T00:00:00+01:00,,2023-06-26T00:00:00+01:00,single log,2023,DLUHC,DLUHC,7,No,2023-06-26T00:00:00+01:00,,,,HIJKLMN,ABCDEFG,No,,fake address,,London,,NW9 5LL,Barnet,Affordable rent basis,Tenant abandoned property,2,2,House,Purpose built,Yes,3,2023-06-24T00:00:00+01:00,,Yes,2023-06-25T00:00:00+01:00,,Don’t know,Yes,Assured Shorthold Tenancy (AST) – Fixed term,,2,,1,0,35,Female,White,Irish,Tenant prefers not to say,Other,Partner,32,Male,Not seeking work,,,,,,,,,,,,,,,,,,,,,,,,,Yes – the person is a current or former regular,No – they left up to and including 5 years ago,Yes,No,Yes,Fully wheelchair accessible housing,1,0,0,0,0,0,No,Yes,0,0,1,0,0,0,0,0,0,0,Less than 1 year,1 year but under 2 years,Loss of tied accommodation,,Other supported housing,No,Yes,TN23 6LZ,Ashford,Yes,0,1,0,0,0,0,0,1,Tenant applied directly (no referral or nomination),,0,68,Weekly,Universal Credit housing element,1,All,,0,Every 2 weeks,,,,200.0,50.0,40.0,35.0,325.0,Yes,12.0,,,,,,,,,,,,,,,,,,,,
diff --git a/spec/models/form/lettings/subsections/household_situation_spec.rb b/spec/models/form/lettings/subsections/household_situation_spec.rb
index 1b43a2280..803f873f8 100644
--- a/spec/models/form/lettings/subsections/household_situation_spec.rb
+++ b/spec/models/form/lettings/subsections/household_situation_spec.rb
@@ -30,6 +30,7 @@ RSpec.describe Form::Lettings::Subsections::HouseholdSituation, type: :model do
         referral_prp
         referral_supported_housing
         referral_supported_housing_prp
+        referral_value_check
       ],
     )
   end
diff --git a/spec/models/validations/soft_validations_spec.rb b/spec/models/validations/soft_validations_spec.rb
index acffc9cac..0a267b9e9 100644
--- a/spec/models/validations/soft_validations_spec.rb
+++ b/spec/models/validations/soft_validations_spec.rb
@@ -345,4 +345,34 @@ RSpec.describe Validations::SoftValidations do
       expect(record).to be_care_home_charge_expected_not_provided
     end
   end
+
+  describe "#la_referral_for_general_needs?" do
+    it "returns false if needstype is 'Supported Housing'" do
+      record.needstype = 2
+      record.referral = 4
+
+      expect(record).not_to be_la_referral_for_general_needs
+    end
+
+    it "returns false if needstype is not given" do
+      record.needstype = nil
+      record.referral = 4
+
+      expect(record).not_to be_la_referral_for_general_needs
+    end
+
+    it "returns false if referral is not given" do
+      record.needstype = 1
+      record.referral = nil
+
+      expect(record).not_to be_la_referral_for_general_needs
+    end
+
+    it "returns true if needstype is 'General needs' and referral is 4" do
+      record.needstype = 1
+      record.referral = 4
+
+      expect(record).to be_la_referral_for_general_needs
+    end
+  end
 end
diff --git a/spec/services/imports/lettings_logs_import_service_spec.rb b/spec/services/imports/lettings_logs_import_service_spec.rb
index ac8dddccd..a9b0b9260 100644
--- a/spec/services/imports/lettings_logs_import_service_spec.rb
+++ b/spec/services/imports/lettings_logs_import_service_spec.rb
@@ -2049,5 +2049,24 @@ RSpec.describe Imports::LettingsLogsImportService do
         expect(lettings_log&.la).to eq("E06000047")
       end
     end
+
+    context "and the referral soft validation is triggered (referral_value_check)" do
+      let(:lettings_log_id) { "00d2343e-d5fa-4c89-8400-ec3854b0f2b4" }
+      let(:lettings_log_file) { open_file(fixture_directory, lettings_log_id) }
+      let(:lettings_log_xml) { Nokogiri::XML(lettings_log_file) }
+
+      before do
+        lettings_log_xml.at_xpath("//xmlns:DAY").content = "2"
+        lettings_log_xml.at_xpath("//xmlns:MONTH").content = "10"
+        lettings_log_xml.at_xpath("//xmlns:YEAR").content = "2022"
+        lettings_log_xml.at_xpath("//xmlns:Q16").content = 4
+      end
+
+      it "completes the log" do
+        lettings_log_service.send(:create_log, lettings_log_xml)
+        lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
+        expect(lettings_log.status).to eq("completed")
+      end
+    end
   end
 end

From 2f986fe11154b9ba1ab126224fb8fa0f18a7c200 Mon Sep 17 00:00:00 2001
From: kosiakkatrina <54268893+kosiakkatrina@users.noreply.github.com>
Date: Wed, 13 Sep 2023 17:54:30 +0100
Subject: [PATCH 32/33] CLDC-2795 Update charges validations (#1912)

* Update charges hard maximums

* Add charges value check fields

* Add 2023 charges value check pages and questions

* add scharge_over_soft_max method

* add LA cases

* Add pscharge_over_soft_max method

* Add supcharg_over_soft_max method

* Add pages to subsections (2023) and update content

* Confirm soft validations when importing

* csv tests

* Fix copy for 2022
---
 .../lettings/pages/pscharge_value_check.rb    |  24 +
 .../lettings/pages/scharge_value_check.rb     |  24 +
 .../lettings/pages/supcharg_value_check.rb    |  24 +
 .../questions/pscharge_value_check.rb         |  15 +
 .../lettings/questions/scharge_value_check.rb |  15 +
 .../questions/supcharg_value_check.rb         |  15 +
 .../subsections/income_and_benefits.rb        |   3 +
 .../validations/financial_validations.rb      |  24 +-
 app/models/validations/soft_validations.rb    |  36 +
 .../imports/lettings_logs_import_service.rb   |   5 +-
 config/forms/2022_2023.json                   | 135 ++++
 config/locales/en.yml                         |  34 +-
 ...20230913093443_add_charges_value_checks.rb |   9 +
 db/schema.rb                                  |   5 +-
 .../files/lettings_log_csv_export_codes.csv   |   4 +-
 .../files/lettings_log_csv_export_labels.csv  |   4 +-
 ...tings_log_csv_export_non_support_codes.csv |   4 +-
 ...ings_log_csv_export_non_support_labels.csv |   4 +-
 .../subsections/income_and_benefits_spec.rb   |   3 +
 .../validations/financial_validations_spec.rb | 144 ++--
 .../validations/soft_validations_spec.rb      | 642 ++++++++++++++++++
 .../lettings_logs_import_service_spec.rb      |  41 +-
 22 files changed, 1105 insertions(+), 109 deletions(-)
 create mode 100644 app/models/form/lettings/pages/pscharge_value_check.rb
 create mode 100644 app/models/form/lettings/pages/scharge_value_check.rb
 create mode 100644 app/models/form/lettings/pages/supcharg_value_check.rb
 create mode 100644 app/models/form/lettings/questions/pscharge_value_check.rb
 create mode 100644 app/models/form/lettings/questions/scharge_value_check.rb
 create mode 100644 app/models/form/lettings/questions/supcharg_value_check.rb
 create mode 100644 db/migrate/20230913093443_add_charges_value_checks.rb

diff --git a/app/models/form/lettings/pages/pscharge_value_check.rb b/app/models/form/lettings/pages/pscharge_value_check.rb
new file mode 100644
index 000000000..89cca160e
--- /dev/null
+++ b/app/models/form/lettings/pages/pscharge_value_check.rb
@@ -0,0 +1,24 @@
+class Form::Lettings::Pages::PschargeValueCheck < ::Form::Page
+  def initialize(id, hsh, subsection)
+    super
+    @id = "pscharge_value_check"
+    @depends_on = [{ "pscharge_over_soft_max?" => true }]
+    @title_text = {
+      "translation" => "soft_validations.pscharge.over_soft_max_title",
+      "arguments" => [{
+        "key" => "pscharge",
+        "label" => true,
+        "i18n_template" => "pscharge",
+      }],
+    }
+    @informative_text = I18n.t("soft_validations.charges.informative_text")
+  end
+
+  def questions
+    @questions ||= [Form::Lettings::Questions::PschargeValueCheck.new(nil, nil, self)]
+  end
+
+  def interruption_screen_question_ids
+    %w[period needstype pscharge]
+  end
+end
diff --git a/app/models/form/lettings/pages/scharge_value_check.rb b/app/models/form/lettings/pages/scharge_value_check.rb
new file mode 100644
index 000000000..72ad8039e
--- /dev/null
+++ b/app/models/form/lettings/pages/scharge_value_check.rb
@@ -0,0 +1,24 @@
+class Form::Lettings::Pages::SchargeValueCheck < ::Form::Page
+  def initialize(id, hsh, subsection)
+    super
+    @id = "scharge_value_check"
+    @depends_on = [{ "scharge_over_soft_max?" => true }]
+    @title_text = {
+      "translation" => "soft_validations.scharge.over_soft_max_title",
+      "arguments" => [{
+        "key" => "scharge",
+        "label" => true,
+        "i18n_template" => "scharge",
+      }],
+    }
+    @informative_text = I18n.t("soft_validations.charges.informative_text")
+  end
+
+  def questions
+    @questions ||= [Form::Lettings::Questions::SchargeValueCheck.new(nil, nil, self)]
+  end
+
+  def interruption_screen_question_ids
+    %w[period needstype scharge]
+  end
+end
diff --git a/app/models/form/lettings/pages/supcharg_value_check.rb b/app/models/form/lettings/pages/supcharg_value_check.rb
new file mode 100644
index 000000000..8155a28bb
--- /dev/null
+++ b/app/models/form/lettings/pages/supcharg_value_check.rb
@@ -0,0 +1,24 @@
+class Form::Lettings::Pages::SupchargValueCheck < ::Form::Page
+  def initialize(id, hsh, subsection)
+    super
+    @id = "supcharg_value_check"
+    @depends_on = [{ "supcharg_over_soft_max?" => true }]
+    @title_text = {
+      "translation" => "soft_validations.supcharg.over_soft_max_title",
+      "arguments" => [{
+        "key" => "supcharg",
+        "label" => true,
+        "i18n_template" => "supcharg",
+      }],
+    }
+    @informative_text = I18n.t("soft_validations.charges.informative_text")
+  end
+
+  def questions
+    @questions ||= [Form::Lettings::Questions::SupchargValueCheck.new(nil, nil, self)]
+  end
+
+  def interruption_screen_question_ids
+    %w[period needstype supcharg]
+  end
+end
diff --git a/app/models/form/lettings/questions/pscharge_value_check.rb b/app/models/form/lettings/questions/pscharge_value_check.rb
new file mode 100644
index 000000000..b44979ae1
--- /dev/null
+++ b/app/models/form/lettings/questions/pscharge_value_check.rb
@@ -0,0 +1,15 @@
+class Form::Lettings::Questions::PschargeValueCheck < ::Form::Question
+  def initialize(id, hsh, page)
+    super
+    @id = "pscharge_value_check"
+    @check_answer_label = "Personal service charge confirmation"
+    @header = "Are you sure?"
+    @type = "interruption_screen"
+    @check_answers_card_number = 0
+    @answer_options = ANSWER_OPTIONS
+    @hidden_in_check_answers = { "depends_on" => [{ "pscharge_value_check" => 0 }, { "pscharge_value_check" => 1 }] }
+    @hint_text = I18n.t("soft_validations.charges.hint_text")
+  end
+
+  ANSWER_OPTIONS = { "0" => { "value" => "Yes" }, "1" => { "value" => "No" } }.freeze
+end
diff --git a/app/models/form/lettings/questions/scharge_value_check.rb b/app/models/form/lettings/questions/scharge_value_check.rb
new file mode 100644
index 000000000..1946d603f
--- /dev/null
+++ b/app/models/form/lettings/questions/scharge_value_check.rb
@@ -0,0 +1,15 @@
+class Form::Lettings::Questions::SchargeValueCheck < ::Form::Question
+  def initialize(id, hsh, page)
+    super
+    @id = "scharge_value_check"
+    @check_answer_label = "Service charge confirmation"
+    @header = "Are you sure?"
+    @type = "interruption_screen"
+    @check_answers_card_number = 0
+    @answer_options = ANSWER_OPTIONS
+    @hidden_in_check_answers = { "depends_on" => [{ "scharge_value_check" => 0 }, { "scharge_value_check" => 1 }] }
+    @hint_text = I18n.t("soft_validations.charges.hint_text")
+  end
+
+  ANSWER_OPTIONS = { "0" => { "value" => "Yes" }, "1" => { "value" => "No" } }.freeze
+end
diff --git a/app/models/form/lettings/questions/supcharg_value_check.rb b/app/models/form/lettings/questions/supcharg_value_check.rb
new file mode 100644
index 000000000..978c5e3ef
--- /dev/null
+++ b/app/models/form/lettings/questions/supcharg_value_check.rb
@@ -0,0 +1,15 @@
+class Form::Lettings::Questions::SupchargValueCheck < ::Form::Question
+  def initialize(id, hsh, page)
+    super
+    @id = "supcharg_value_check"
+    @check_answer_label = "Support charge confirmation"
+    @header = "Are you sure?"
+    @type = "interruption_screen"
+    @check_answers_card_number = 0
+    @answer_options = ANSWER_OPTIONS
+    @hidden_in_check_answers = { "depends_on" => [{ "supcharg_value_check" => 0 }, { "supcharg_value_check" => 1 }] }
+    @hint_text = I18n.t("soft_validations.charges.hint_text")
+  end
+
+  ANSWER_OPTIONS = { "0" => { "value" => "Yes" }, "1" => { "value" => "No" } }.freeze
+end
diff --git a/app/models/form/lettings/subsections/income_and_benefits.rb b/app/models/form/lettings/subsections/income_and_benefits.rb
index 25f5d0ce4..1404ebf04 100644
--- a/app/models/form/lettings/subsections/income_and_benefits.rb
+++ b/app/models/form/lettings/subsections/income_and_benefits.rb
@@ -26,6 +26,9 @@ class Form::Lettings::Subsections::IncomeAndBenefits < ::Form::Subsection
       Form::Lettings::Pages::RentMonthly.new(nil, nil, self),
       Form::Lettings::Pages::MinRentValueCheck.new("brent_min_rent_value_check", nil, self, check_answers_card_number: 0),
       Form::Lettings::Pages::MaxRentValueCheck.new("brent_max_rent_value_check", nil, self, check_answers_card_number: 0),
+      Form::Lettings::Pages::SchargeValueCheck.new(nil, nil, self),
+      Form::Lettings::Pages::PschargeValueCheck.new(nil, nil, self),
+      Form::Lettings::Pages::SupchargValueCheck.new(nil, nil, self),
       Form::Lettings::Pages::Outstanding.new(nil, nil, self),
       Form::Lettings::Pages::OutstandingAmount.new(nil, nil, self),
     ].compact
diff --git a/app/models/validations/financial_validations.rb b/app/models/validations/financial_validations.rb
index 5a1775db4..29c22eb62 100644
--- a/app/models/validations/financial_validations.rb
+++ b/app/models/validations/financial_validations.rb
@@ -149,32 +149,32 @@ private
   CHARGE_MAXIMUMS = {
     scharge: {
       private_registered_provider: {
-        general_needs: 155,
-        supported_housing: 480,
+        general_needs: 800,
+        supported_housing: 800,
       },
       local_authority: {
-        general_needs: 155,
-        supported_housing: 365,
+        general_needs: 500,
+        supported_housing: 500,
       },
     },
     pscharge: {
       private_registered_provider: {
-        general_needs: 30,
-        supported_housing: 200,
+        general_needs: 700,
+        supported_housing: 700,
       },
       local_authority: {
-        general_needs: 35,
-        supported_housing: 75,
+        general_needs: 200,
+        supported_housing: 200,
       },
     },
     supcharg: {
       private_registered_provider: {
-        general_needs: 40,
-        supported_housing: 465,
+        general_needs: 800,
+        supported_housing: 800,
       },
       local_authority: {
-        general_needs: 60,
-        supported_housing: 120,
+        general_needs: 200,
+        supported_housing: 200,
       },
     },
   }.freeze
diff --git a/app/models/validations/soft_validations.rb b/app/models/validations/soft_validations.rb
index 303f525cd..5720c0f0d 100644
--- a/app/models/validations/soft_validations.rb
+++ b/app/models/validations/soft_validations.rb
@@ -97,6 +97,42 @@ module Validations::SoftValidations
     net_income_in_soft_max_range? ? "higher" : "lower"
   end
 
+  def scharge_over_soft_max?
+    return unless scharge && period && needstype
+    return if weekly_value(scharge).blank?
+
+    max = if needstype == 1
+            owning_organisation.provider_type == "LA" ? 25 : 35
+          else
+            owning_organisation.provider_type == "LA" ? 100 : 200
+          end
+    weekly_value(scharge) > max
+  end
+
+  def pscharge_over_soft_max?
+    return unless pscharge && period && needstype
+    return if weekly_value(pscharge).blank?
+
+    max = if needstype == 1
+            owning_organisation.provider_type == "LA" ? 25 : 35
+          else
+            owning_organisation.provider_type == "LA" ? 75 : 100
+          end
+    weekly_value(pscharge) > max
+  end
+
+  def supcharg_over_soft_max?
+    return unless supcharg && period && needstype
+    return if weekly_value(supcharg).blank?
+
+    max = if needstype == 1
+            owning_organisation.provider_type == "LA" ? 25 : 35
+          else
+            owning_organisation.provider_type == "LA" ? 75 : 85
+          end
+    weekly_value(supcharg) > max
+  end
+
 private
 
   def details_known_or_lead_tenant?(tenant_number)
diff --git a/app/services/imports/lettings_logs_import_service.rb b/app/services/imports/lettings_logs_import_service.rb
index 865f8417f..e8ed9a058 100644
--- a/app/services/imports/lettings_logs_import_service.rb
+++ b/app/services/imports/lettings_logs_import_service.rb
@@ -243,6 +243,9 @@ module Imports
       attributes["net_income_value_check"] = 0
       attributes["carehome_charges_value_check"] = 0
       attributes["referral_value_check"] = 0
+      attributes["scharge_value_check"] = 0
+      attributes["pscharge_value_check"] = 0
+      attributes["supcharg_value_check"] = 0
 
       # Sets the log creator
       owner_id = meta_field_value(xml_doc, "owner-user-id").strip
@@ -402,7 +405,7 @@ module Imports
     end
 
     def fields_not_present_in_softwire_data
-      %w[majorrepairs illness_type_0 tshortfall_known pregnancy_value_check retirement_value_check rent_value_check net_income_value_check major_repairs_date_value_check void_date_value_check carehome_charges_value_check referral_value_check housingneeds_type housingneeds_other created_by uprn_known uprn_confirmed]
+      %w[majorrepairs illness_type_0 tshortfall_known pregnancy_value_check retirement_value_check rent_value_check net_income_value_check major_repairs_date_value_check void_date_value_check carehome_charges_value_check referral_value_check scharge_value_check pscharge_value_check supcharg_value_check housingneeds_type housingneeds_other created_by uprn_known uprn_confirmed]
     end
 
     def check_status_completed(lettings_log, previous_status)
diff --git a/config/forms/2022_2023.json b/config/forms/2022_2023.json
index 0fea72f3b..ac2da385d 100644
--- a/config/forms/2022_2023.json
+++ b/config/forms/2022_2023.json
@@ -8364,6 +8364,141 @@
               },
               "interruption_screen_question_ids": ["brent", "startdate", "la", "beds", "rent_type", "needstype"]
             },
+            "scharge_value_check": {
+              "depends_on": [
+                {
+                  "scharge_over_soft_max?": true
+                }
+              ],
+              "title_text": {
+                "translation": "soft_validations.scharge.over_soft_max_title",
+                "arguments": [
+                  {
+                    "key": "scharge",
+                    "label": true,
+                    "i18n_template": "scharge"
+                  }
+                ]
+              },
+              "informative_text": {},
+              "questions": {
+                "scharge_value_check": {
+                  "check_answer_label": "Service charge confirmation",
+                  "hidden_in_check_answers": {
+                    "depends_on": [
+                      {
+                        "scharge_value_check": 0
+                      },
+                      {
+                        "scharge_value_check": 1
+                      }
+                    ]
+                  },
+                  "header": "Are you sure?",
+                  "hint_text": "This is higher than we would expect. Check:<ul class=\"govuk-body-l app-panel--interruption\"><li>the decimal point</li><li>the frequency, for example every week or every calendar month</li><li>the needs type</li></ul>",
+                  "type": "interruption_screen",
+                  "answer_options": {
+                    "0": {
+                      "value": "Yes"
+                    },
+                    "1": {
+                      "value": "No"
+                    }
+                  }
+                }
+              },
+              "interruption_screen_question_ids": ["period", "needstype", "scharge"]
+            },
+            "pscharge_value_check": {
+              "depends_on": [
+                {
+                  "pscharge_over_soft_max?": true
+                }
+              ],
+              "title_text": {
+                "translation": "soft_validations.pscharge.over_soft_max_title",
+                "arguments": [
+                  {
+                    "key": "pscharge",
+                    "label": true,
+                    "i18n_template": "pscharge"
+                  }
+                ]
+              },
+              "informative_text": {},
+              "questions": {
+                "pscharge_value_check": {
+                  "check_answer_label": "Personal service charge confirmation",
+                  "hidden_in_check_answers": {
+                    "depends_on": [
+                      {
+                        "pscharge_value_check": 0
+                      },
+                      {
+                        "pscharge_value_check": 1
+                      }
+                    ]
+                  },
+                  "header": "Are you sure?",
+                  "hint_text": "This is higher than we would expect. Check:<ul class=\"govuk-body-l app-panel--interruption\"><li>the decimal point</li><li>the frequency, for example every week or every calendar month</li><li>the needs type</li></ul>",
+                  "type": "interruption_screen",
+                  "answer_options": {
+                    "0": {
+                      "value": "Yes"
+                    },
+                    "1": {
+                      "value": "No"
+                    }
+                  }
+                }
+              },
+              "interruption_screen_question_ids": ["period", "needstype", "pscharge"]
+            },
+            "supcharg_value_check": {
+              "depends_on": [
+                {
+                  "supcharg_over_soft_max?": true
+                }
+              ],
+              "title_text": {
+                "translation": "soft_validations.supcharg.over_soft_max_title",
+                "arguments": [
+                  {
+                    "key": "supcharg",
+                    "label": true,
+                    "i18n_template": "supcharg"
+                  }
+                ]
+              },
+              "informative_text": {},
+              "questions": {
+                "supcharg_value_check": {
+                  "check_answer_label": "Personal service charge confirmation",
+                  "hidden_in_check_answers": {
+                    "depends_on": [
+                      {
+                        "supcharg_value_check": 0
+                      },
+                      {
+                        "supcharg_value_check": 1
+                      }
+                    ]
+                  },
+                  "header": "Are you sure?",
+                  "hint_text": "This is higher than we would expect. Check:<ul class=\"govuk-body-l app-panel--interruption\"><li>the decimal point</li><li>the frequency, for example every week or every calendar month</li><li>the needs type</li></ul>",
+                  "type": "interruption_screen",
+                  "answer_options": {
+                    "0": {
+                      "value": "Yes"
+                    },
+                    "1": {
+                      "value": "No"
+                    }
+                  }
+                }
+              },
+              "interruption_screen_question_ids": ["period", "needstype", "supcharg"]
+            },
             "outstanding": {
               "header": "",
               "description": "",
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 8f53b4590..c982f9592 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -369,25 +369,25 @@ en:
         less_than_shortfall: "Enter an amount that is more than the shortfall in basic rent"
         scharge:
           private_registered_provider:
-            general_needs: "Enter a value for the service charge between £0 and £155 per week if the landlord is a private registered provider and it is a general needs letting"
-            supported_housing: "Enter a value for the service charge between £0 and £480 per week if the landlord is a private registered provider and it is a supported housing letting"
+            general_needs: "Enter a value for the service charge between £0 and £800 per week if the landlord is a private registered provider and it is a general needs letting"
+            supported_housing: "Enter a value for the service charge between £0 and £800 per week if the landlord is a private registered provider and it is a supported housing letting"
           local_authority:
-            general_needs: "Enter a value for the service charge between £0 and £155 per week if the landlord is a local authority and it is a general needs letting"
-            supported_housing: "Enter a value for the service charge between £0 and £365 per week if the landlord is a local authority and it is a supported housing letting"
+            general_needs: "Enter a value for the service charge between £0 and £500 per week if the landlord is a local authority and it is a general needs letting"
+            supported_housing: "Enter a value for the service charge between £0 and £500 per week if the landlord is a local authority and it is a supported housing letting"
         pscharge:
           private_registered_provider:
-            general_needs: "Enter a value for the personal service charge between £0 and £30 per week if the landlord is a private registered provider and it is a general needs letting"
-            supported_housing: "Enter a value for the personal service charge between £0 and £200 per week if the landlord is a private registered provider and it is a supported housing letting"
+            general_needs: "Enter a value for the personal service charge between £0 and £700 per week if the landlord is a private registered provider and it is a general needs letting"
+            supported_housing: "Enter a value for the personal service charge between £0 and £700 per week if the landlord is a private registered provider and it is a supported housing letting"
           local_authority:
-            general_needs: "Enter a value for the personal service charge between £0 and £35 per week if the landlord is a local authority and it is a general needs letting"
-            supported_housing: "Enter a value for the personal service charge between £0 and £75 per week if the landlord is a local authority and it is a supported housing letting"
+            general_needs: "Enter a value for the personal service charge between £0 and £200 per week if the landlord is a local authority and it is a general needs letting"
+            supported_housing: "Enter a value for the personal service charge between £0 and £200 per week if the landlord is a local authority and it is a supported housing letting"
         supcharg:
           private_registered_provider:
-            general_needs: "Enter a value for the support charge between £0 and £40 per week if the landlord is a private registered provider and it is a general needs letting"
-            supported_housing: "Enter a value for the support charge between £0 and £465 per week if the landlord is a private registered provider and it is a supported housing letting"
+            general_needs: "Enter a value for the support charge between £0 and £800 per week if the landlord is a private registered provider and it is a general needs letting"
+            supported_housing: "Enter a value for the support charge between £0 and £800 per week if the landlord is a private registered provider and it is a supported housing letting"
           local_authority:
-            general_needs: "Enter a value for the support charge between £0 and £60 per week if the landlord is a local authority and it is a general needs letting"
-            supported_housing: "Enter a value for the support charge between £0 and £120 per week if the landlord is a local authority and it is a supported housing letting"
+            general_needs: "Enter a value for the support charge between £0 and £200 per week if the landlord is a local authority and it is a general needs letting"
+            supported_housing: "Enter a value for the support charge between £0 and £200 per week if the landlord is a local authority and it is a supported housing letting"
       ecstat:
         over_hard_max: "Net income of %{hard_max} per week is too high given the tenant’s working situation"
       brent:
@@ -709,6 +709,16 @@ Make sure these answers are correct."
     referral:
       title_text: "Are you sure?"
       hint_text: "This is a general needs log, and this referral type is for supported housing."
+    scharge:
+      over_soft_max_title: "You told us the service charge is %{scharge}"
+    pscharge:
+      over_soft_max_title: "You told us the personal service charge is %{pscharge}"
+    supcharg:
+      over_soft_max_title: "You told us the support charge is %{supcharg}"
+    charges:
+      informative_text: "This is higher than we would expect."
+      hint_text: "Check the following:<ul class=\"govuk-body-l app-panel--interruption\"><li>the decimal point</li><li>the frequency, for example every week or every calendar month</li><li>the needs type</li></ul>"
+  
 
   devise:
     email:
diff --git a/db/migrate/20230913093443_add_charges_value_checks.rb b/db/migrate/20230913093443_add_charges_value_checks.rb
new file mode 100644
index 000000000..c220b0998
--- /dev/null
+++ b/db/migrate/20230913093443_add_charges_value_checks.rb
@@ -0,0 +1,9 @@
+class AddChargesValueChecks < ActiveRecord::Migration[7.0]
+  def change
+    change_table :lettings_logs, bulk: true do |t|
+      t.column :supcharg_value_check, :integer
+      t.column :scharge_value_check, :integer
+      t.column :pscharge_value_check, :integer
+    end
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 26aef31ac..e353ff9c7 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2023_09_12_115233) do
+ActiveRecord::Schema[7.0].define(version: 2023_09_13_093443) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
@@ -299,6 +299,9 @@ ActiveRecord::Schema[7.0].define(version: 2023_09_12_115233) do
     t.integer "creation_method", default: 1
     t.datetime "values_updated_at"
     t.integer "referral_value_check"
+    t.integer "supcharg_value_check"
+    t.integer "scharge_value_check"
+    t.integer "pscharge_value_check"
     t.index ["bulk_upload_id"], name: "index_lettings_logs_on_bulk_upload_id"
     t.index ["created_by_id"], name: "index_lettings_logs_on_created_by_id"
     t.index ["location_id"], name: "index_lettings_logs_on_location_id"
diff --git a/spec/fixtures/files/lettings_log_csv_export_codes.csv b/spec/fixtures/files/lettings_log_csv_export_codes.csv
index 6076d253b..41e8cadde 100644
--- a/spec/fixtures/files/lettings_log_csv_export_codes.csv
+++ b/spec/fixtures/files/lettings_log_csv_export_codes.csv
@@ -1,2 +1,2 @@
-id,status,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,old_id,old_form_id,collection_start_year,owning_organisation_name,managing_organisation_name,needstype,lettype,renewal,startdate,renttype,rent_type_detail,irproduct,irproduct_other,lar,tenancycode,propcode,uprn_known,uprn,uprn_confirmed,address_line1,address_line2,town_or_city,county,postcode_full,is_la_inferred,la_label,la,first_time_property_let_as_social_housing,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,vacdays,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,hhmemb,pregnancy_value_check,age1_known,refused,hhtype,totchild,totelder,totadult,age1,retirement_value_check,sex1,ethnic_group,ethnic,national,ecstat1,details_known_2,relat2,age2_known,age2,sex2,ecstat2,details_known_3,relat3,age3_known,age3,sex3,ecstat3,details_known_4,relat4,age4_known,age4,sex4,ecstat4,details_known_5,relat5,age5_known,age5,sex5,ecstat5,details_known_6,relat6,age6_known,age6,sex6,ecstat6,details_known_7,relat7,age7_known,age7,sex7,ecstat7,details_known_8,relat8,age8_known,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,new_old,homeless,ppcodenk,ppostcode_full,previous_la_known,is_previous_la_inferred,prevloc_label,prevloc,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,letting_allocation_unknown,referral,referral_value_check,net_income_known,incref,earnings,incfreq,net_income_value_check,hb,has_benefits,benefits,household_charge,nocharge,period,is_carehome,chcharge,wchchrg,carehome_charges_value_check,brent,wrent,rent_value_check,scharge,wscharge,pscharge,wpschrge,supcharg,wsupchrg,tcharge,wtcharge,hbrentshortfall,tshortfall_known,tshortfall,wtshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_admin_district,location_startdate
-,completed,s.port@jeemayle.com,false,2023-06-26T00:00:00+01:00,,2023-06-26T00:00:00+01:00,1,,,2023,DLUHC,DLUHC,1,7,0,2023-06-26T00:00:00+01:00,2,1,,,,HIJKLMN,ABCDEFG,0,,,fake address,,London,,NW9 5LL,false,Barnet,E09000003,0,2,6,2,2,7,1,1,3,2023-06-24T00:00:00+01:00,,,1,2023-06-25T00:00:00+01:00,,3,1,4,,2,,1,2,,0,0,4,0,0,2,35,,F,0,2,13,0,0,P,0,32,M,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,4,1,2,1,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,2,7,4,,6,2,1,0,TN23 6LZ,1,false,Ashford,E07000105,1,0,1,0,0,0,0,0,1,,2,,0,0,68,1,,6,1,1,,0,2,,,,,200.0,100.0,,50.0,25.0,40.0,20.0,35.0,17.5,325.0,162.5,1,0,12.0,6.0,,,,,,,,,,,,,,,,,,,,
+id,status,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,old_id,old_form_id,collection_start_year,owning_organisation_name,managing_organisation_name,needstype,lettype,renewal,startdate,renttype,rent_type_detail,irproduct,irproduct_other,lar,tenancycode,propcode,uprn_known,uprn,uprn_confirmed,address_line1,address_line2,town_or_city,county,postcode_full,is_la_inferred,la_label,la,first_time_property_let_as_social_housing,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,vacdays,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,hhmemb,pregnancy_value_check,age1_known,refused,hhtype,totchild,totelder,totadult,age1,retirement_value_check,sex1,ethnic_group,ethnic,national,ecstat1,details_known_2,relat2,age2_known,age2,sex2,ecstat2,details_known_3,relat3,age3_known,age3,sex3,ecstat3,details_known_4,relat4,age4_known,age4,sex4,ecstat4,details_known_5,relat5,age5_known,age5,sex5,ecstat5,details_known_6,relat6,age6_known,age6,sex6,ecstat6,details_known_7,relat7,age7_known,age7,sex7,ecstat7,details_known_8,relat8,age8_known,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,new_old,homeless,ppcodenk,ppostcode_full,previous_la_known,is_previous_la_inferred,prevloc_label,prevloc,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,letting_allocation_unknown,referral,referral_value_check,net_income_known,incref,earnings,incfreq,net_income_value_check,hb,has_benefits,benefits,household_charge,nocharge,period,is_carehome,chcharge,wchchrg,carehome_charges_value_check,brent,wrent,rent_value_check,scharge,wscharge,pscharge,wpschrge,supcharg,wsupchrg,tcharge,wtcharge,scharge_value_check,pscharge_value_check,supcharg_value_check,hbrentshortfall,tshortfall_known,tshortfall,wtshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_admin_district,location_startdate
+,completed,s.port@jeemayle.com,false,2023-06-26T00:00:00+01:00,,2023-06-26T00:00:00+01:00,1,,,2023,DLUHC,DLUHC,1,7,0,2023-06-26T00:00:00+01:00,2,1,,,,HIJKLMN,ABCDEFG,0,,,fake address,,London,,NW9 5LL,false,Barnet,E09000003,0,2,6,2,2,7,1,1,3,2023-06-24T00:00:00+01:00,,,1,2023-06-25T00:00:00+01:00,,3,1,4,,2,,1,2,,0,0,4,0,0,2,35,,F,0,2,13,0,0,P,0,32,M,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,4,1,2,1,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,2,7,4,,6,2,1,0,TN23 6LZ,1,false,Ashford,E07000105,1,0,1,0,0,0,0,0,1,,2,,0,0,68,1,,6,1,1,,0,2,,,,,200.0,100.0,,50.0,25.0,40.0,20.0,35.0,17.5,325.0,162.5,,,,1,0,12.0,6.0,,,,,,,,,,,,,,,,,,,,
diff --git a/spec/fixtures/files/lettings_log_csv_export_labels.csv b/spec/fixtures/files/lettings_log_csv_export_labels.csv
index e553083c1..729f1a63d 100644
--- a/spec/fixtures/files/lettings_log_csv_export_labels.csv
+++ b/spec/fixtures/files/lettings_log_csv_export_labels.csv
@@ -1,2 +1,2 @@
-id,status,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,old_id,old_form_id,collection_start_year,owning_organisation_name,managing_organisation_name,needstype,lettype,renewal,startdate,renttype,rent_type_detail,irproduct,irproduct_other,lar,tenancycode,propcode,uprn_known,uprn,uprn_confirmed,address_line1,address_line2,town_or_city,county,postcode_full,is_la_inferred,la_label,la,first_time_property_let_as_social_housing,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,vacdays,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,hhmemb,pregnancy_value_check,age1_known,refused,hhtype,totchild,totelder,totadult,age1,retirement_value_check,sex1,ethnic_group,ethnic,national,ecstat1,details_known_2,relat2,age2_known,age2,sex2,ecstat2,details_known_3,relat3,age3_known,age3,sex3,ecstat3,details_known_4,relat4,age4_known,age4,sex4,ecstat4,details_known_5,relat5,age5_known,age5,sex5,ecstat5,details_known_6,relat6,age6_known,age6,sex6,ecstat6,details_known_7,relat7,age7_known,age7,sex7,ecstat7,details_known_8,relat8,age8_known,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,new_old,homeless,ppcodenk,ppostcode_full,previous_la_known,is_previous_la_inferred,prevloc_label,prevloc,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,letting_allocation_unknown,referral,referral_value_check,net_income_known,incref,earnings,incfreq,net_income_value_check,hb,has_benefits,benefits,household_charge,nocharge,period,is_carehome,chcharge,wchchrg,carehome_charges_value_check,brent,wrent,rent_value_check,scharge,wscharge,pscharge,wpschrge,supcharg,wsupchrg,tcharge,wtcharge,hbrentshortfall,tshortfall_known,tshortfall,wtshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_admin_district,location_startdate
-,completed,s.port@jeemayle.com,false,2023-06-26T00:00:00+01:00,,2023-06-26T00:00:00+01:00,single log,,,2023,DLUHC,DLUHC,General needs,7,No,2023-06-26T00:00:00+01:00,2,Affordable Rent,,,,HIJKLMN,ABCDEFG,No,,,fake address,,London,,NW9 5LL,No,Barnet,E09000003,No,Affordable rent basis,Tenant abandoned property,2,2,House,Purpose built,Yes,3,2023-06-24T00:00:00+01:00,,,Yes,2023-06-25T00:00:00+01:00,,Don’t know,Yes,Assured Shorthold Tenancy (AST) – Fixed term,,2,,1,2,,Yes,0,4,0,0,2,35,,Female,White,Irish,Tenant prefers not to say,Other,Yes,Partner,Yes,32,Male,Not seeking work,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,Yes – the person is a current or former regular,No – they left up to and including 5 years ago,Yes,No,Yes,Fully wheelchair accessible housing,1,0,0,0,0,0,No,Yes,0,0,1,0,0,0,0,0,0,0,Less than 1 year,1 year but under 2 years,Loss of tied accommodation,,Other supported housing,2,No,Yes,TN23 6LZ,Yes,No,Ashford,E07000105,Yes,0,1,0,0,0,0,0,1,,Tenant applied directly (no referral or nomination),,Yes,0,68,Weekly,,Universal Credit housing element,1,All,,0,Every 2 weeks,,,,,200.0,100.0,,50.0,25.0,40.0,20.0,35.0,17.5,325.0,162.5,Yes,Yes,12.0,6.0,,,,,,,,,,,,,,,,,,,,
+id,status,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,old_id,old_form_id,collection_start_year,owning_organisation_name,managing_organisation_name,needstype,lettype,renewal,startdate,renttype,rent_type_detail,irproduct,irproduct_other,lar,tenancycode,propcode,uprn_known,uprn,uprn_confirmed,address_line1,address_line2,town_or_city,county,postcode_full,is_la_inferred,la_label,la,first_time_property_let_as_social_housing,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,vacdays,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,hhmemb,pregnancy_value_check,age1_known,refused,hhtype,totchild,totelder,totadult,age1,retirement_value_check,sex1,ethnic_group,ethnic,national,ecstat1,details_known_2,relat2,age2_known,age2,sex2,ecstat2,details_known_3,relat3,age3_known,age3,sex3,ecstat3,details_known_4,relat4,age4_known,age4,sex4,ecstat4,details_known_5,relat5,age5_known,age5,sex5,ecstat5,details_known_6,relat6,age6_known,age6,sex6,ecstat6,details_known_7,relat7,age7_known,age7,sex7,ecstat7,details_known_8,relat8,age8_known,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,new_old,homeless,ppcodenk,ppostcode_full,previous_la_known,is_previous_la_inferred,prevloc_label,prevloc,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,letting_allocation_unknown,referral,referral_value_check,net_income_known,incref,earnings,incfreq,net_income_value_check,hb,has_benefits,benefits,household_charge,nocharge,period,is_carehome,chcharge,wchchrg,carehome_charges_value_check,brent,wrent,rent_value_check,scharge,wscharge,pscharge,wpschrge,supcharg,wsupchrg,tcharge,wtcharge,scharge_value_check,pscharge_value_check,supcharg_value_check,hbrentshortfall,tshortfall_known,tshortfall,wtshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_admin_district,location_startdate
+,completed,s.port@jeemayle.com,false,2023-06-26T00:00:00+01:00,,2023-06-26T00:00:00+01:00,single log,,,2023,DLUHC,DLUHC,General needs,7,No,2023-06-26T00:00:00+01:00,2,Affordable Rent,,,,HIJKLMN,ABCDEFG,No,,,fake address,,London,,NW9 5LL,No,Barnet,E09000003,No,Affordable rent basis,Tenant abandoned property,2,2,House,Purpose built,Yes,3,2023-06-24T00:00:00+01:00,,,Yes,2023-06-25T00:00:00+01:00,,Don’t know,Yes,Assured Shorthold Tenancy (AST) – Fixed term,,2,,1,2,,Yes,0,4,0,0,2,35,,Female,White,Irish,Tenant prefers not to say,Other,Yes,Partner,Yes,32,Male,Not seeking work,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,Yes – the person is a current or former regular,No – they left up to and including 5 years ago,Yes,No,Yes,Fully wheelchair accessible housing,1,0,0,0,0,0,No,Yes,0,0,1,0,0,0,0,0,0,0,Less than 1 year,1 year but under 2 years,Loss of tied accommodation,,Other supported housing,2,No,Yes,TN23 6LZ,Yes,No,Ashford,E07000105,Yes,0,1,0,0,0,0,0,1,,Tenant applied directly (no referral or nomination),,Yes,0,68,Weekly,,Universal Credit housing element,1,All,,0,Every 2 weeks,,,,,200.0,100.0,,50.0,25.0,40.0,20.0,35.0,17.5,325.0,162.5,,,,Yes,Yes,12.0,6.0,,,,,,,,,,,,,,,,,,,,
diff --git a/spec/fixtures/files/lettings_log_csv_export_non_support_codes.csv b/spec/fixtures/files/lettings_log_csv_export_non_support_codes.csv
index 6908e829b..46d68db05 100644
--- a/spec/fixtures/files/lettings_log_csv_export_non_support_codes.csv
+++ b/spec/fixtures/files/lettings_log_csv_export_non_support_codes.csv
@@ -1,2 +1,2 @@
-id,status,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,collection_start_year,owning_organisation_name,managing_organisation_name,lettype,renewal,startdate,irproduct,irproduct_other,lar,tenancycode,propcode,uprn_known,uprn,address_line1,address_line2,town_or_city,county,postcode_full,la_label,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,refused,age1,sex1,ethnic_group,ethnic,national,ecstat1,relat2,age2,sex2,ecstat2,relat3,age3,sex3,ecstat3,relat4,age4,sex4,ecstat4,relat5,age5,sex5,ecstat5,relat6,age6,sex6,ecstat6,relat7,age7,sex7,ecstat7,relat8,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,homeless,ppcodenk,ppostcode_full,prevloc_label,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,referral,referral_value_check,incref,earnings,incfreq,hb,has_benefits,benefits,household_charge,nocharge,period,chcharge,wchchrg,carehome_charges_value_check,brent,scharge,pscharge,supcharg,tcharge,hbrentshortfall,tshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_admin_district,location_startdate
-,completed,choreographer@owtluk.com,false,2023-06-26T00:00:00+01:00,,2023-06-26T00:00:00+01:00,1,2023,DLUHC,DLUHC,7,0,2023-06-26T00:00:00+01:00,,,,HIJKLMN,ABCDEFG,0,,fake address,,London,,NW9 5LL,Barnet,2,6,2,2,7,1,1,3,2023-06-24T00:00:00+01:00,,1,2023-06-25T00:00:00+01:00,,3,1,4,,2,,1,0,35,F,0,2,13,0,P,32,M,6,,,,,,,,,,,,,,,,,,,,,,,,,1,4,1,2,1,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,2,7,4,,6,1,0,TN23 6LZ,Ashford,1,0,1,0,0,0,0,0,1,2,,0,68,1,6,1,1,,0,2,,,,200.0,50.0,40.0,35.0,325.0,1,12.0,,,,,,,,,,,,,,,,,,,,
+id,status,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,collection_start_year,owning_organisation_name,managing_organisation_name,lettype,renewal,startdate,irproduct,irproduct_other,lar,tenancycode,propcode,uprn_known,uprn,address_line1,address_line2,town_or_city,county,postcode_full,la_label,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,refused,age1,sex1,ethnic_group,ethnic,national,ecstat1,relat2,age2,sex2,ecstat2,relat3,age3,sex3,ecstat3,relat4,age4,sex4,ecstat4,relat5,age5,sex5,ecstat5,relat6,age6,sex6,ecstat6,relat7,age7,sex7,ecstat7,relat8,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,homeless,ppcodenk,ppostcode_full,prevloc_label,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,referral,referral_value_check,incref,earnings,incfreq,hb,has_benefits,benefits,household_charge,nocharge,period,chcharge,wchchrg,carehome_charges_value_check,brent,scharge,pscharge,supcharg,tcharge,scharge_value_check,pscharge_value_check,supcharg_value_check,hbrentshortfall,tshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_admin_district,location_startdate
+,completed,choreographer@owtluk.com,false,2023-06-26T00:00:00+01:00,,2023-06-26T00:00:00+01:00,1,2023,DLUHC,DLUHC,7,0,2023-06-26T00:00:00+01:00,,,,HIJKLMN,ABCDEFG,0,,fake address,,London,,NW9 5LL,Barnet,2,6,2,2,7,1,1,3,2023-06-24T00:00:00+01:00,,1,2023-06-25T00:00:00+01:00,,3,1,4,,2,,1,0,35,F,0,2,13,0,P,32,M,6,,,,,,,,,,,,,,,,,,,,,,,,,1,4,1,2,1,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,2,7,4,,6,1,0,TN23 6LZ,Ashford,1,0,1,0,0,0,0,0,1,2,,0,68,1,6,1,1,,0,2,,,,200.0,50.0,40.0,35.0,325.0,,,,1,12.0,,,,,,,,,,,,,,,,,,,,
diff --git a/spec/fixtures/files/lettings_log_csv_export_non_support_labels.csv b/spec/fixtures/files/lettings_log_csv_export_non_support_labels.csv
index fd70049dc..0b475c037 100644
--- a/spec/fixtures/files/lettings_log_csv_export_non_support_labels.csv
+++ b/spec/fixtures/files/lettings_log_csv_export_non_support_labels.csv
@@ -1,2 +1,2 @@
-id,status,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,collection_start_year,owning_organisation_name,managing_organisation_name,lettype,renewal,startdate,irproduct,irproduct_other,lar,tenancycode,propcode,uprn_known,uprn,address_line1,address_line2,town_or_city,county,postcode_full,la_label,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,refused,age1,sex1,ethnic_group,ethnic,national,ecstat1,relat2,age2,sex2,ecstat2,relat3,age3,sex3,ecstat3,relat4,age4,sex4,ecstat4,relat5,age5,sex5,ecstat5,relat6,age6,sex6,ecstat6,relat7,age7,sex7,ecstat7,relat8,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,homeless,ppcodenk,ppostcode_full,prevloc_label,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,referral,referral_value_check,incref,earnings,incfreq,hb,has_benefits,benefits,household_charge,nocharge,period,chcharge,wchchrg,carehome_charges_value_check,brent,scharge,pscharge,supcharg,tcharge,hbrentshortfall,tshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_admin_district,location_startdate
-,completed,choreographer@owtluk.com,false,2023-06-26T00:00:00+01:00,,2023-06-26T00:00:00+01:00,single log,2023,DLUHC,DLUHC,7,No,2023-06-26T00:00:00+01:00,,,,HIJKLMN,ABCDEFG,No,,fake address,,London,,NW9 5LL,Barnet,Affordable rent basis,Tenant abandoned property,2,2,House,Purpose built,Yes,3,2023-06-24T00:00:00+01:00,,Yes,2023-06-25T00:00:00+01:00,,Don’t know,Yes,Assured Shorthold Tenancy (AST) – Fixed term,,2,,1,0,35,Female,White,Irish,Tenant prefers not to say,Other,Partner,32,Male,Not seeking work,,,,,,,,,,,,,,,,,,,,,,,,,Yes – the person is a current or former regular,No – they left up to and including 5 years ago,Yes,No,Yes,Fully wheelchair accessible housing,1,0,0,0,0,0,No,Yes,0,0,1,0,0,0,0,0,0,0,Less than 1 year,1 year but under 2 years,Loss of tied accommodation,,Other supported housing,No,Yes,TN23 6LZ,Ashford,Yes,0,1,0,0,0,0,0,1,Tenant applied directly (no referral or nomination),,0,68,Weekly,Universal Credit housing element,1,All,,0,Every 2 weeks,,,,200.0,50.0,40.0,35.0,325.0,Yes,12.0,,,,,,,,,,,,,,,,,,,,
+id,status,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,collection_start_year,owning_organisation_name,managing_organisation_name,lettype,renewal,startdate,irproduct,irproduct_other,lar,tenancycode,propcode,uprn_known,uprn,address_line1,address_line2,town_or_city,county,postcode_full,la_label,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,refused,age1,sex1,ethnic_group,ethnic,national,ecstat1,relat2,age2,sex2,ecstat2,relat3,age3,sex3,ecstat3,relat4,age4,sex4,ecstat4,relat5,age5,sex5,ecstat5,relat6,age6,sex6,ecstat6,relat7,age7,sex7,ecstat7,relat8,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,homeless,ppcodenk,ppostcode_full,prevloc_label,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,referral,referral_value_check,incref,earnings,incfreq,hb,has_benefits,benefits,household_charge,nocharge,period,chcharge,wchchrg,carehome_charges_value_check,brent,scharge,pscharge,supcharg,tcharge,scharge_value_check,pscharge_value_check,supcharg_value_check,hbrentshortfall,tshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_admin_district,location_startdate
+,completed,choreographer@owtluk.com,false,2023-06-26T00:00:00+01:00,,2023-06-26T00:00:00+01:00,single log,2023,DLUHC,DLUHC,7,No,2023-06-26T00:00:00+01:00,,,,HIJKLMN,ABCDEFG,No,,fake address,,London,,NW9 5LL,Barnet,Affordable rent basis,Tenant abandoned property,2,2,House,Purpose built,Yes,3,2023-06-24T00:00:00+01:00,,Yes,2023-06-25T00:00:00+01:00,,Don’t know,Yes,Assured Shorthold Tenancy (AST) – Fixed term,,2,,1,0,35,Female,White,Irish,Tenant prefers not to say,Other,Partner,32,Male,Not seeking work,,,,,,,,,,,,,,,,,,,,,,,,,Yes – the person is a current or former regular,No – they left up to and including 5 years ago,Yes,No,Yes,Fully wheelchair accessible housing,1,0,0,0,0,0,No,Yes,0,0,1,0,0,0,0,0,0,0,Less than 1 year,1 year but under 2 years,Loss of tied accommodation,,Other supported housing,No,Yes,TN23 6LZ,Ashford,Yes,0,1,0,0,0,0,0,1,Tenant applied directly (no referral or nomination),,0,68,Weekly,Universal Credit housing element,1,All,,0,Every 2 weeks,,,,200.0,50.0,40.0,35.0,325.0,,,,Yes,12.0,,,,,,,,,,,,,,,,,,,,
diff --git a/spec/models/form/lettings/subsections/income_and_benefits_spec.rb b/spec/models/form/lettings/subsections/income_and_benefits_spec.rb
index e914ccc6a..c260ff56d 100644
--- a/spec/models/form/lettings/subsections/income_and_benefits_spec.rb
+++ b/spec/models/form/lettings/subsections/income_and_benefits_spec.rb
@@ -32,6 +32,9 @@ RSpec.describe Form::Lettings::Subsections::IncomeAndBenefits, type: :model do
         rent_monthly
         brent_min_rent_value_check
         brent_max_rent_value_check
+        scharge_value_check
+        pscharge_value_check
+        supcharg_value_check
         outstanding
         outstanding_amount
       ],
diff --git a/spec/models/validations/financial_validations_spec.rb b/spec/models/validations/financial_validations_spec.rb
index ab3924d90..3e645f164 100644
--- a/spec/models/validations/financial_validations_spec.rb
+++ b/spec/models/validations/financial_validations_spec.rb
@@ -263,39 +263,39 @@ RSpec.describe Validations::FinancialValidations do
 
         [{
           period: { label: "weekly", value: 1 },
-          charge: { field: "scharge", value: 156 },
+          charge: { field: "scharge", value: 801 },
         },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "scharge", value: 672 },
+           charge: { field: "scharge", value: 3471 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "scharge", value: 311 },
+           charge: { field: "scharge", value: 1601 },
          },
          {
            period: { label: "weekly", value: 1 },
-           charge: { field: "pscharge", value: 31 },
+           charge: { field: "pscharge", value: 701 },
          },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "pscharge", value: 150 },
+           charge: { field: "pscharge", value: 3200 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "pscharge", value: 61 },
+           charge: { field: "pscharge", value: 1401 },
          },
          {
            period: { label: "weekly", value: 1 },
-           charge: { field: "supcharg", value: 41 },
+           charge: { field: "supcharg", value: 801 },
          },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "supcharg", value: 200 },
+           charge: { field: "supcharg", value: 3471 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "supcharg", value: 81 },
+           charge: { field: "supcharg", value: 1601 },
          }].each do |test_case|
           it "does not allow charges outside the range when period is #{test_case[:period][:label]}" do
             record.period = test_case[:period][:value]
@@ -308,39 +308,39 @@ RSpec.describe Validations::FinancialValidations do
 
         [{
           period: { label: "weekly", value: 1 },
-          charge: { field: "scharge", value: 154 },
+          charge: { field: "scharge", value: 799 },
         },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "scharge", value: 670 },
+           charge: { field: "scharge", value: 3400 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "scharge", value: 309 },
+           charge: { field: "scharge", value: 1599 },
          },
          {
            period: { label: "weekly", value: 1 },
-           charge: { field: "pscharge", value: 30 },
+           charge: { field: "pscharge", value: 699 },
          },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "pscharge", value: 120 },
+           charge: { field: "pscharge", value: 2500 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "pscharge", value: 59 },
+           charge: { field: "pscharge", value: 1399 },
          },
          {
            period: { label: "weekly", value: 1 },
-           charge: { field: "supcharg", value: 39 },
+           charge: { field: "supcharg", value: 799 },
          },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "supcharg", value: 120 },
+           charge: { field: "supcharg", value: 3000 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "supcharg", value: 79 },
+           charge: { field: "supcharg", value: 1599 },
          }].each do |test_case|
           it "does allow charges inside the range when period is #{test_case[:period][:label]}" do
             record.period = test_case[:period][:value]
@@ -357,39 +357,39 @@ RSpec.describe Validations::FinancialValidations do
 
         [{
           period: { label: "weekly", value: 1 },
-          charge: { field: "scharge", value: 481 },
+          charge: { field: "scharge", value: 801 },
         },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "scharge", value: 2081 },
+           charge: { field: "scharge", value: 3471 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "scharge", value: 961 },
+           charge: { field: "scharge", value: 1601 },
          },
          {
            period: { label: "weekly", value: 1 },
-           charge: { field: "pscharge", value: 201 },
+           charge: { field: "pscharge", value: 701 },
          },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "pscharge", value: 1000 },
+           charge: { field: "pscharge", value: 3200 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "pscharge", value: 400.80 },
+           charge: { field: "pscharge", value: 1401 },
          },
          {
            period: { label: "weekly", value: 1 },
-           charge: { field: "supcharg", value: 466 },
+           charge: { field: "supcharg", value: 801 },
          },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "supcharg", value: 3100 },
+           charge: { field: "supcharg", value: 3471 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "supcharg", value: 990 },
+           charge: { field: "supcharg", value: 1601 },
          }].each do |test_case|
           it "does not allow charges outside the range when period is #{test_case[:period][:label]}" do
             record.period = test_case[:period][:value]
@@ -402,39 +402,39 @@ RSpec.describe Validations::FinancialValidations do
 
         [{
           period: { label: "weekly", value: 1 },
-          charge: { field: "scharge", value: 366 },
+          charge: { field: "scharge", value: 799 },
         },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "scharge", value: 1582 },
+           charge: { field: "scharge", value: 3400 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "scharge", value: 731 },
+           charge: { field: "scharge", value: 1599 },
          },
          {
            period: { label: "weekly", value: 1 },
-           charge: { field: "pscharge", value: 199.99 },
+           charge: { field: "pscharge", value: 699 },
          },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "pscharge", value: 800 },
+           charge: { field: "pscharge", value: 2500 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "pscharge", value: 400 },
+           charge: { field: "pscharge", value: 1399 },
          },
          {
            period: { label: "weekly", value: 1 },
-           charge: { field: "supcharg", value: 464 },
+           charge: { field: "supcharg", value: 799 },
          },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "supcharg", value: 2000 },
+           charge: { field: "supcharg", value: 3400 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "supcharg", value: 880 },
+           charge: { field: "supcharg", value: 1599 },
          }].each do |test_case|
           it "does allow charges inside the range when period is #{test_case[:period][:label]}" do
             record.period = test_case[:period][:value]
@@ -455,39 +455,39 @@ RSpec.describe Validations::FinancialValidations do
 
         [{
           period: { label: "weekly", value: 1 },
-          charge: { field: "scharge", value: 156 },
+          charge: { field: "scharge", value: 501 },
         },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "scharge", value: 672 },
+           charge: { field: "scharge", value: 2300 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "scharge", value: 311 },
+           charge: { field: "scharge", value: 1001 },
          },
          {
            period: { label: "weekly", value: 1 },
-           charge: { field: "pscharge", value: 36 },
+           charge: { field: "pscharge", value: 201 },
          },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "pscharge", value: 190 },
+           charge: { field: "pscharge", value: 1000 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "pscharge", value: 71 },
+           charge: { field: "pscharge", value: 401 },
          },
          {
            period: { label: "weekly", value: 1 },
-           charge: { field: "supcharg", value: 61 },
+           charge: { field: "supcharg", value: 201 },
          },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "supcharg", value: 300 },
+           charge: { field: "supcharg", value: 1000 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "supcharg", value: 122 },
+           charge: { field: "supcharg", value: 401 },
          }].each do |test_case|
           it "does not allow charges outside the range when period is #{test_case[:period][:label]}" do
             record.period = test_case[:period][:value]
@@ -500,39 +500,39 @@ RSpec.describe Validations::FinancialValidations do
 
         [{
           period: { label: "weekly", value: 1 },
-          charge: { field: "scharge", value: 44 },
+          charge: { field: "scharge", value: 499 },
         },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "scharge", value: 160 },
+           charge: { field: "scharge", value: 2000 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "scharge", value: 89 },
+           charge: { field: "scharge", value: 999 },
          },
          {
            period: { label: "weekly", value: 1 },
-           charge: { field: "pscharge", value: 34 },
+           charge: { field: "pscharge", value: 199 },
          },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "pscharge", value: 140 },
+           charge: { field: "pscharge", value: 800 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "pscharge", value: 69 },
+           charge: { field: "pscharge", value: 399 },
          },
          {
            period: { label: "weekly", value: 1 },
-           charge: { field: "supcharg", value: 59.99 },
+           charge: { field: "supcharg", value: 199.99 },
          },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "supcharg", value: 240 },
+           charge: { field: "supcharg", value: 800 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "supcharg", value: 119 },
+           charge: { field: "supcharg", value: 399 },
          }].each do |test_case|
           it "does allow charges inside the range when period is #{test_case[:period][:label]}" do
             record.period = test_case[:period][:value]
@@ -549,39 +549,39 @@ RSpec.describe Validations::FinancialValidations do
 
         [{
           period: { label: "weekly", value: 1 },
-          charge: { field: "scharge", value: 365.90 },
+          charge: { field: "scharge", value: 501 },
         },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "scharge", value: 2081 },
+           charge: { field: "scharge", value: 2300 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "scharge", value: 961 },
+           charge: { field: "scharge", value: 1001 },
          },
          {
            period: { label: "weekly", value: 1 },
-           charge: { field: "pscharge", value: 76 },
+           charge: { field: "pscharge", value: 201 },
          },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "pscharge", value: 400 },
+           charge: { field: "pscharge", value: 1000 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "pscharge", value: 151 },
+           charge: { field: "pscharge", value: 401 },
          },
          {
            period: { label: "weekly", value: 1 },
-           charge: { field: "supcharg", value: 121 },
+           charge: { field: "supcharg", value: 201 },
          },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "supcharg", value: 620 },
+           charge: { field: "supcharg", value: 1000 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "supcharg", value: 241 },
+           charge: { field: "supcharg", value: 401 },
          }].each do |test_case|
           it "does not allow charges outside the range when period is #{test_case[:period][:label]}" do
             record.period = test_case[:period][:value]
@@ -617,39 +617,39 @@ RSpec.describe Validations::FinancialValidations do
 
         [{
           period: { label: "weekly", value: 1 },
-          charge: { field: "scharge", value: 364.88 },
+          charge: { field: "scharge", value: 499 },
         },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "scharge", value: 1200 },
+           charge: { field: "scharge", value: 2000 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "scharge", value: 700.99 },
+           charge: { field: "scharge", value: 999 },
          },
          {
            period: { label: "weekly", value: 1 },
-           charge: { field: "pscharge", value: 74 },
+           charge: { field: "pscharge", value: 199 },
          },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "pscharge", value: 210 },
+           charge: { field: "pscharge", value: 800 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "pscharge", value: 149 },
+           charge: { field: "pscharge", value: 399 },
          },
          {
            period: { label: "weekly", value: 1 },
-           charge: { field: "supcharg", value: 119 },
+           charge: { field: "supcharg", value: 199.99 },
          },
          {
            period: { label: "monthly", value: 4 },
-           charge: { field: "supcharg", value: 480 },
+           charge: { field: "supcharg", value: 800 },
          },
          {
            period: { label: "every 2 weeks", value: 2 },
-           charge: { field: "supcharg", value: 239 },
+           charge: { field: "supcharg", value: 399 },
          }].each do |test_case|
           it "does allow charges inside the range when period is #{test_case[:period][:label]}" do
             record.period = test_case[:period][:value]
diff --git a/spec/models/validations/soft_validations_spec.rb b/spec/models/validations/soft_validations_spec.rb
index 0a267b9e9..8f00799ff 100644
--- a/spec/models/validations/soft_validations_spec.rb
+++ b/spec/models/validations/soft_validations_spec.rb
@@ -375,4 +375,646 @@ RSpec.describe Validations::SoftValidations do
       expect(record).to be_la_referral_for_general_needs
     end
   end
+
+  describe "scharge_over_soft_max?" do
+    context "and organisation is PRP" do
+      before do
+        record.owning_organisation.update(provider_type: "PRP")
+      end
+
+      it "returns false if scharge is not given" do
+        record.scharge = nil
+        record.needstype = 1
+        record.period = 1
+
+        expect(record).not_to be_scharge_over_soft_max
+      end
+
+      it "returns false if period is not given" do
+        record.scharge = 201
+        record.needstype = 1
+        record.period = nil
+
+        expect(record).not_to be_scharge_over_soft_max
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        scharge: 34,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         scharge: 100,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         scharge: 69,
+       }].each do |test_case|
+        it "returns false if scharge is under soft max for general needs #{test_case[:period][:label]}(35)" do
+          record.scharge = test_case[:scharge]
+          record.needstype = 1
+          record.period = test_case[:period][:value]
+
+          expect(record).not_to be_scharge_over_soft_max
+        end
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        scharge: 199,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         scharge: 400,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         scharge: 399,
+       }].each do |test_case|
+        it "returns false if scharge is under soft max for supported housing #{test_case[:period][:label]} (200)" do
+          record.scharge = test_case[:scharge]
+          record.needstype = 2
+          record.period = test_case[:period][:value]
+
+          expect(record).not_to be_scharge_over_soft_max
+        end
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        scharge: 36,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         scharge: 180,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         scharge: 71,
+       }].each do |test_case|
+        it "returns true if scharge is over soft max for general needs #{test_case[:period][:label]} (35)" do
+          record.scharge = test_case[:scharge]
+          record.needstype = 1
+          record.period = test_case[:period][:value]
+
+          expect(record).to be_scharge_over_soft_max
+        end
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        scharge: 201,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         scharge: 1000,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         scharge: 401,
+       }].each do |test_case|
+        it "returns true if scharge is over soft max for supported housing #{test_case[:period][:label]} (200)" do
+          record.scharge = test_case[:scharge]
+          record.needstype = 2
+          record.period = test_case[:period][:value]
+
+          expect(record).to be_scharge_over_soft_max
+        end
+      end
+    end
+
+    context "and organisation is LA" do
+      before do
+        record.owning_organisation.update(provider_type: "LA")
+      end
+
+      it "returns false if scharge is not given" do
+        record.scharge = nil
+        record.needstype = 1
+        record.period = 1
+
+        expect(record).not_to be_scharge_over_soft_max
+      end
+
+      it "returns false if period is not given" do
+        record.scharge = 201
+        record.needstype = 1
+        record.period = nil
+
+        expect(record).not_to be_scharge_over_soft_max
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        scharge: 24,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         scharge: 88,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         scharge: 49,
+       }].each do |test_case|
+        it "returns false if scharge is under soft max for general needs #{test_case[:period][:label]}(25)" do
+          record.scharge = test_case[:scharge]
+          record.needstype = 1
+          record.period = test_case[:period][:value]
+
+          expect(record).not_to be_scharge_over_soft_max
+        end
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        scharge: 99,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         scharge: 400,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         scharge: 199,
+       }].each do |test_case|
+        it "returns false if scharge is under soft max for supported housing #{test_case[:period][:label]} (100)" do
+          record.scharge = test_case[:scharge]
+          record.needstype = 2
+          record.period = test_case[:period][:value]
+
+          expect(record).not_to be_scharge_over_soft_max
+        end
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        scharge: 26,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         scharge: 120,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         scharge: 51,
+       }].each do |test_case|
+        it "returns true if scharge is over soft max for general needs #{test_case[:period][:label]} (25)" do
+          record.scharge = test_case[:scharge]
+          record.needstype = 1
+          record.period = test_case[:period][:value]
+
+          expect(record).to be_scharge_over_soft_max
+        end
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        scharge: 101,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         scharge: 450,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         scharge: 201,
+       }].each do |test_case|
+        it "returns true if scharge is over soft max for supported housing #{test_case[:period][:label]} (100)" do
+          record.scharge = test_case[:scharge]
+          record.needstype = 2
+          record.period = test_case[:period][:value]
+
+          expect(record).to be_scharge_over_soft_max
+        end
+      end
+    end
+  end
+
+  describe "pscharge_over_soft_max?" do
+    context "and organisation is PRP" do
+      before do
+        record.owning_organisation.update(provider_type: "PRP")
+      end
+
+      it "returns false if pscharge is not given" do
+        record.pscharge = nil
+        record.needstype = 1
+        record.period = 1
+
+        expect(record).not_to be_pscharge_over_soft_max
+      end
+
+      it "returns false if period is not given" do
+        record.pscharge = 201
+        record.needstype = 1
+        record.period = nil
+
+        expect(record).not_to be_pscharge_over_soft_max
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        pscharge: 34,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         pscharge: 100,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         pscharge: 69,
+       }].each do |test_case|
+        it "returns false if pscharge is under soft max for general needs #{test_case[:period][:label]}(35)" do
+          record.pscharge = test_case[:pscharge]
+          record.needstype = 1
+          record.period = test_case[:period][:value]
+
+          expect(record).not_to be_pscharge_over_soft_max
+        end
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        pscharge: 99,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         pscharge: 400,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         pscharge: 199,
+       }].each do |test_case|
+        it "returns false if pscharge is under soft max for supported housing #{test_case[:period][:label]} (100)" do
+          record.pscharge = test_case[:pscharge]
+          record.needstype = 2
+          record.period = test_case[:period][:value]
+
+          expect(record).not_to be_pscharge_over_soft_max
+        end
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        pscharge: 36,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         pscharge: 180,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         pscharge: 71,
+       }].each do |test_case|
+        it "returns true if pscharge is over soft max for general needs #{test_case[:period][:label]} (35)" do
+          record.pscharge = test_case[:pscharge]
+          record.needstype = 1
+          record.period = test_case[:period][:value]
+
+          expect(record).to be_pscharge_over_soft_max
+        end
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        pscharge: 101,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         pscharge: 450,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         pscharge: 201,
+       }].each do |test_case|
+        it "returns true if pscharge is over soft max for supported housing #{test_case[:period][:label]} (100)" do
+          record.pscharge = test_case[:pscharge]
+          record.needstype = 2
+          record.period = test_case[:period][:value]
+
+          expect(record).to be_pscharge_over_soft_max
+        end
+      end
+    end
+
+    context "and organisation is LA" do
+      before do
+        record.owning_organisation.update(provider_type: "LA")
+      end
+
+      it "returns false if pscharge is not given" do
+        record.pscharge = nil
+        record.needstype = 1
+        record.period = 1
+
+        expect(record).not_to be_pscharge_over_soft_max
+      end
+
+      it "returns false if period is not given" do
+        record.pscharge = 201
+        record.needstype = 1
+        record.period = nil
+
+        expect(record).not_to be_pscharge_over_soft_max
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        pscharge: 24,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         pscharge: 88,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         pscharge: 49,
+       }].each do |test_case|
+        it "returns false if pscharge is under soft max for general needs #{test_case[:period][:label]}(25)" do
+          record.pscharge = test_case[:pscharge]
+          record.needstype = 1
+          record.period = test_case[:period][:value]
+
+          expect(record).not_to be_pscharge_over_soft_max
+        end
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        pscharge: 74,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         pscharge: 250,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         pscharge: 149,
+       }].each do |test_case|
+        it "returns false if pscharge is under soft max for supported housing #{test_case[:period][:label]} (75)" do
+          record.pscharge = test_case[:pscharge]
+          record.needstype = 2
+          record.period = test_case[:period][:value]
+
+          expect(record).not_to be_pscharge_over_soft_max
+        end
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        pscharge: 26,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         pscharge: 120,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         pscharge: 51,
+       }].each do |test_case|
+        it "returns true if pscharge is over soft max for general needs #{test_case[:period][:label]} (25)" do
+          record.pscharge = test_case[:pscharge]
+          record.needstype = 1
+          record.period = test_case[:period][:value]
+
+          expect(record).to be_pscharge_over_soft_max
+        end
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        pscharge: 76,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         pscharge: 350,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         pscharge: 151,
+       }].each do |test_case|
+        it "returns true if pscharge is over soft max for supported housing #{test_case[:period][:label]} (75)" do
+          record.pscharge = test_case[:pscharge]
+          record.needstype = 2
+          record.period = test_case[:period][:value]
+
+          expect(record).to be_pscharge_over_soft_max
+        end
+      end
+    end
+  end
+
+  describe "supcharg_over_soft_max?" do
+    context "and organisation is PRP" do
+      before do
+        record.owning_organisation.update(provider_type: "PRP")
+      end
+
+      it "returns false if supcharg is not given" do
+        record.supcharg = nil
+        record.needstype = 1
+        record.period = 1
+
+        expect(record).not_to be_supcharg_over_soft_max
+      end
+
+      it "returns false if period is not given" do
+        record.supcharg = 201
+        record.needstype = 1
+        record.period = nil
+
+        expect(record).not_to be_supcharg_over_soft_max
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        supcharg: 34,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         supcharg: 100,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         supcharg: 69,
+       }].each do |test_case|
+        it "returns false if supcharg is under soft max for general needs #{test_case[:period][:label]}(35)" do
+          record.supcharg = test_case[:supcharg]
+          record.needstype = 1
+          record.period = test_case[:period][:value]
+
+          expect(record).not_to be_supcharg_over_soft_max
+        end
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        supcharg: 84,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         supcharg: 320,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         supcharg: 169,
+       }].each do |test_case|
+        it "returns false if supcharg is under soft max for supported housing #{test_case[:period][:label]} (85)" do
+          record.supcharg = test_case[:supcharg]
+          record.needstype = 2
+          record.period = test_case[:period][:value]
+
+          expect(record).not_to be_supcharg_over_soft_max
+        end
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        supcharg: 36,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         supcharg: 180,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         supcharg: 71,
+       }].each do |test_case|
+        it "returns true if supcharg is over soft max for general needs #{test_case[:period][:label]} (35)" do
+          record.supcharg = test_case[:supcharg]
+          record.needstype = 1
+          record.period = test_case[:period][:value]
+
+          expect(record).to be_supcharg_over_soft_max
+        end
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        supcharg: 86,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         supcharg: 400,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         supcharg: 171,
+       }].each do |test_case|
+        it "returns true if supcharg is over soft max for supported housing #{test_case[:period][:label]} (85)" do
+          record.supcharg = test_case[:supcharg]
+          record.needstype = 2
+          record.period = test_case[:period][:value]
+
+          expect(record).to be_supcharg_over_soft_max
+        end
+      end
+    end
+
+    context "and organisation is LA" do
+      before do
+        record.owning_organisation.update(provider_type: "LA")
+      end
+
+      it "returns false if supcharg is not given" do
+        record.supcharg = nil
+        record.needstype = 1
+        record.period = 1
+
+        expect(record).not_to be_supcharg_over_soft_max
+      end
+
+      it "returns false if period is not given" do
+        record.supcharg = 201
+        record.needstype = 1
+        record.period = nil
+
+        expect(record).not_to be_supcharg_over_soft_max
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        supcharg: 24,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         supcharg: 88,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         supcharg: 49,
+       }].each do |test_case|
+        it "returns false if supcharg is under soft max for general needs #{test_case[:period][:label]}(25)" do
+          record.supcharg = test_case[:supcharg]
+          record.needstype = 1
+          record.period = test_case[:period][:value]
+
+          expect(record).not_to be_supcharg_over_soft_max
+        end
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        supcharg: 74,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         supcharg: 250,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         supcharg: 149,
+       }].each do |test_case|
+        it "returns false if supcharg is under soft max for supported housing #{test_case[:period][:label]} (75)" do
+          record.supcharg = test_case[:supcharg]
+          record.needstype = 2
+          record.period = test_case[:period][:value]
+
+          expect(record).not_to be_supcharg_over_soft_max
+        end
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        supcharg: 26,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         supcharg: 120,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         supcharg: 51,
+       }].each do |test_case|
+        it "returns true if supcharg is over soft max for general needs #{test_case[:period][:label]} (25)" do
+          record.supcharg = test_case[:supcharg]
+          record.needstype = 1
+          record.period = test_case[:period][:value]
+
+          expect(record).to be_supcharg_over_soft_max
+        end
+      end
+
+      [{
+        period: { label: "weekly", value: 1 },
+        supcharg: 76,
+      },
+       {
+         period: { label: "monthly", value: 4 },
+         supcharg: 350,
+       },
+       {
+         period: { label: "every 2 weeks", value: 2 },
+         supcharg: 151,
+       }].each do |test_case|
+        it "returns true if supcharg is over soft max for supported housing #{test_case[:period][:label]} (75)" do
+          record.supcharg = test_case[:supcharg]
+          record.needstype = 2
+          record.period = test_case[:period][:value]
+
+          expect(record).to be_supcharg_over_soft_max
+        end
+      end
+    end
+  end
 end
diff --git a/spec/services/imports/lettings_logs_import_service_spec.rb b/spec/services/imports/lettings_logs_import_service_spec.rb
index a9b0b9260..1138fd203 100644
--- a/spec/services/imports/lettings_logs_import_service_spec.rb
+++ b/spec/services/imports/lettings_logs_import_service_spec.rb
@@ -891,7 +891,7 @@ RSpec.describe Imports::LettingsLogsImportService do
       context "and pscharge is out of range" do
         before do
           lettings_log_xml.at_xpath("//xmlns:Q17").content = "1"
-          lettings_log_xml.at_xpath("//xmlns:Q18aiii").content = "36"
+          lettings_log_xml.at_xpath("//xmlns:Q18aiii").content = "701"
         end
 
         it "intercepts the relevant validation error" do
@@ -914,7 +914,7 @@ RSpec.describe Imports::LettingsLogsImportService do
       context "and supcharg is out of range" do
         before do
           lettings_log_xml.at_xpath("//xmlns:Q17").content = "1"
-          lettings_log_xml.at_xpath("//xmlns:Q18aiv").content = "46"
+          lettings_log_xml.at_xpath("//xmlns:Q18aiv").content = "801"
         end
 
         it "intercepts the relevant validation error" do
@@ -937,7 +937,7 @@ RSpec.describe Imports::LettingsLogsImportService do
       context "and scharge is out of range" do
         before do
           lettings_log_xml.at_xpath("//xmlns:Q17").content = "1"
-          lettings_log_xml.at_xpath("//xmlns:Q18aii").content = "156"
+          lettings_log_xml.at_xpath("//xmlns:Q18aii").content = "801"
         end
 
         it "intercepts the relevant validation error" do
@@ -2068,5 +2068,40 @@ RSpec.describe Imports::LettingsLogsImportService do
         expect(lettings_log.status).to eq("completed")
       end
     end
+
+    context "and the scharge/pscharge/supcharg soft validations are triggered" do
+      let(:lettings_log_id) { "00d2343e-d5fa-4c89-8400-ec3854b0f2b4" }
+      let(:lettings_log_file) { open_file(fixture_directory, lettings_log_id) }
+      let(:lettings_log_xml) { Nokogiri::XML(lettings_log_file) }
+
+      around do |example|
+        Timecop.freeze(Time.zone.local(2023, 4, 1)) do
+          Singleton.__init__(FormHandler)
+          example.run
+        end
+        Timecop.return
+        Singleton.__init__(FormHandler)
+      end
+
+      before do
+        lettings_log_xml.at_xpath("//xmlns:AddressLine1").content = "address 1"
+        lettings_log_xml.at_xpath("//xmlns:TownCity").content = "towncity"
+        lettings_log_xml.at_xpath("//xmlns:DAY").content = "10"
+        lettings_log_xml.at_xpath("//xmlns:MONTH").content = "4"
+        lettings_log_xml.at_xpath("//xmlns:YEAR").content = "2023"
+        lettings_log_xml.at_xpath("//xmlns:P1Nat").content = "18"
+
+        lettings_log_xml.at_xpath("//xmlns:Q17").content = "1"
+        lettings_log_xml.at_xpath("//xmlns:Q18aii").content = "800"
+        lettings_log_xml.at_xpath("//xmlns:Q18aiii").content = "300"
+        lettings_log_xml.at_xpath("//xmlns:Q18aiv").content = "300"
+      end
+
+      it "completes the log" do
+        lettings_log_service.send(:create_log, lettings_log_xml)
+        lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
+        expect(lettings_log.status).to eq("completed")
+      end
+    end
   end
 end

From ea9c84e2a19d7713125ea1a0a1c3ad80f525fdcc Mon Sep 17 00:00:00 2001
From: kosiakkatrina <54268893+kosiakkatrina@users.noreply.github.com>
Date: Thu, 14 Sep 2023 12:51:43 +0100
Subject: [PATCH 33/33] Display old log id in the missing answers examples
 (#1914)

---
 app/services/imports/import_report_service.rb |  8 ++---
 ...lettings_logs_missing_answers_examples.csv | 32 +++++++++----------
 .../imports/import_report_service_spec.rb     | 10 +++---
 3 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/app/services/imports/import_report_service.rb b/app/services/imports/import_report_service.rb
index ffcd78fc5..4db8151cf 100644
--- a/app/services/imports/import_report_service.rb
+++ b/app/services/imports/import_report_service.rb
@@ -115,10 +115,10 @@ module Imports
 
         if unanswered_question_counts[unanswered_questions].present?
           unanswered_question_counts[unanswered_questions] += 1
-          missing_answers_example_sets[unanswered_questions] << { id: log.id, old_form_id: log.old_form_id, owning_organisation_id: log.owning_organisation_id } unless unanswered_question_counts[unanswered_questions] > 10
+          missing_answers_example_sets[unanswered_questions] << { id: log.id, old_form_id: log.old_form_id, old_id: log.old_id, owning_organisation_id: log.owning_organisation_id } unless unanswered_question_counts[unanswered_questions] > 10
         else
           unanswered_question_counts[unanswered_questions] = 1
-          missing_answers_example_sets[unanswered_questions] = [{ id: log.id, old_form_id: log.old_form_id, owning_organisation_id: log.owning_organisation_id }]
+          missing_answers_example_sets[unanswered_questions] = [{ id: log.id, old_form_id: log.old_form_id, old_id: log.old_id, owning_organisation_id: log.owning_organisation_id }]
         end
       end
 
@@ -138,12 +138,12 @@ module Imports
 
     def missing_answers_examples(missing_answers_example_sets)
       CSV.generate do |report|
-        headers = ["Missing answers", "Organisation ID", "Log ID", "Old Form ID"]
+        headers = ["Missing answers", "Organisation ID", "Log ID", "Old Form ID", "Old Log ID"]
         report << headers
 
         missing_answers_example_sets.each do |missing_answers, examples|
           examples.each do |example|
-            report << [missing_answers, example[:owning_organisation_id], example[:id], example[:old_form_id]]
+            report << [missing_answers, example[:owning_organisation_id], example[:id], example[:old_form_id], example[:old_id]]
           end
         end
       end
diff --git a/spec/fixtures/files/imported_lettings_logs_missing_answers_examples.csv b/spec/fixtures/files/imported_lettings_logs_missing_answers_examples.csv
index 0c35bae0b..a50ad4ea2 100644
--- a/spec/fixtures/files/imported_lettings_logs_missing_answers_examples.csv
+++ b/spec/fixtures/files/imported_lettings_logs_missing_answers_examples.csv
@@ -1,16 +1,16 @@
-Missing answers,Organisation ID,Log ID,Old Form ID
-age1_known,{org_id0},{id0},1000
-age1_known,{org_id1},{id1},1001
-age1_known,{org_id2},{id2},1002
-age1_known,{org_id3},{id3},1003
-age1_known,{org_id4},{id4},1004
-age1_known,{org_id5},{id5},1005
-age1_known,{org_id6},{id6},1006
-age1_known,{org_id7},{id7},1007
-age1_known,{org_id8},{id8},1008
-age1_known,{org_id9},{id9},1009
-beds,{org_id2_0},{id2_0},2000
-beds,{org_id2_1},{id2_1},2001
-beds,{org_id2_2},{id2_2},2002
-beds,{org_id2_3},{id2_3},2003
-"beds, age1_known",{org_id},{id},300
+Missing answers,Organisation ID,Log ID,Old Form ID,Old Log ID
+age1_known,{org_id0},{id0},1000,old_id_age1_known_0
+age1_known,{org_id1},{id1},1001,old_id_age1_known_1
+age1_known,{org_id2},{id2},1002,old_id_age1_known_2
+age1_known,{org_id3},{id3},1003,old_id_age1_known_3
+age1_known,{org_id4},{id4},1004,old_id_age1_known_4
+age1_known,{org_id5},{id5},1005,old_id_age1_known_5
+age1_known,{org_id6},{id6},1006,old_id_age1_known_6
+age1_known,{org_id7},{id7},1007,old_id_age1_known_7
+age1_known,{org_id8},{id8},1008,old_id_age1_known_8
+age1_known,{org_id9},{id9},1009,old_id_age1_known_9
+beds,{org_id2_0},{id2_0},2000,old_id_beds_0
+beds,{org_id2_1},{id2_1},2001,old_id_beds_1
+beds,{org_id2_2},{id2_2},2002,old_id_beds_2
+beds,{org_id2_3},{id2_3},2003,old_id_beds_3
+"beds, age1_known",{org_id},{id},300,beds_and_age
diff --git a/spec/services/imports/import_report_service_spec.rb b/spec/services/imports/import_report_service_spec.rb
index 938ab9f1b..1f47537a4 100644
--- a/spec/services/imports/import_report_service_spec.rb
+++ b/spec/services/imports/import_report_service_spec.rb
@@ -120,7 +120,7 @@ RSpec.describe Imports::ImportReportService do
           expected_answers_examples_content.sub!("{org_id2_#{i}}", log.owning_organisation_id.to_s)
           log.save!
         end
-        create(:lettings_log, :completed, age1_known: nil, beds: nil, old_form_id: "300", old_id: "123") do |log|
+        create(:lettings_log, :completed, age1_known: nil, beds: nil, old_form_id: "300", old_id: "beds_and_age") do |log|
           expected_answers_examples_content.sub!("{id}", log.id.to_s)
           expected_answers_examples_content.sub!("{org_id}", log.owning_organisation_id.to_s)
         end
@@ -132,7 +132,7 @@ RSpec.describe Imports::ImportReportService do
         expect(storage_service).to receive(:write_file).with("MissingAnswersReportLettingsLog_report_suffix.csv", "#{expected_content}")
         expect(storage_service).to receive(:write_file).with("MissingAnswersExamplesLettingsLog_report_suffix.csv", "#{expected_answers_examples_content}")
         expect(storage_service).to receive(:write_file).with("MissingAnswersReportSalesLog_report_suffix.csv", "\uFEFFMissing answers,Total number of affected logs\n")
-        expect(storage_service).to receive(:write_file).with("MissingAnswersExamplesSalesLog_report_suffix.csv", "\uFEFFMissing answers,Organisation ID,Log ID,Old Form ID\n")
+        expect(storage_service).to receive(:write_file).with("MissingAnswersExamplesSalesLog_report_suffix.csv", "\uFEFFMissing answers,Organisation ID,Log ID,Old Form ID,Old Log ID\n")
 
         report_service.generate_missing_answers_report("report_suffix")
       end
@@ -145,14 +145,14 @@ RSpec.describe Imports::ImportReportService do
 
       before do
         create_list(:sales_log, 11, :completed, age1_known: nil) do |log, i|
-          log.old_id = "age1_known_#{i}"
+          log.old_id = "old_id_age1_known_#{i}"
           log.old_form_id = "100#{i}"
           log.save!
           expected_answers_examples_content.sub!("{id#{i}}", log.id.to_s)
           expected_answers_examples_content.sub!("{org_id#{i}}", log.owning_organisation_id.to_s)
         end
         create_list(:sales_log, 4, :completed, beds: nil) do |log, i|
-          log.old_id = "beds_#{i}"
+          log.old_id = "old_id_beds_#{i}"
           log.old_form_id = "200#{i}"
           expected_answers_examples_content.sub!("{id2_#{i}}", log.id.to_s)
           expected_answers_examples_content.sub!("{org_id2_#{i}}", log.owning_organisation_id.to_s)
@@ -168,7 +168,7 @@ RSpec.describe Imports::ImportReportService do
 
       it "generates a csv with expected missing fields" do
         expect(storage_service).to receive(:write_file).with("MissingAnswersReportLettingsLog_report_suffix.csv", "\uFEFFMissing answers,Total number of affected logs\n")
-        expect(storage_service).to receive(:write_file).with("MissingAnswersExamplesLettingsLog_report_suffix.csv", "\uFEFFMissing answers,Organisation ID,Log ID,Old Form ID\n")
+        expect(storage_service).to receive(:write_file).with("MissingAnswersExamplesLettingsLog_report_suffix.csv", "\uFEFFMissing answers,Organisation ID,Log ID,Old Form ID,Old Log ID\n")
         expect(storage_service).to receive(:write_file).with("MissingAnswersReportSalesLog_report_suffix.csv", "#{expected_content}")
         expect(storage_service).to receive(:write_file).with("MissingAnswersExamplesSalesLog_report_suffix.csv", "#{expected_answers_examples_content}")