CLDC-2964 Save and export duplicate logs (#2141) (#2205)

* Set and clear duplicate log references
* Export duplicate_log_reference_id for lettings
* Export duplicate_log_reference_id for sales
* Export duplicate_log_reference_id in xml
* Update existing duplicate log references
* Add missing sales deduplication steps
* Set duplicate_set_id straight on logs
* Do not update updated_at to prevent export
* Increment duplicate_set_id
* Update duplicates when they change sets
* Remove duplicates when deleting logs

app/controllers/delete_logs_controller.rb

@@ -25,8 +25,14 @@ class DeleteLogsController < ApplicationController
 
   def discard_lettings_logs
     logs = LettingsLog.find(params.require(:ids))
+    remove_lettings_duplicate_set_ids(logs)
     discard logs
     if request.referer&.include?("delete-duplicates")
+      logs.each do |log|
+        log.update!(duplicate_set_id: nil)
+      end
+      LettingsLog.find(params["remaining_log_id"]).update!(duplicate_set_id: nil)
+
       redirect_to lettings_log_duplicate_logs_path(lettings_log_id: params["remaining_log_id"], original_log_id: params["original_log_id"], referrer: params[:referrer], organisation_id: params[:organisation_id]), notice: I18n.t("notification.duplicate_logs_deleted", count: logs.count, log_ids: duplicate_log_ids(logs))
     else
       redirect_to lettings_logs_path, notice: I18n.t("notification.logs_deleted", count: logs.count)
@@ -54,8 +60,14 @@ class DeleteLogsController < ApplicationController
 
   def discard_sales_logs
     logs = SalesLog.find(params.require(:ids))
+    remove_sales_duplicate_set_ids(logs)
     discard logs
     if request.referer&.include?("delete-duplicates")
+      logs.each do |log|
+        log.update!(duplicate_set_id: nil)
+      end
+      SalesLog.find(params["remaining_log_id"]).update!(duplicate_set_id: nil)
+
       redirect_to sales_log_duplicate_logs_path(sales_log_id: params["remaining_log_id"], original_log_id: params["original_log_id"], referrer: params[:referrer], organisation_id: params[:organisation_id]), notice: I18n.t("notification.duplicate_logs_deleted", count: logs.count, log_ids: duplicate_log_ids(logs))
     else
       redirect_to sales_logs_path, notice: I18n.t("notification.logs_deleted", count: logs.count)
@@ -203,4 +215,30 @@ private
   def duplicate_log_ids(logs)
     logs.map { |log| "Log #{log.id}" }.to_sentence(last_word_connector: " and ")
   end
+
+  def remove_lettings_duplicate_set_ids(logs)
+    duplicate_set_ids = []
+    logs.each do |log|
+      if log.duplicate_set_id.present?
+        duplicate_set_ids << log.duplicate_set_id
+        log.update!(duplicate_set_id: nil)
+      end
+    end
+    duplicate_set_ids.uniq.each do |duplicate_set_id|
+      LettingsLog.where(duplicate_set_id:).update!(duplicate_set_id: nil) if LettingsLog.where(duplicate_set_id:).count == 1
+    end
+  end
+
+  def remove_sales_duplicate_set_ids(logs)
+    duplicate_set_ids = []
+    logs.each do |log|
+      if log.duplicate_set_id.present?
+        duplicate_set_ids << log.duplicate_set_id
+        log.update!(duplicate_set_id: nil)
+      end
+    end
+    duplicate_set_ids.uniq.each do |duplicate_set_id|
+      SalesLog.where(duplicate_set_id:).update!(duplicate_set_id: nil) if SalesLog.where(duplicate_set_id:).count == 1
+    end
+  end
 end

app/controllers/form_controller.rb

@@ -171,12 +171,15 @@ private
         return correcting_duplicate_logs_redirect_path
       end
 
-      if @log.lettings? && current_user.lettings_logs.duplicate_logs(@log).count.positive?
+      dynamic_duplicates = @log.lettings? ? current_user.lettings_logs.duplicate_logs(@log) : current_user.sales_logs.duplicate_logs(@log)
-        return send("lettings_log_duplicate_logs_path", @log, original_log_id: @log.id)
+      if dynamic_duplicates.any?
+        saved_duplicates = @log.duplicates
+        if saved_duplicates.none? || duplicates_changed?(dynamic_duplicates, saved_duplicates)
+          duplicate_set_id = dynamic_duplicates.first.duplicate_set_id || new_duplicate_set_id(@log)
+          update_logs_with_duplicate_set_id(@log, dynamic_duplicates, duplicate_set_id)
+          saved_duplicates.first.update!(duplicate_set_id: nil) if saved_duplicates.count == 1
         end
-
+        return send("#{@log.class.name.underscore}_duplicate_logs_path", @log, original_log_id: @log.id)
-      if @log.sales? && current_user.sales_logs.duplicate_logs(@log).count.positive?
-        return send("sales_log_duplicate_logs_path", @log, original_log_id: @log.id)
       end
     end
 
@@ -250,11 +253,26 @@ private
     class_name = @log.class.name.underscore
 
     original_log = current_user.send(class_name.pluralize).find_by(id: from_referrer_query("original_log_id"))
+    dynamic_duplicates = current_user.send(class_name.pluralize).duplicate_logs(@log)
+
+    if dynamic_duplicates.any?
+      saved_duplicates = @log.duplicates
+      if duplicates_changed?(dynamic_duplicates, saved_duplicates)
+        duplicate_set_id = dynamic_duplicates.first.duplicate_set_id || new_duplicate_set_id(@log)
+        update_logs_with_duplicate_set_id(@log, dynamic_duplicates, duplicate_set_id)
+        saved_duplicates.first.update!(duplicate_set_id: nil) if saved_duplicates.count == 1
+      end
+    else
+      remove_fixed_duplicate_set_ids(@log)
+    end
 
-    if original_log.present? && current_user.send(class_name.pluralize).duplicate_logs(original_log).count.positive?
+    if original_log.present? && current_user.send(class_name.pluralize).duplicate_logs(original_log).any?
-      flash[:notice] = deduplication_success_banner unless current_user.send(class_name.pluralize).duplicate_logs(@log).count.positive?
+      if dynamic_duplicates.none?
+        flash[:notice] = deduplication_success_banner
+      end
       send("#{class_name}_duplicate_logs_path", original_log, original_log_id: original_log.id, referrer: params[:referrer], organisation_id: params[:organisation_id])
     else
+      remove_fixed_duplicate_set_ids(original_log)
       flash[:notice] = deduplication_success_banner
       send("#{class_name}_duplicate_logs_path", "#{class_name}_id".to_sym => from_referrer_query("first_remaining_duplicate_id"), original_log_id: from_referrer_query("original_log_id"), referrer: params[:referrer], organisation_id: params[:organisation_id])
     end
@@ -275,4 +293,32 @@ private
 
     I18n.t("notification.duplicate_logs.deduplication_success_banner", log_link: deduplicated_log_link, changed_question_label:).html_safe
   end
+
+  def remove_fixed_duplicate_set_ids(log)
+    duplicate_set_id = log.duplicate_set_id
+    return unless duplicate_set_id
+
+    log.update!(duplicate_set_id: nil)
+    LettingsLog.find_by(duplicate_set_id:)&.update!(duplicate_set_id: nil) if log.lettings? && LettingsLog.where(duplicate_set_id:).count == 1
+    SalesLog.find_by(duplicate_set_id:)&.update!(duplicate_set_id: nil) if log.sales? && SalesLog.where(duplicate_set_id:).count == 1
+  end
+
+  def new_duplicate_set_id(log)
+    if log.lettings?
+      LettingsLog.maximum(:duplicate_set_id).to_i + 1
+    else
+      SalesLog.maximum(:duplicate_set_id).to_i + 1
+    end
+  end
+
+  def duplicates_changed?(dynamic_duplicates, saved_duplicates)
+    dynamic_duplicates.present? && saved_duplicates.present? && dynamic_duplicates.order(:id).pluck(:id) != saved_duplicates.order(:id).pluck(:id)
+  end
+
+  def update_logs_with_duplicate_set_id(log, dynamic_duplicates, duplicate_set_id)
+    log.update!(duplicate_set_id:)
+    dynamic_duplicates.each do |duplicate|
+      duplicate.update!(duplicate_set_id: log.duplicate_set_id) if duplicate.duplicate_set_id != log.duplicate_set_id
+    end
+  end
 end

app/models/lettings_log.rb

@@ -652,6 +652,10 @@ class LettingsLog < Log
     renttype == 1 || renttype == 2
   end
 
+  def duplicates
+    LettingsLog.where.not(duplicate_set_id: nil).where(duplicate_set_id:).where.not(id:)
+  end
+
 private
 
   def reset_invalid_unresolved_log_fields!

app/models/sales_log.rb

@@ -469,4 +469,8 @@ class SalesLog < Log
      form.start_date.year < 2023 || uprn.blank? ? "postcode_full" : nil,
      form.start_date.year >= 2023 && uprn.present? ? "uprn" : nil].compact
   end
+
+  def duplicates
+    SalesLog.where.not(duplicate_set_id: nil).where(duplicate_set_id:).where.not(id:)
+  end
 end

app/services/csv/lettings_log_csv_service.rb

@@ -299,7 +299,7 @@ module Csv
           ATTRIBUTE_MAPPINGS.fetch(question.id, question.id)
         end
       end
-      non_question_fields = %w[id status created_by is_dpo created_at updated_by updated_at creation_method old_id old_form_id collection_start_year]
+      non_question_fields = %w[id status duplicate_set_id created_by is_dpo created_at updated_by updated_at creation_method old_id old_form_id collection_start_year]
       scheme_and_location_attributes = %w[scheme_code scheme_service_name scheme_sensitive SCHTYPE scheme_registered_under_care_act scheme_owning_organisation_name scheme_primary_client_group scheme_has_other_client_group scheme_secondary_client_group scheme_support_type scheme_intended_stay scheme_created_at location_code location_postcode location_name location_units location_type_of_unit location_mobility_type location_local_authority location_startdate]
       final_attributes = non_question_fields + attributes + scheme_and_location_attributes
       @user.support? ? final_attributes : final_attributes - SUPPORT_ONLY_ATTRIBUTES

app/services/csv/sales_log_csv_service.rb

@@ -144,7 +144,7 @@ module Csv
           question.id
         end
       end
-      non_question_fields = %w[id status created_at updated_at old_form_id collection_start_year creation_method is_dpo]
+      non_question_fields = %w[id status duplicate_set_id created_at updated_at old_form_id collection_start_year creation_method is_dpo]
       non_question_fields + attributes
     end
 

app/services/exports/lettings_log_export_constants.rb

@@ -136,7 +136,8 @@ module Exports::LettingsLogExportConstants
     "scheme_status",
     "location_status",
     "created_by",
-    "amended_by"
+    "amended_by",
+    "duplicate_set_id"
   ]
 
   (1..8).each do |index|

db/migrate/20240118183843_add_duplicate_set_id.rb

@@ -0,0 +1,6 @@
+class AddDuplicateSetId < ActiveRecord::Migration[7.0]
+  def change
+    add_column :lettings_logs, :duplicate_set_id, :integer
+    add_column :sales_logs, :duplicate_set_id, :integer
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2024_01_08_152935) do
+ActiveRecord::Schema[7.0].define(version: 2024_01_18_183843) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
@@ -302,6 +302,7 @@ ActiveRecord::Schema[7.0].define(version: 2024_01_08_152935) do
     t.integer "supcharg_value_check"
     t.integer "scharge_value_check"
     t.integer "pscharge_value_check"
+    t.integer "duplicate_set_id"
     t.index ["bulk_upload_id"], name: "index_lettings_logs_on_bulk_upload_id"
     t.index ["created_by_id"], name: "index_lettings_logs_on_created_by_id"
     t.index ["location_id"], name: "index_lettings_logs_on_location_id"
@@ -463,6 +464,8 @@ ActiveRecord::Schema[7.0].define(version: 2024_01_08_152935) do
     t.string "reader_type", null: false
     t.bigint "reader_id"
     t.datetime "timestamp", precision: nil, null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
     t.index ["readable_type", "readable_id"], name: "index_read_marks_on_readable_type_and_readable_id"
     t.index ["reader_id", "reader_type", "readable_type", "readable_id"], name: "read_marks_reader_readable_index", unique: true
     t.index ["reader_type", "reader_id"], name: "index_read_marks_on_reader_type_and_reader_id"
@@ -648,6 +651,7 @@ ActiveRecord::Schema[7.0].define(version: 2024_01_08_152935) do
     t.integer "old_form_id"
     t.datetime "values_updated_at"
     t.bigint "managing_organisation_id"
+    t.integer "duplicate_set_id"
     t.index ["bulk_upload_id"], name: "index_sales_logs_on_bulk_upload_id"
     t.index ["created_by_id"], name: "index_sales_logs_on_created_by_id"
     t.index ["managing_organisation_id"], name: "index_sales_logs_on_managing_organisation_id"

lib/tasks/set_duplicate_references.rake

@@ -0,0 +1,24 @@
+desc "Set duplicate references for sales and lettings logs"
+task set_duplicate_references: :environment do
+  SalesLog.filter_by_year(2023).duplicate_sets.each do |duplicate_set|
+    duplicate_set_id = SalesLog.maximum(:duplicate_set_id).to_i + 1
+    next if duplicate_set.any? { |_log_id| SalesLog.exists?(duplicate_set_id:) }
+
+    duplicate_set.each do |log_id|
+      log = SalesLog.find(log_id)
+      log.duplicate_set_id = duplicate_set_id
+      log.save!(touch: false, validate: false)
+    end
+  end
+
+  LettingsLog.filter_by_year(2023).duplicate_sets.each do |duplicate_set|
+    duplicate_set_id = LettingsLog.maximum(:duplicate_set_id).to_i + 1
+    next if duplicate_set.any? { |_log_id| LettingsLog.exists?(duplicate_set_id:) }
+
+    duplicate_set.each do |log_id|
+      log = LettingsLog.find(log_id)
+      log.duplicate_set_id = duplicate_set_id
+      log.save!(touch: false, validate: false)
+    end
+  end
+end

spec/features/form/form_navigation_spec.rb

@@ -178,16 +178,25 @@ RSpec.describe "Form Navigation" do
   end
 
   describe "fixing duplicate logs" do
+    let!(:lettings_log) { create(:lettings_log, :duplicate, created_by: user, duplicate_set_id: 1) }
+    let!(:second_log) { create(:lettings_log, :duplicate, created_by: user, duplicate_set_id: 1) }
+
     it "shows a correct cancel link" do
-      visit("lettings-logs/#{id}/tenant-code-test?first_remaining_duplicate_id=x&original_log_id=#{id}&referrer=duplicate_logs")
+      expect(lettings_log.duplicates.count).to eq(1)
+      visit("lettings-logs/#{id}/tenant-code-test?first_remaining_duplicate_id=#{second_log.id}&original_log_id=#{id}&referrer=duplicate_logs")
       click_link(text: "Cancel")
       expect(page).to have_current_path("/lettings-logs/#{id}/duplicate-logs?original_log_id=#{id}")
+      lettings_log.reload
+      expect(lettings_log.duplicates.count).to eq(1)
     end
 
     it "shows a correct Save Changes buttons" do
+      expect(lettings_log.duplicates.count).to eq(1)
       visit("lettings-logs/#{id}/tenant-code-test?first_remaining_duplicate_id=#{id}&original_log_id=#{id}&referrer=duplicate_logs")
       click_button(text: "Save changes")
       expect(page).to have_current_path("/lettings-logs/#{id}/duplicate-logs?original_log_id=#{id}&referrer=duplicate_logs")
+      lettings_log.reload
+      expect(lettings_log.duplicates.count).to eq(1)
     end
   end
 end

spec/features/lettings_log_spec.rb

@@ -459,6 +459,13 @@ RSpec.describe "Lettings Log Features" do
       end
 
       it "allows keeping the original log and deleting duplicates" do
+        lettings_log.reload
+        duplicate_log.reload
+        expect(lettings_log.duplicates.count).to eq(1)
+        expect(lettings_log.duplicate_set_id).not_to be_nil
+        expect(duplicate_log.duplicate_set_id).not_to be_nil
+        expect(lettings_log.duplicates).to include(duplicate_log)
+
         expect(page).to have_current_path("/lettings-logs/#{lettings_log.id}/duplicate-logs?original_log_id=#{lettings_log.id}")
         click_link("Keep this log and delete duplicates", href: "/lettings-logs/#{lettings_log.id}/delete-duplicates?original_log_id=#{lettings_log.id}")
         expect(page).to have_current_path("/lettings-logs/#{lettings_log.id}/delete-duplicates?original_log_id=#{lettings_log.id}")
@@ -471,6 +478,14 @@ RSpec.describe "Lettings Log Features" do
         expect(page).not_to have_content("These logs are duplicates")
         expect(page).not_to have_link("Keep this log and delete duplicates")
         expect(page).to have_link("Back to Log #{lettings_log.id}", href: "/lettings-logs/#{lettings_log.id}")
+
+        lettings_log.reload
+        duplicate_log.reload
+
+        expect(lettings_log.duplicates.count).to eq(0)
+        expect(duplicate_log.duplicates.count).to eq(0)
+        expect(lettings_log.duplicate_set_id).to be_nil
+        expect(duplicate_log.duplicate_set_id).to be_nil
       end
 
       it "allows changing answers on remaining original log" do
@@ -483,6 +498,13 @@ RSpec.describe "Lettings Log Features" do
       end
 
       it "allows keeping the duplicate log and deleting the original one" do
+        lettings_log.reload
+        duplicate_log.reload
+        expect(lettings_log.duplicates.count).to eq(1)
+        expect(lettings_log.duplicate_set_id).not_to be_nil
+        expect(duplicate_log.duplicate_set_id).not_to be_nil
+        expect(duplicate_log.duplicates).to include(lettings_log)
+
         expect(page).to have_current_path("/lettings-logs/#{lettings_log.id}/duplicate-logs?original_log_id=#{lettings_log.id}")
         click_link("Keep this log and delete duplicates", href: "/lettings-logs/#{duplicate_log.id}/delete-duplicates?original_log_id=#{lettings_log.id}")
         expect(page).to have_current_path("/lettings-logs/#{duplicate_log.id}/delete-duplicates?original_log_id=#{lettings_log.id}")
@@ -495,6 +517,14 @@ RSpec.describe "Lettings Log Features" do
         expect(page).not_to have_content("These logs are duplicates")
         expect(page).not_to have_link("Keep this log and delete duplicates")
         expect(page).to have_link("Back to lettings logs", href: "/lettings-logs")
+
+        lettings_log.reload
+        duplicate_log.reload
+
+        expect(lettings_log.duplicates.count).to eq(0)
+        expect(duplicate_log.duplicates.count).to eq(0)
+        expect(lettings_log.duplicate_set_id).to be_nil
+        expect(duplicate_log.duplicate_set_id).to be_nil
       end
 
       it "allows changing answers to remaining duplicate log" do
@@ -507,6 +537,13 @@ RSpec.describe "Lettings Log Features" do
       end
 
       it "allows deduplicating logs by changing the answers on the duplicate log" do
+        lettings_log.reload
+        duplicate_log.reload
+        expect(lettings_log.duplicates.count).to eq(1)
+        expect(lettings_log.duplicate_set_id).not_to be_nil
+        expect(duplicate_log.duplicate_set_id).not_to be_nil
+        expect(lettings_log.duplicates).to include(duplicate_log)
+
         expect(page).to have_current_path("/lettings-logs/#{lettings_log.id}/duplicate-logs?original_log_id=#{lettings_log.id}")
         click_link("Change", href: "/lettings-logs/#{duplicate_log.id}/tenant-code?first_remaining_duplicate_id=#{lettings_log.id}&original_log_id=#{lettings_log.id}&referrer=duplicate_logs")
         fill_in("lettings-log-tenancycode-field", with: "something else")
@@ -516,6 +553,14 @@ RSpec.describe "Lettings Log Features" do
         expect(page).to have_css(".govuk-notification-banner.govuk-notification-banner--success")
         expect(page).to have_content("Log #{duplicate_log.id} is no longer a duplicate and has been removed from the list")
         expect(page).to have_content("You changed the tenant code.")
+
+        lettings_log.reload
+        duplicate_log.reload
+
+        expect(lettings_log.duplicates.count).to eq(0)
+        expect(duplicate_log.duplicates.count).to eq(0)
+        expect(lettings_log.duplicate_set_id).to be_nil
+        expect(duplicate_log.duplicate_set_id).to be_nil
       end
 
       it "allows deduplicating logs by changing the answers on the original log" do
@@ -527,6 +572,8 @@ RSpec.describe "Lettings Log Features" do
         expect(page).to have_css(".govuk-notification-banner.govuk-notification-banner--success")
         expect(page).to have_content("Log #{lettings_log.id} is no longer a duplicate and has been removed from the list")
         expect(page).to have_content("You changed the tenant code.")
+        expect(duplicate_log.duplicates.count).to eq(0)
+        expect(duplicate_log.duplicate_set_id).to be_nil
       end
     end
   end

spec/features/sales_log_spec.rb

@@ -198,6 +198,14 @@ RSpec.describe "Sales Log Features" do
     end
 
     it "allows keeping the original log and deleting duplicates" do
+      sales_log.reload
+      duplicate_log.reload
+      expect(sales_log.duplicates.count).to eq(1)
+      expect(duplicate_log.duplicates.count).to eq(1)
+      expect(sales_log.duplicate_set_id).not_to be_nil
+      expect(duplicate_log.duplicate_set_id).not_to be_nil
+      expect(sales_log.duplicates).to include(duplicate_log)
+
       expect(page).to have_current_path("/sales-logs/#{sales_log.id}/duplicate-logs?original_log_id=#{sales_log.id}")
       click_link("Keep this log and delete duplicates", href: "/sales-logs/#{sales_log.id}/delete-duplicates?original_log_id=#{sales_log.id}")
       expect(page).to have_current_path("/sales-logs/#{sales_log.id}/delete-duplicates?original_log_id=#{sales_log.id}")
@@ -210,6 +218,14 @@ RSpec.describe "Sales Log Features" do
       expect(page).not_to have_content("These logs are duplicates")
       expect(page).not_to have_link("Keep this log and delete duplicates")
       expect(page).to have_link("Back to Log #{sales_log.id}", href: "/sales-logs/#{sales_log.id}")
+
+      sales_log.reload
+      duplicate_log.reload
+
+      expect(sales_log.duplicates.count).to eq(0)
+      expect(duplicate_log.duplicates.count).to eq(0)
+      expect(sales_log.duplicate_set_id).to be_nil
+      expect(duplicate_log.duplicate_set_id).to be_nil
     end
 
     it "allows changing answer on remaining original log" do
@@ -222,6 +238,14 @@ RSpec.describe "Sales Log Features" do
     end
 
     it "allows keeping the duplicate log and deleting the original one" do
+      sales_log.reload
+      duplicate_log.reload
+      expect(sales_log.duplicates.count).to eq(1)
+      expect(duplicate_log.duplicates.count).to eq(1)
+      expect(sales_log.duplicate_set_id).not_to be_nil
+      expect(duplicate_log.duplicate_set_id).not_to be_nil
+      expect(duplicate_log.duplicates).to include(sales_log)
+
       expect(page).to have_current_path("/sales-logs/#{sales_log.id}/duplicate-logs?original_log_id=#{sales_log.id}")
       click_link("Keep this log and delete duplicates", href: "/sales-logs/#{duplicate_log.id}/delete-duplicates?original_log_id=#{sales_log.id}")
       expect(page).to have_current_path("/sales-logs/#{duplicate_log.id}/delete-duplicates?original_log_id=#{sales_log.id}")
@@ -234,6 +258,14 @@ RSpec.describe "Sales Log Features" do
       expect(page).not_to have_content("These logs are duplicates")
       expect(page).not_to have_link("Keep this log and delete duplicates")
       expect(page).to have_link("Back to sales logs", href: "/sales-logs")
+
+      sales_log.reload
+      duplicate_log.reload
+
+      expect(sales_log.duplicates.count).to eq(0)
+      expect(duplicate_log.duplicates.count).to eq(0)
+      expect(sales_log.duplicate_set_id).to be_nil
+      expect(duplicate_log.duplicate_set_id).to be_nil
     end
 
     it "allows changing answers on remaining duplicate log" do
@@ -246,6 +278,14 @@ RSpec.describe "Sales Log Features" do
     end
 
     it "allows deduplicating logs by changing the answers on the duplicate log" do
+      sales_log.reload
+      duplicate_log.reload
+      expect(sales_log.duplicates.count).to eq(1)
+      expect(duplicate_log.duplicates.count).to eq(1)
+      expect(sales_log.duplicate_set_id).not_to be_nil
+      expect(duplicate_log.duplicate_set_id).not_to be_nil
+      expect(sales_log.duplicates).to include(duplicate_log)
+
       expect(page).to have_current_path("/sales-logs/#{sales_log.id}/duplicate-logs?original_log_id=#{sales_log.id}")
       click_link("Change", href: "/sales-logs/#{duplicate_log.id}/purchaser-code?first_remaining_duplicate_id=#{sales_log.id}&original_log_id=#{sales_log.id}&referrer=duplicate_logs")
       fill_in("sales-log-purchid-field", with: "something else")
@@ -255,6 +295,14 @@ RSpec.describe "Sales Log Features" do
       expect(page).to have_css(".govuk-notification-banner.govuk-notification-banner--success")
       expect(page).to have_content("Log #{duplicate_log.id} is no longer a duplicate and has been removed from the list")
       expect(page).to have_content("You changed the purchaser code.")
+
+      sales_log.reload
+      duplicate_log.reload
+
+      expect(sales_log.duplicates.count).to eq(0)
+      expect(duplicate_log.duplicates.count).to eq(0)
+      expect(sales_log.duplicate_set_id).to be_nil
+      expect(duplicate_log.duplicate_set_id).to be_nil
     end
 
     it "allows deduplicating logs by changing the answers on the original log" do
@@ -266,6 +314,11 @@ RSpec.describe "Sales Log Features" do
       expect(page).to have_css(".govuk-notification-banner.govuk-notification-banner--success")
       expect(page).to have_content("Log #{sales_log.id} is no longer a duplicate and has been removed from the list")
       expect(page).to have_content("You changed the purchaser code.")
+
+      expect(sales_log.duplicates.count).to eq(0)
+      expect(duplicate_log.duplicates.count).to eq(0)
+      expect(sales_log.duplicate_set_id).to be_nil
+      expect(duplicate_log.duplicate_set_id).to be_nil
     end
   end
 end

spec/fixtures/exports/general_needs_log.xml

@@ -144,6 +144,7 @@
     <county/>
     <discarded_at/>
     <creation_method>1</creation_method>
+    <duplicate_set_id/>
     <formid>{id}</formid>
     <owningorgid>{owning_org_id}</owningorgid>
     <owningorgname>DLUHC</owningorgname>

spec/fixtures/exports/general_needs_log_23_24.xml

@@ -145,6 +145,7 @@
     <county/>
     <discarded_at/>
     <creation_method>1</creation_method>
+    <duplicate_set_id/>
     <formid>{id}</formid>
     <owningorgid>{owning_org_id}</owningorgid>
     <owningorgname>DLUHC</owningorgname>

spec/fixtures/exports/supported_housing_logs.xml

@@ -143,6 +143,7 @@
     <county/>
     <discarded_at/>
     <creation_method>1</creation_method>
+    <duplicate_set_id/>
     <formid>{id}</formid>
     <owningorgid>{owning_org_id}</owningorgid>
     <owningorgname>DLUHC</owningorgname>

spec/fixtures/files/lettings_log_csv_export_codes.csv

@@ -1,2 +1,2 @@
-id,status,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,old_id,old_form_id,collection_start_year,owning_organisation_name,managing_organisation_name,needstype,lettype,renewal,startdate,renttype,rent_type_detail,irproduct,irproduct_other,lar,tenancycode,propcode,postcode_known,uprn_known,uprn,uprn_confirmed,address_line1,address_line2,town_or_city,county,postcode_full,is_la_inferred,la_label,la,first_time_property_let_as_social_housing,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,vacdays,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,hhmemb,pregnancy_value_check,refused,hhtype,totchild,totelder,totadult,age1,retirement_value_check,sex1,ethnic_group,ethnic,national,ecstat1,details_known_2,relat2,age2,sex2,ecstat2,details_known_3,relat3,age3,sex3,ecstat3,details_known_4,relat4,age4,sex4,ecstat4,details_known_5,relat5,age5,sex5,ecstat5,details_known_6,relat6,age6,sex6,ecstat6,details_known_7,relat7,age7,sex7,ecstat7,details_known_8,relat8,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,new_old,homeless,ppcodenk,ppostcode_full,previous_la_known,is_previous_la_inferred,prevloc_label,prevloc,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,letting_allocation_none,referral,referral_value_check,net_income_known,incref,earnings,incfreq,net_income_value_check,hb,has_benefits,benefits,household_charge,nocharge,period,is_carehome,chcharge,wchchrg,carehome_charges_value_check,brent,wrent,rent_value_check,scharge,wscharge,pscharge,wpschrge,supcharg,wsupchrg,tcharge,wtcharge,scharge_value_check,pscharge_value_check,supcharg_value_check,hbrentshortfall,tshortfall_known,tshortfall,wtshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_local_authority,location_startdate
+id,status,duplicate_set_id,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,old_id,old_form_id,collection_start_year,owning_organisation_name,managing_organisation_name,needstype,lettype,renewal,startdate,renttype,rent_type_detail,irproduct,irproduct_other,lar,tenancycode,propcode,postcode_known,uprn_known,uprn,uprn_confirmed,address_line1,address_line2,town_or_city,county,postcode_full,is_la_inferred,la_label,la,first_time_property_let_as_social_housing,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,vacdays,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,hhmemb,pregnancy_value_check,refused,hhtype,totchild,totelder,totadult,age1,retirement_value_check,sex1,ethnic_group,ethnic,national,ecstat1,details_known_2,relat2,age2,sex2,ecstat2,details_known_3,relat3,age3,sex3,ecstat3,details_known_4,relat4,age4,sex4,ecstat4,details_known_5,relat5,age5,sex5,ecstat5,details_known_6,relat6,age6,sex6,ecstat6,details_known_7,relat7,age7,sex7,ecstat7,details_known_8,relat8,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,new_old,homeless,ppcodenk,ppostcode_full,previous_la_known,is_previous_la_inferred,prevloc_label,prevloc,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,letting_allocation_none,referral,referral_value_check,net_income_known,incref,earnings,incfreq,net_income_value_check,hb,has_benefits,benefits,household_charge,nocharge,period,is_carehome,chcharge,wchchrg,carehome_charges_value_check,brent,wrent,rent_value_check,scharge,wscharge,pscharge,wpschrge,supcharg,wsupchrg,tcharge,wtcharge,scharge_value_check,pscharge_value_check,supcharg_value_check,hbrentshortfall,tshortfall_known,tshortfall,wtshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_local_authority,location_startdate
-,completed,s.port@jeemayle.com,false,2023-11-26T00:00:00+00:00,,2023-11-26T00:00:00+00:00,1,,,2023,DLUHC,DLUHC,1,7,0,2023-11-26,2,1,1,,2,HIJKLMN,ABCDEFG,1,0,,,fake address,,London,,NW9 5LL,false,Barnet,E09000003,0,2,6,2,2,7,1,1,3,2023-11-24,,,1,2023-11-25,,3,1,4,,2,,1,4,,1,4,0,0,2,35,,F,0,2,13,0,0,P,32,M,6,1,R,-9,R,10,0,R,-9,R,10,,,,,,,,,,,,,,,,,,,,,1,4,1,2,1,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,2,7,4,,6,2,1,0,TN23 6LZ,1,false,Ashford,E07000105,1,0,1,0,0,0,0,0,1,,2,,0,0,268,1,,6,1,1,,0,2,,,,,200.0,100.0,,50.0,25.0,40.0,20.0,35.0,17.5,325.0,162.5,,,,1,0,12.0,6.0,,,,,,,,,,,,,,,,,,,,
+,completed,,s.port@jeemayle.com,false,2023-11-26T00:00:00+00:00,,2023-11-26T00:00:00+00:00,1,,,2023,DLUHC,DLUHC,1,7,0,2023-11-26,2,1,1,,2,HIJKLMN,ABCDEFG,1,0,,,fake address,,London,,NW9 5LL,false,Barnet,E09000003,0,2,6,2,2,7,1,1,3,2023-11-24,,,1,2023-11-25,,3,1,4,,2,,1,4,,1,4,0,0,2,35,,F,0,2,13,0,0,P,32,M,6,1,R,-9,R,10,0,R,-9,R,10,,,,,,,,,,,,,,,,,,,,,1,4,1,2,1,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,2,7,4,,6,2,1,0,TN23 6LZ,1,false,Ashford,E07000105,1,0,1,0,0,0,0,0,1,,2,,0,0,268,1,,6,1,1,,0,2,,,,,200.0,100.0,,50.0,25.0,40.0,20.0,35.0,17.5,325.0,162.5,,,,1,0,12.0,6.0,,,,,,,,,,,,,,,,,,,,

spec/fixtures/files/lettings_log_csv_export_labels.csv

@@ -1,2 +1,2 @@
-id,status,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,old_id,old_form_id,collection_start_year,owning_organisation_name,managing_organisation_name,needstype,lettype,renewal,startdate,renttype,rent_type_detail,irproduct,irproduct_other,lar,tenancycode,propcode,postcode_known,uprn_known,uprn,uprn_confirmed,address_line1,address_line2,town_or_city,county,postcode_full,is_la_inferred,la_label,la,first_time_property_let_as_social_housing,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,vacdays,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,hhmemb,pregnancy_value_check,refused,hhtype,totchild,totelder,totadult,age1,retirement_value_check,sex1,ethnic_group,ethnic,national,ecstat1,details_known_2,relat2,age2,sex2,ecstat2,details_known_3,relat3,age3,sex3,ecstat3,details_known_4,relat4,age4,sex4,ecstat4,details_known_5,relat5,age5,sex5,ecstat5,details_known_6,relat6,age6,sex6,ecstat6,details_known_7,relat7,age7,sex7,ecstat7,details_known_8,relat8,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,new_old,homeless,ppcodenk,ppostcode_full,previous_la_known,is_previous_la_inferred,prevloc_label,prevloc,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,letting_allocation_none,referral,referral_value_check,net_income_known,incref,earnings,incfreq,net_income_value_check,hb,has_benefits,benefits,household_charge,nocharge,period,is_carehome,chcharge,wchchrg,carehome_charges_value_check,brent,wrent,rent_value_check,scharge,wscharge,pscharge,wpschrge,supcharg,wsupchrg,tcharge,wtcharge,scharge_value_check,pscharge_value_check,supcharg_value_check,hbrentshortfall,tshortfall_known,tshortfall,wtshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_local_authority,location_startdate
+id,status,duplicate_set_id,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,old_id,old_form_id,collection_start_year,owning_organisation_name,managing_organisation_name,needstype,lettype,renewal,startdate,renttype,rent_type_detail,irproduct,irproduct_other,lar,tenancycode,propcode,postcode_known,uprn_known,uprn,uprn_confirmed,address_line1,address_line2,town_or_city,county,postcode_full,is_la_inferred,la_label,la,first_time_property_let_as_social_housing,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,vacdays,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,hhmemb,pregnancy_value_check,refused,hhtype,totchild,totelder,totadult,age1,retirement_value_check,sex1,ethnic_group,ethnic,national,ecstat1,details_known_2,relat2,age2,sex2,ecstat2,details_known_3,relat3,age3,sex3,ecstat3,details_known_4,relat4,age4,sex4,ecstat4,details_known_5,relat5,age5,sex5,ecstat5,details_known_6,relat6,age6,sex6,ecstat6,details_known_7,relat7,age7,sex7,ecstat7,details_known_8,relat8,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,new_old,homeless,ppcodenk,ppostcode_full,previous_la_known,is_previous_la_inferred,prevloc_label,prevloc,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,letting_allocation_none,referral,referral_value_check,net_income_known,incref,earnings,incfreq,net_income_value_check,hb,has_benefits,benefits,household_charge,nocharge,period,is_carehome,chcharge,wchchrg,carehome_charges_value_check,brent,wrent,rent_value_check,scharge,wscharge,pscharge,wpschrge,supcharg,wsupchrg,tcharge,wtcharge,scharge_value_check,pscharge_value_check,supcharg_value_check,hbrentshortfall,tshortfall_known,tshortfall,wtshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_local_authority,location_startdate
-,completed,s.port@jeemayle.com,false,2023-11-26T00:00:00+00:00,,2023-11-26T00:00:00+00:00,single log,,,2023,DLUHC,DLUHC,General needs,Affordable rent general needs local authority,No,2023-11-26,2,Affordable Rent,Rent to Buy,,No,HIJKLMN,ABCDEFG,Yes,No,,,fake address,,London,,NW9 5LL,No,Barnet,E09000003,No,Affordable rent basis,Tenant abandoned property,No,2,House,Purpose built,Yes,3,2023-11-24,,,Yes,2023-11-25,,Don’t know,Yes,Assured Shorthold Tenancy (AST) – Fixed term,,2,,Yes,4,,Yes,4,0,0,2,35,,Female,White,Irish,Tenant prefers not to say,Other,Yes,Partner,32,Male,Not seeking work,No,Prefers not to say,Not known,Prefers not to say,Prefers not to say,Yes,Person prefers not to say,Not known,Person prefers not to say,Person prefers not to say,,,,,,,,,,,,,,,,,,,,,Yes – the person is a current or former regular,No – they left up to and including 5 years ago,Yes,No,Yes,Fully wheelchair accessible housing,Yes,No,No,No,No,No,No,Yes,No,No,Yes,No,No,No,No,No,No,No,Less than 1 year,1 year but under 2 years,Loss of tied accommodation,,Other supported housing,2,No,Yes,TN23 6LZ,Yes,No,Ashford,E07000105,Yes,,Yes,,,,No,No,Yes,,Tenant applied directly (no referral or nomination),,Yes,No,268,Weekly,,Universal Credit housing element,Yes,All,,No,Every 2 weeks,,,,,200.0,100.0,,50.0,25.0,40.0,20.0,35.0,17.5,325.0,162.5,,,,Yes,Yes,12.0,6.0,,,,,,,,,,,,,,,,,,,,
+,completed,,s.port@jeemayle.com,false,2023-11-26T00:00:00+00:00,,2023-11-26T00:00:00+00:00,single log,,,2023,DLUHC,DLUHC,General needs,Affordable rent general needs local authority,No,2023-11-26,2,Affordable Rent,Rent to Buy,,No,HIJKLMN,ABCDEFG,Yes,No,,,fake address,,London,,NW9 5LL,No,Barnet,E09000003,No,Affordable rent basis,Tenant abandoned property,No,2,House,Purpose built,Yes,3,2023-11-24,,,Yes,2023-11-25,,Don’t know,Yes,Assured Shorthold Tenancy (AST) – Fixed term,,2,,Yes,4,,Yes,4,0,0,2,35,,Female,White,Irish,Tenant prefers not to say,Other,Yes,Partner,32,Male,Not seeking work,No,Prefers not to say,Not known,Prefers not to say,Prefers not to say,Yes,Person prefers not to say,Not known,Person prefers not to say,Person prefers not to say,,,,,,,,,,,,,,,,,,,,,Yes – the person is a current or former regular,No – they left up to and including 5 years ago,Yes,No,Yes,Fully wheelchair accessible housing,Yes,No,No,No,No,No,No,Yes,No,No,Yes,No,No,No,No,No,No,No,Less than 1 year,1 year but under 2 years,Loss of tied accommodation,,Other supported housing,2,No,Yes,TN23 6LZ,Yes,No,Ashford,E07000105,Yes,,Yes,,,,No,No,Yes,,Tenant applied directly (no referral or nomination),,Yes,No,268,Weekly,,Universal Credit housing element,Yes,All,,No,Every 2 weeks,,,,,200.0,100.0,,50.0,25.0,40.0,20.0,35.0,17.5,325.0,162.5,,,,Yes,Yes,12.0,6.0,,,,,,,,,,,,,,,,,,,,

spec/fixtures/files/lettings_log_csv_export_non_support_codes.csv

@@ -1,2 +1,2 @@
-id,status,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,collection_start_year,owning_organisation_name,managing_organisation_name,needstype,lettype,renewal,startdate,renttype,rent_type_detail,irproduct,irproduct_other,lar,tenancycode,propcode,uprn_known,uprn,address_line1,address_line2,town_or_city,county,postcode_full,la_label,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,vacdays,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,hhmemb,refused,age1,sex1,ethnic_group,ethnic,national,ecstat1,relat2,age2,sex2,ecstat2,relat3,age3,sex3,ecstat3,relat4,age4,sex4,ecstat4,relat5,age5,sex5,ecstat5,relat6,age6,sex6,ecstat6,relat7,age7,sex7,ecstat7,relat8,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,homeless,ppcodenk,ppostcode_full,prevloc_label,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,letting_allocation_none,referral,referral_value_check,incref,earnings,incfreq,hb,has_benefits,benefits,household_charge,nocharge,period,is_carehome,chcharge,wchchrg,carehome_charges_value_check,brent,scharge,pscharge,supcharg,tcharge,scharge_value_check,pscharge_value_check,supcharg_value_check,hbrentshortfall,tshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_local_authority,location_startdate
+id,status,duplicate_set_id,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,collection_start_year,owning_organisation_name,managing_organisation_name,needstype,lettype,renewal,startdate,renttype,rent_type_detail,irproduct,irproduct_other,lar,tenancycode,propcode,uprn_known,uprn,address_line1,address_line2,town_or_city,county,postcode_full,la_label,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,vacdays,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,hhmemb,refused,age1,sex1,ethnic_group,ethnic,national,ecstat1,relat2,age2,sex2,ecstat2,relat3,age3,sex3,ecstat3,relat4,age4,sex4,ecstat4,relat5,age5,sex5,ecstat5,relat6,age6,sex6,ecstat6,relat7,age7,sex7,ecstat7,relat8,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,homeless,ppcodenk,ppostcode_full,prevloc_label,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,letting_allocation_none,referral,referral_value_check,incref,earnings,incfreq,hb,has_benefits,benefits,household_charge,nocharge,period,is_carehome,chcharge,wchchrg,carehome_charges_value_check,brent,scharge,pscharge,supcharg,tcharge,scharge_value_check,pscharge_value_check,supcharg_value_check,hbrentshortfall,tshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_local_authority,location_startdate
-,completed,choreographer@owtluk.com,false,2023-11-26T00:00:00+00:00,,2023-11-26T00:00:00+00:00,1,2023,DLUHC,DLUHC,1,7,0,2023-11-26,2,1,1,,2,HIJKLMN,ABCDEFG,0,,fake address,,London,,NW9 5LL,Barnet,2,6,2,2,7,1,1,3,2023-11-24,1,,1,2023-11-25,,3,1,4,,2,,1,4,1,35,F,0,2,13,0,P,32,M,6,R,-9,R,10,R,-9,R,10,,,,,,,,,,,,,,,,,1,4,1,2,1,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,2,7,4,,6,1,0,TN23 6LZ,Ashford,1,0,1,0,0,0,0,0,1,,2,,0,268,1,6,1,1,,0,2,,,,,200.0,50.0,40.0,35.0,325.0,,,,1,12.0,,,,,,,,,,,,,,,,,,,,
+,completed,,choreographer@owtluk.com,false,2023-11-26T00:00:00+00:00,,2023-11-26T00:00:00+00:00,1,2023,DLUHC,DLUHC,1,7,0,2023-11-26,2,1,1,,2,HIJKLMN,ABCDEFG,0,,fake address,,London,,NW9 5LL,Barnet,2,6,2,2,7,1,1,3,2023-11-24,1,,1,2023-11-25,,3,1,4,,2,,1,4,1,35,F,0,2,13,0,P,32,M,6,R,-9,R,10,R,-9,R,10,,,,,,,,,,,,,,,,,1,4,1,2,1,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,2,7,4,,6,1,0,TN23 6LZ,Ashford,1,0,1,0,0,0,0,0,1,,2,,0,268,1,6,1,1,,0,2,,,,,200.0,50.0,40.0,35.0,325.0,,,,1,12.0,,,,,,,,,,,,,,,,,,,,

spec/fixtures/files/lettings_log_csv_export_non_support_labels.csv

@@ -1,2 +1,2 @@
-id,status,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,collection_start_year,owning_organisation_name,managing_organisation_name,needstype,lettype,renewal,startdate,renttype,rent_type_detail,irproduct,irproduct_other,lar,tenancycode,propcode,uprn_known,uprn,address_line1,address_line2,town_or_city,county,postcode_full,la_label,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,vacdays,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,hhmemb,refused,age1,sex1,ethnic_group,ethnic,national,ecstat1,relat2,age2,sex2,ecstat2,relat3,age3,sex3,ecstat3,relat4,age4,sex4,ecstat4,relat5,age5,sex5,ecstat5,relat6,age6,sex6,ecstat6,relat7,age7,sex7,ecstat7,relat8,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,homeless,ppcodenk,ppostcode_full,prevloc_label,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,letting_allocation_none,referral,referral_value_check,incref,earnings,incfreq,hb,has_benefits,benefits,household_charge,nocharge,period,is_carehome,chcharge,wchchrg,carehome_charges_value_check,brent,scharge,pscharge,supcharg,tcharge,scharge_value_check,pscharge_value_check,supcharg_value_check,hbrentshortfall,tshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_local_authority,location_startdate
+id,status,duplicate_set_id,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,collection_start_year,owning_organisation_name,managing_organisation_name,needstype,lettype,renewal,startdate,renttype,rent_type_detail,irproduct,irproduct_other,lar,tenancycode,propcode,uprn_known,uprn,address_line1,address_line2,town_or_city,county,postcode_full,la_label,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,vacdays,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,hhmemb,refused,age1,sex1,ethnic_group,ethnic,national,ecstat1,relat2,age2,sex2,ecstat2,relat3,age3,sex3,ecstat3,relat4,age4,sex4,ecstat4,relat5,age5,sex5,ecstat5,relat6,age6,sex6,ecstat6,relat7,age7,sex7,ecstat7,relat8,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,homeless,ppcodenk,ppostcode_full,prevloc_label,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,letting_allocation_none,referral,referral_value_check,incref,earnings,incfreq,hb,has_benefits,benefits,household_charge,nocharge,period,is_carehome,chcharge,wchchrg,carehome_charges_value_check,brent,scharge,pscharge,supcharg,tcharge,scharge_value_check,pscharge_value_check,supcharg_value_check,hbrentshortfall,tshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_local_authority,location_startdate
-,completed,choreographer@owtluk.com,false,2023-11-26T00:00:00+00:00,,2023-11-26T00:00:00+00:00,single log,2023,DLUHC,DLUHC,General needs,Affordable rent general needs local authority,No,2023-11-26,2,Affordable Rent,Rent to Buy,,No,HIJKLMN,ABCDEFG,No,,fake address,,London,,NW9 5LL,Barnet,Affordable rent basis,Tenant abandoned property,No,2,House,Purpose built,Yes,3,2023-11-24,1,,Yes,2023-11-25,,Don’t know,Yes,Assured Shorthold Tenancy (AST) – Fixed term,,2,,Yes,4,Yes,35,Female,White,Irish,Tenant prefers not to say,Other,Partner,32,Male,Not seeking work,Prefers not to say,Not known,Prefers not to say,Prefers not to say,Person prefers not to say,Not known,Person prefers not to say,Person prefers not to say,,,,,,,,,,,,,,,,,Yes – the person is a current or former regular,No – they left up to and including 5 years ago,Yes,No,Yes,Fully wheelchair accessible housing,Yes,No,No,No,No,No,No,Yes,No,No,Yes,No,No,No,No,No,No,No,Less than 1 year,1 year but under 2 years,Loss of tied accommodation,,Other supported housing,No,Yes,TN23 6LZ,Ashford,Yes,,Yes,,,,No,No,Yes,,Tenant applied directly (no referral or nomination),,No,268,Weekly,Universal Credit housing element,Yes,All,,No,Every 2 weeks,,,,,200.0,50.0,40.0,35.0,325.0,,,,Yes,12.0,,,,,,,,,,,,,,,,,,,,
+,completed,,choreographer@owtluk.com,false,2023-11-26T00:00:00+00:00,,2023-11-26T00:00:00+00:00,single log,2023,DLUHC,DLUHC,General needs,Affordable rent general needs local authority,No,2023-11-26,2,Affordable Rent,Rent to Buy,,No,HIJKLMN,ABCDEFG,No,,fake address,,London,,NW9 5LL,Barnet,Affordable rent basis,Tenant abandoned property,No,2,House,Purpose built,Yes,3,2023-11-24,1,,Yes,2023-11-25,,Don’t know,Yes,Assured Shorthold Tenancy (AST) – Fixed term,,2,,Yes,4,Yes,35,Female,White,Irish,Tenant prefers not to say,Other,Partner,32,Male,Not seeking work,Prefers not to say,Not known,Prefers not to say,Prefers not to say,Person prefers not to say,Not known,Person prefers not to say,Person prefers not to say,,,,,,,,,,,,,,,,,Yes – the person is a current or former regular,No – they left up to and including 5 years ago,Yes,No,Yes,Fully wheelchair accessible housing,Yes,No,No,No,No,No,No,Yes,No,No,Yes,No,No,No,No,No,No,No,Less than 1 year,1 year but under 2 years,Loss of tied accommodation,,Other supported housing,No,Yes,TN23 6LZ,Ashford,Yes,,Yes,,,,No,No,Yes,,Tenant applied directly (no referral or nomination),,No,268,Weekly,Universal Credit housing element,Yes,All,,No,Every 2 weeks,,,,,200.0,50.0,40.0,35.0,325.0,,,,Yes,12.0,,,,,,,,,,,,,,,,,,,,

spec/fixtures/files/sales_logs_csv_export_codes.csv

@@ -1,2 +1,2 @@
-id,status,created_at,updated_at,old_form_id,collection_start_year,creation_method,is_dpo,owning_organisation_name,managing_organisation_name,created_by,day,month,year,purchid,ownershipsch,type,othtype,companybuy,buylivein,jointpur,jointmore,uprn,uprn_confirmed,address_line1,address_line2,town_or_city,county,pcode1,pcode2,la_known,la,la_label,beds,proptype,builtype,pcodenk,wchair,noint,privacynotice,age1,sex1,ethnic_group,ethnic,national,ecstat1,buy1livein,relat2,age2,sex2,ethnic_group2,ethnicbuy2,nationalbuy2,ecstat2,buy2livein,hholdcount,relat3,age3,sex3,ecstat3,relat4,age4,sex4,ecstat4,relat5,age5,sex5,ecstat5,relat6,age6,sex6,ecstat6,prevten,ppcodenk,ppostc1,ppostc2,previous_la_known,prevloc,prevloc_label,pregyrha,pregother,pregla,pregghb,pregblank,buy2living,prevtenbuy2,hhregres,hhregresstill,armedforcesspouse,disabled,wheel,income1nk,income1,inc1mort,income2nk,income2,inc2mort,hb,savingsnk,savings,prevown,prevshared,proplen,staircase,stairbought,stairowned,staircasesale,resale,exday,exmonth,exyear,hoday,homonth,hoyear,lanomagr,soctenant,frombeds,fromprop,socprevten,value,equity,mortgageused,mortgage,mortgagelender,mortgagelenderother,mortlen,extrabor,deposit,cashdis,mrent,has_mscharge,mscharge,discount,grant
+id,status,duplicate_set_id,created_at,updated_at,old_form_id,collection_start_year,creation_method,is_dpo,owning_organisation_name,managing_organisation_name,created_by,day,month,year,purchid,ownershipsch,type,othtype,companybuy,buylivein,jointpur,jointmore,uprn,uprn_confirmed,address_line1,address_line2,town_or_city,county,pcode1,pcode2,la_known,la,la_label,beds,proptype,builtype,pcodenk,wchair,noint,privacynotice,age1,sex1,ethnic_group,ethnic,national,ecstat1,buy1livein,relat2,age2,sex2,ethnic_group2,ethnicbuy2,nationalbuy2,ecstat2,buy2livein,hholdcount,relat3,age3,sex3,ecstat3,relat4,age4,sex4,ecstat4,relat5,age5,sex5,ecstat5,relat6,age6,sex6,ecstat6,prevten,ppcodenk,ppostc1,ppostc2,previous_la_known,prevloc,prevloc_label,pregyrha,pregother,pregla,pregghb,pregblank,buy2living,prevtenbuy2,hhregres,hhregresstill,armedforcesspouse,disabled,wheel,income1nk,income1,inc1mort,income2nk,income2,inc2mort,hb,savingsnk,savings,prevown,prevshared,proplen,staircase,stairbought,stairowned,staircasesale,resale,exday,exmonth,exyear,hoday,homonth,hoyear,lanomagr,soctenant,frombeds,fromprop,socprevten,value,equity,mortgageused,mortgage,mortgagelender,mortgagelenderother,mortlen,extrabor,deposit,cashdis,mrent,has_mscharge,mscharge,discount,grant
-,completed,2023-12-08T00:00:00+00:00,2023-12-08T00:00:00+00:00,,2023,1,false,DLUHC,DLUHC,billyboy@eyeklaud.com,8,12,2023,,2,8,,,,1,1,,,Address line 1,,Town or city,,SW1A,1AA,1,E09000003,Barnet,2,1,1,0,1,2,1,30,X,17,17,18,1,1,P,35,X,17,,13,1,1,3,C,14,X,9,X,-9,X,3,R,-9,R,10,,,,,1,1,,,0,,,1,1,1,1,,3,,1,4,5,1,1,0,10000,1,0,10000,1,4,1,,1,2,10,,,,,,,,,,,,,,,,,110000.0,,1,20000.0,5,,10,1,80000.0,,,1,100.0,,10000.0
+,completed,,2023-12-08T00:00:00+00:00,2023-12-08T00:00:00+00:00,,2023,1,false,DLUHC,DLUHC,billyboy@eyeklaud.com,8,12,2023,,2,8,,,,1,1,,,Address line 1,,Town or city,,SW1A,1AA,1,E09000003,Barnet,2,1,1,0,1,2,1,30,X,17,17,18,1,1,P,35,X,17,,13,1,1,3,C,14,X,9,X,-9,X,3,R,-9,R,10,,,,,1,1,,,0,,,1,1,1,1,,3,,1,4,5,1,1,0,10000,1,0,10000,1,4,1,,1,2,10,,,,,,,,,,,,,,,,,110000.0,,1,20000.0,5,,10,1,80000.0,,,1,100.0,,10000.0

spec/fixtures/files/sales_logs_csv_export_labels.csv

@@ -1,2 +1,2 @@
-id,status,created_at,updated_at,old_form_id,collection_start_year,creation_method,is_dpo,owning_organisation_name,managing_organisation_name,created_by,day,month,year,purchid,ownershipsch,type,othtype,companybuy,buylivein,jointpur,jointmore,uprn,uprn_confirmed,address_line1,address_line2,town_or_city,county,pcode1,pcode2,la_known,la,la_label,beds,proptype,builtype,pcodenk,wchair,noint,privacynotice,age1,sex1,ethnic_group,ethnic,national,ecstat1,buy1livein,relat2,age2,sex2,ethnic_group2,ethnicbuy2,nationalbuy2,ecstat2,buy2livein,hholdcount,relat3,age3,sex3,ecstat3,relat4,age4,sex4,ecstat4,relat5,age5,sex5,ecstat5,relat6,age6,sex6,ecstat6,prevten,ppcodenk,ppostc1,ppostc2,previous_la_known,prevloc,prevloc_label,pregyrha,pregother,pregla,pregghb,pregblank,buy2living,prevtenbuy2,hhregres,hhregresstill,armedforcesspouse,disabled,wheel,income1nk,income1,inc1mort,income2nk,income2,inc2mort,hb,savingsnk,savings,prevown,prevshared,proplen,staircase,stairbought,stairowned,staircasesale,resale,exday,exmonth,exyear,hoday,homonth,hoyear,lanomagr,soctenant,frombeds,fromprop,socprevten,value,equity,mortgageused,mortgage,mortgagelender,mortgagelenderother,mortlen,extrabor,deposit,cashdis,mrent,has_mscharge,mscharge,discount,grant
+id,status,duplicate_set_id,created_at,updated_at,old_form_id,collection_start_year,creation_method,is_dpo,owning_organisation_name,managing_organisation_name,created_by,day,month,year,purchid,ownershipsch,type,othtype,companybuy,buylivein,jointpur,jointmore,uprn,uprn_confirmed,address_line1,address_line2,town_or_city,county,pcode1,pcode2,la_known,la,la_label,beds,proptype,builtype,pcodenk,wchair,noint,privacynotice,age1,sex1,ethnic_group,ethnic,national,ecstat1,buy1livein,relat2,age2,sex2,ethnic_group2,ethnicbuy2,nationalbuy2,ecstat2,buy2livein,hholdcount,relat3,age3,sex3,ecstat3,relat4,age4,sex4,ecstat4,relat5,age5,sex5,ecstat5,relat6,age6,sex6,ecstat6,prevten,ppcodenk,ppostc1,ppostc2,previous_la_known,prevloc,prevloc_label,pregyrha,pregother,pregla,pregghb,pregblank,buy2living,prevtenbuy2,hhregres,hhregresstill,armedforcesspouse,disabled,wheel,income1nk,income1,inc1mort,income2nk,income2,inc2mort,hb,savingsnk,savings,prevown,prevshared,proplen,staircase,stairbought,stairowned,staircasesale,resale,exday,exmonth,exyear,hoday,homonth,hoyear,lanomagr,soctenant,frombeds,fromprop,socprevten,value,equity,mortgageused,mortgage,mortgagelender,mortgagelenderother,mortlen,extrabor,deposit,cashdis,mrent,has_mscharge,mscharge,discount,grant
-,completed,2023-12-08T00:00:00+00:00,2023-12-08T00:00:00+00:00,,2023,single log,false,DLUHC,DLUHC,billyboy@eyeklaud.com,8,12,2023,,Yes - a discounted ownership scheme,Right to Acquire (RTA),,,,Yes,Yes,,,Address line 1,,Town or city,,SW1A,1AA,1,E09000003,Barnet,2,Flat or maisonette,Purpose built,0,Yes,Yes,1,30,Non-binary,Buyer 1 prefers not to say,17,United Kingdom,Full-time - 30 hours or more,Yes,Partner,35,Non-binary,Buyer 2 prefers not to say,,Buyer prefers not to say,Full-time - 30 hours or more,Yes,3,Child,14,Non-binary,Child under 16,Other,Not known,Non-binary,"In government training into work, such as New Deal",Prefers not to say,Not known,Prefers not to say,Prefers not to say,,,,,Local authority tenant,No,,,No,,,1,1,1,1,,Don't know,,Yes,Yes,No,Yes,Yes,Yes,10000,Yes,Yes,10000,Yes,"Don’t know ",No,,Yes,No,10,,,,,,,,,,,,,,,,,110000.0,,Yes,20000.0,Cambridge Building Society,,10,Yes,80000.0,,,Yes,100.0,,10000.0
+,completed,,2023-12-08T00:00:00+00:00,2023-12-08T00:00:00+00:00,,2023,single log,false,DLUHC,DLUHC,billyboy@eyeklaud.com,8,12,2023,,Yes - a discounted ownership scheme,Right to Acquire (RTA),,,,Yes,Yes,,,Address line 1,,Town or city,,SW1A,1AA,1,E09000003,Barnet,2,Flat or maisonette,Purpose built,0,Yes,Yes,1,30,Non-binary,Buyer 1 prefers not to say,17,United Kingdom,Full-time - 30 hours or more,Yes,Partner,35,Non-binary,Buyer 2 prefers not to say,,Buyer prefers not to say,Full-time - 30 hours or more,Yes,3,Child,14,Non-binary,Child under 16,Other,Not known,Non-binary,"In government training into work, such as New Deal",Prefers not to say,Not known,Prefers not to say,Prefers not to say,,,,,Local authority tenant,No,,,No,,,1,1,1,1,,Don't know,,Yes,Yes,No,Yes,Yes,Yes,10000,Yes,Yes,10000,Yes,"Don’t know ",No,,Yes,No,10,,,,,,,,,,,,,,,,,110000.0,,Yes,20000.0,Cambridge Building Society,,10,Yes,80000.0,,,Yes,100.0,,10000.0

spec/lib/tasks/set_duplicate_references_spec.rb

@@ -0,0 +1,235 @@
+require "rails_helper"
+require "rake"
+
+RSpec.describe "set_duplicate_references" do
+  describe ":set_duplicate_references", type: :task do
+    subject(:task) { Rake::Task["set_duplicate_references"] }
+
+    before do
+      Rake.application.rake_require("tasks/set_duplicate_references")
+      Rake::Task.define_task(:environment)
+      task.reenable
+    end
+
+    context "when the rake task is run" do
+      context "and there are sales duplicates in 1 organisation" do
+        let(:user) { create(:user) }
+        let!(:sales_log) { create(:sales_log, :duplicate, created_by: user) }
+        let!(:duplicate_sales_log) { create(:sales_log, :duplicate, created_by: user) }
+        let!(:second_duplicate_sales_log) { create(:sales_log, :duplicate, created_by: user) }
+        let!(:sales_log_without_duplicates) { create(:sales_log, created_by: user) }
+
+        it "creates duplicate references for sales logs" do
+          initial_sales_log_updated_at = sales_log.updated_at
+          expect(sales_log.duplicates.count).to eq(0)
+          expect(sales_log.duplicate_set_id).to be_nil
+          expect(duplicate_sales_log.duplicates.count).to eq(0)
+          expect(duplicate_sales_log.duplicate_set_id).to be_nil
+          expect(second_duplicate_sales_log.duplicates.count).to eq(0)
+          expect(second_duplicate_sales_log.duplicate_set_id).to be_nil
+          expect(sales_log_without_duplicates.duplicates.count).to eq(0)
+          expect(sales_log_without_duplicates.duplicate_set_id).to be_nil
+
+          task.invoke
+          sales_log.reload
+          duplicate_sales_log.reload
+          second_duplicate_sales_log.reload
+          sales_log_without_duplicates.reload
+
+          expect(sales_log.duplicates.count).to eq(2)
+          expect(duplicate_sales_log.duplicates.count).to eq(2)
+          expect(second_duplicate_sales_log.duplicates.count).to eq(2)
+          expect(sales_log_without_duplicates.duplicates.count).to eq(0)
+          expect(sales_log.duplicate_set_id).to eq(duplicate_sales_log.duplicate_set_id)
+          expect(sales_log.duplicate_set_id).to eq(second_duplicate_sales_log.duplicate_set_id)
+          expect(sales_log.updated_at).to eq(initial_sales_log_updated_at)
+        end
+      end
+
+      context "and there are sales duplicates in multiple organisations" do
+        let(:user) { create(:user) }
+        let!(:sales_log) { create(:sales_log, :duplicate, created_by: user) }
+        let!(:duplicate_sales_log) { create(:sales_log, :duplicate, created_by: user) }
+        let!(:sales_log_without_duplicates) { create(:sales_log, created_by: user) }
+        let(:other_user) { create(:user) }
+        let!(:other_sales_log) { create(:sales_log, :duplicate, created_by: other_user) }
+        let!(:other_duplicate_sales_log) { create(:sales_log, :duplicate, created_by: other_user) }
+        let!(:other_sales_log_without_duplicates) { create(:sales_log, created_by: other_user) }
+
+        it "creates separate duplicate references for sales logs" do
+          expect(sales_log.duplicates.count).to eq(0)
+          expect(sales_log.duplicate_set_id).to be_nil
+          expect(duplicate_sales_log.duplicates.count).to eq(0)
+          expect(duplicate_sales_log.duplicate_set_id).to be_nil
+          expect(sales_log_without_duplicates.duplicates.count).to eq(0)
+          expect(sales_log_without_duplicates.duplicate_set_id).to be_nil
+
+          expect(other_sales_log.duplicates.count).to eq(0)
+          expect(other_sales_log.duplicate_set_id).to be_nil
+          expect(other_duplicate_sales_log.duplicates.count).to eq(0)
+          expect(other_duplicate_sales_log.duplicate_set_id).to be_nil
+          expect(other_sales_log_without_duplicates.duplicates.count).to eq(0)
+          expect(other_sales_log_without_duplicates.duplicate_set_id).to be_nil
+
+          task.invoke
+          sales_log.reload
+          duplicate_sales_log.reload
+          sales_log_without_duplicates.reload
+          other_sales_log.reload
+          other_duplicate_sales_log.reload
+
+          expect(sales_log.duplicates.count).to eq(1)
+          expect(duplicate_sales_log.duplicates.count).to eq(1)
+          expect(sales_log_without_duplicates.duplicates.count).to eq(0)
+          expect(sales_log.duplicate_set_id).to eq(duplicate_sales_log.duplicate_set_id)
+
+          expect(other_sales_log.duplicates.count).to eq(1)
+          expect(other_duplicate_sales_log.duplicates.count).to eq(1)
+          expect(other_sales_log_without_duplicates.duplicates.count).to eq(0)
+          expect(other_sales_log.duplicate_set_id).to eq(other_duplicate_sales_log.duplicate_set_id)
+          expect(other_sales_log.duplicate_set_id).not_to eq(sales_log.duplicate_set_id)
+        end
+      end
+
+      context "and there are sales duplicates for non 2023/24 collection period" do
+        let(:user) { create(:user) }
+        let!(:sales_log) { create(:sales_log, :duplicate, created_by: user) }
+        let!(:duplicate_sales_log) { create(:sales_log, :duplicate, created_by: user) }
+
+        before do
+          sales_log.saledate = Time.zone.local(2022, 4, 4)
+          sales_log.save!(validate: false)
+          duplicate_sales_log.saledate = Time.zone.local(2022, 4, 4)
+          duplicate_sales_log.save!(validate: false)
+        end
+
+        it "does not create duplicate references for sales logs" do
+          expect(sales_log.duplicates.count).to eq(0)
+          expect(sales_log.duplicate_set_id).to be_nil
+          expect(duplicate_sales_log.duplicates.count).to eq(0)
+          expect(duplicate_sales_log.duplicate_set_id).to be_nil
+
+          task.invoke
+          sales_log.reload
+          duplicate_sales_log.reload
+
+          expect(sales_log.duplicates.count).to eq(0)
+          expect(sales_log.duplicate_set_id).to be_nil
+          expect(duplicate_sales_log.duplicates.count).to eq(0)
+          expect(duplicate_sales_log.duplicate_set_id).to be_nil
+        end
+      end
+
+      context "and there are lettings duplicates in 1 organisation" do
+        let(:user) { create(:user) }
+        let!(:lettings_log) { create(:lettings_log, :duplicate, created_by: user) }
+        let!(:duplicate_lettings_log) { create(:lettings_log, :duplicate, created_by: user) }
+        let!(:second_duplicate_lettings_log) { create(:lettings_log, :duplicate, created_by: user) }
+        let!(:lettings_log_without_duplicates) { create(:lettings_log, created_by: user) }
+
+        it "creates duplicate references for lettings logs" do
+          initial_lettings_log_updated_at = lettings_log.updated_at
+          expect(lettings_log.duplicates.count).to eq(0)
+          expect(lettings_log.duplicate_set_id).to be_nil
+          expect(duplicate_lettings_log.duplicates.count).to eq(0)
+          expect(duplicate_lettings_log.duplicate_set_id).to be_nil
+          expect(second_duplicate_lettings_log.duplicates.count).to eq(0)
+          expect(second_duplicate_lettings_log.duplicate_set_id).to be_nil
+          expect(lettings_log_without_duplicates.duplicates.count).to eq(0)
+          expect(lettings_log_without_duplicates.duplicate_set_id).to be_nil
+
+          task.invoke
+          lettings_log.reload
+          duplicate_lettings_log.reload
+          second_duplicate_lettings_log.reload
+          lettings_log_without_duplicates.reload
+
+          expect(lettings_log.duplicates.count).to eq(2)
+          expect(duplicate_lettings_log.duplicates.count).to eq(2)
+          expect(second_duplicate_lettings_log.duplicates.count).to eq(2)
+          expect(lettings_log_without_duplicates.duplicates.count).to eq(0)
+          expect(lettings_log_without_duplicates.duplicate_set_id).to be_nil
+          expect(lettings_log.duplicate_set_id).to eq(duplicate_lettings_log.duplicate_set_id)
+          expect(lettings_log.duplicate_set_id).to eq(second_duplicate_lettings_log.duplicate_set_id)
+          expect(lettings_log.updated_at).to eq(initial_lettings_log_updated_at)
+        end
+      end
+
+      context "and there are lettings duplicates in multiple organisations" do
+        let(:user) { create(:user) }
+        let!(:lettings_log) { create(:lettings_log, :duplicate, created_by: user) }
+        let!(:duplicate_lettings_log) { create(:lettings_log, :duplicate, created_by: user) }
+        let!(:lettings_log_without_duplicates) { create(:lettings_log, created_by: user) }
+        let(:other_user) { create(:user) }
+        let!(:other_lettings_log) { create(:lettings_log, :duplicate, created_by: other_user) }
+        let!(:other_duplicate_lettings_log) { create(:lettings_log, :duplicate, created_by: other_user) }
+        let!(:other_lettings_log_without_duplicates) { create(:lettings_log, created_by: other_user) }
+
+        it "creates separate duplicate references for lettings logs" do
+          expect(lettings_log.duplicates.count).to eq(0)
+          expect(lettings_log.duplicate_set_id).to be_nil
+          expect(duplicate_lettings_log.duplicates.count).to eq(0)
+          expect(duplicate_lettings_log.duplicate_set_id).to be_nil
+          expect(lettings_log_without_duplicates.duplicates.count).to eq(0)
+          expect(lettings_log_without_duplicates.duplicate_set_id).to be_nil
+
+          expect(other_lettings_log.duplicates.count).to eq(0)
+          expect(other_lettings_log.duplicate_set_id).to be_nil
+          expect(other_duplicate_lettings_log.duplicates.count).to eq(0)
+          expect(other_duplicate_lettings_log.duplicate_set_id).to be_nil
+          expect(other_lettings_log_without_duplicates.duplicates.count).to eq(0)
+          expect(other_lettings_log_without_duplicates.duplicate_set_id).to be_nil
+
+          task.invoke
+          lettings_log.reload
+          duplicate_lettings_log.reload
+          lettings_log_without_duplicates.reload
+          other_lettings_log.reload
+          other_duplicate_lettings_log.reload
+
+          expect(lettings_log.duplicates.count).to eq(1)
+          expect(duplicate_lettings_log.duplicates.count).to eq(1)
+          expect(lettings_log_without_duplicates.duplicates.count).to eq(0)
+          expect(lettings_log_without_duplicates.duplicate_set_id).to be_nil
+          expect(lettings_log.duplicate_set_id).to eq(duplicate_lettings_log.duplicate_set_id)
+
+          expect(other_lettings_log.duplicates.count).to eq(1)
+          expect(other_duplicate_lettings_log.duplicates.count).to eq(1)
+          expect(other_lettings_log_without_duplicates.duplicates.count).to eq(0)
+          expect(other_lettings_log_without_duplicates.duplicate_set_id).to be_nil
+          expect(other_lettings_log.duplicate_set_id).to eq(other_duplicate_lettings_log.duplicate_set_id)
+          expect(other_lettings_log.duplicate_set_id).not_to eq(lettings_log.duplicate_set_id)
+        end
+      end
+
+      context "and there are lettings duplicates for non 2023/24 collection period" do
+        let(:user) { create(:user) }
+        let!(:lettings_log) { create(:lettings_log, :duplicate, created_by: user) }
+        let!(:duplicate_lettings_log) { create(:lettings_log, :duplicate, created_by: user) }
+
+        before do
+          lettings_log.startdate = Time.zone.local(2022, 4, 4)
+          lettings_log.save!(validate: false)
+          duplicate_lettings_log.startdate = Time.zone.local(2022, 4, 4)
+          duplicate_lettings_log.save!(validate: false)
+        end
+
+        it "does not create duplicate references for lettings logs" do
+          expect(lettings_log.duplicates.count).to eq(0)
+          expect(lettings_log.duplicate_set_id).to be_nil
+          expect(duplicate_lettings_log.duplicates.count).to eq(0)
+          expect(duplicate_lettings_log.duplicate_set_id).to be_nil
+
+          task.invoke
+          lettings_log.reload
+          duplicate_lettings_log.reload
+
+          expect(lettings_log.duplicates.count).to eq(0)
+          expect(lettings_log.duplicate_set_id).to be_nil
+          expect(duplicate_lettings_log.duplicates.count).to eq(0)
+          expect(duplicate_lettings_log.duplicate_set_id).to be_nil
+        end
+      end
+    end
+  end
+end

spec/requests/delete_logs_controller_spec.rb

@@ -247,6 +247,53 @@ RSpec.describe "DeleteLogs", type: :request do
         expect(log_2.discarded_at).to be nil
       end
     end
+
+    context "when an authorized user deletes a log that had duplicates" do
+      context "and only 1 log remains in the duplicate set" do
+        let!(:log_1) { create(:lettings_log, :duplicate, duplicate_set_id: 5, created_by: user) }
+        let!(:log_2) { create(:lettings_log, :duplicate, duplicate_set_id: 5, created_by: user) }
+        let!(:log_3) { create(:lettings_log, :duplicate, duplicate_set_id: 5, created_by: user) }
+
+        it "deletes the log and marks related logs deduplicated" do
+          delete delete_logs_lettings_logs_path, params: params
+          log_1.reload
+          expect(log_1.status).to eq "deleted"
+          expect(log_1.discarded_at).not_to be nil
+          expect(log_1.duplicates.count).to eq(0)
+          expect(log_1.duplicate_set_id).to be nil
+          log_2.reload
+          expect(log_2.status).to eq "deleted"
+          expect(log_2.discarded_at).not_to be nil
+          expect(log_2.duplicates.count).to eq(0)
+          expect(log_2.duplicate_set_id).to be nil
+          log_3.reload
+          expect(log_3.duplicates.count).to eq(0)
+          expect(log_3.duplicate_set_id).to be nil
+        end
+      end
+
+      context "and multiple logs remains in the duplicate set" do
+        let!(:log_1) { create(:lettings_log, :duplicate, duplicate_set_id: 5, created_by: user) }
+        let!(:log_2) { create(:lettings_log, :duplicate, duplicate_set_id: 5, created_by: user) }
+        let!(:log_3) { create(:lettings_log, :duplicate, duplicate_set_id: 5, created_by: user) }
+        let(:params) { { ids: [log_1.id] } }
+
+        it "deletes the log and marks related logs deduplicated" do
+          delete delete_logs_lettings_logs_path, params: params
+          log_1.reload
+          expect(log_1.status).to eq "deleted"
+          expect(log_1.discarded_at).not_to be nil
+          expect(log_1.duplicates.count).to eq(0)
+          expect(log_1.duplicate_set_id).to be nil
+          log_2.reload
+          log_3.reload
+          expect(log_2.duplicates.count).to eq(1)
+          expect(log_3.duplicates.count).to eq(1)
+          expect(log_3.duplicate_set_id).not_to be nil
+          expect(log_3.duplicate_set_id).to eq(log_2.duplicate_set_id)
+        end
+      end
+    end
   end
 
   describe "GET sales-logs/delete-logs" do
@@ -487,6 +534,53 @@ RSpec.describe "DeleteLogs", type: :request do
         expect(log_2.discarded_at).to be nil
       end
     end
+
+    context "when an authorized user deletes a log that had duplicates" do
+      context "and only 1 log remains in the duplicate set" do
+        let!(:log_1) { create(:sales_log, :duplicate, duplicate_set_id: 5, created_by: user) }
+        let!(:log_2) { create(:sales_log, :duplicate, duplicate_set_id: 5, created_by: user) }
+        let!(:log_3) { create(:sales_log, :duplicate, duplicate_set_id: 5, created_by: user) }
+
+        it "deletes the log and marks related logs deduplicated" do
+          delete delete_logs_sales_logs_path, params: params
+          log_1.reload
+          expect(log_1.status).to eq "deleted"
+          expect(log_1.discarded_at).not_to be nil
+          expect(log_1.duplicates.count).to eq(0)
+          expect(log_1.duplicate_set_id).to be nil
+          log_2.reload
+          expect(log_2.status).to eq "deleted"
+          expect(log_2.discarded_at).not_to be nil
+          expect(log_2.duplicates.count).to eq(0)
+          expect(log_2.duplicate_set_id).to be nil
+          log_3.reload
+          expect(log_3.duplicates.count).to eq(0)
+          expect(log_3.duplicate_set_id).to be nil
+        end
+      end
+
+      context "and multiple logs remains in the duplicate set" do
+        let!(:log_1) { create(:sales_log, :duplicate, duplicate_set_id: 5, created_by: user) }
+        let!(:log_2) { create(:sales_log, :duplicate, duplicate_set_id: 5, created_by: user) }
+        let!(:log_3) { create(:sales_log, :duplicate, duplicate_set_id: 5, created_by: user) }
+        let(:params) { { ids: [log_1.id] } }
+
+        it "deletes the log and marks related logs deduplicated" do
+          delete delete_logs_sales_logs_path, params: params
+          log_1.reload
+          expect(log_1.status).to eq "deleted"
+          expect(log_1.discarded_at).not_to be nil
+          expect(log_1.duplicates.count).to eq(0)
+          expect(log_1.duplicate_set_id).to be nil
+          log_2.reload
+          log_3.reload
+          expect(log_2.duplicates.count).to eq(1)
+          expect(log_3.duplicates.count).to eq(1)
+          expect(log_3.duplicate_set_id).not_to be nil
+          expect(log_3.duplicate_set_id).to eq(log_2.duplicate_set_id)
+        end
+      end
+    end
   end
 
   context "when a support user navigates to the organisations tab" do

spec/requests/form_controller_spec.rb

@@ -685,7 +685,7 @@ RSpec.describe FormController, type: :request do
           end
 
           context "and duplicate logs" do
-            let(:duplicate_logs) { create_list(:sales_log, 2) }
+            let!(:duplicate_logs) { create_list(:sales_log, 2) }
 
             before do
               allow(SalesLog).to receive(:duplicate_logs).and_return(duplicate_logs)
@@ -882,8 +882,8 @@ RSpec.describe FormController, type: :request do
         end
 
         context "when the question was accessed from a duplicate logs screen" do
-          let(:lettings_log) { create(:lettings_log, :duplicate, created_by: user) }
+          let(:lettings_log) { create(:lettings_log, :duplicate, created_by: user, duplicate_set_id: 1) }
-          let(:duplicate_log) { create(:lettings_log, :duplicate, created_by: user) }
+          let(:duplicate_log) { create(:lettings_log, :duplicate, created_by: user, duplicate_set_id: 1) }
           let(:referrer) { "/lettings-logs/#{lettings_log.id}/lead-tenant-age?referrer=duplicate_logs&first_remaining_duplicate_id=#{duplicate_log.id}&original_log_id=#{lettings_log.id}" }
           let(:params) do
             {
@@ -896,12 +896,30 @@ RSpec.describe FormController, type: :request do
             }
           end
 
+          context "and the answer changes" do
             before do
               post "/lettings-logs/#{lettings_log.id}/lead-tenant-age", params:, headers: headers.merge({ "HTTP_REFERER" => referrer })
             end
 
             it "redirects back to the duplicates page for remaining duplicates" do
               expect(response).to redirect_to("/lettings-logs/#{duplicate_log.id}/duplicate-logs?original_log_id=#{lettings_log.id}")
+              expect(lettings_log.duplicates.count).to eq(0)
+              expect(duplicate_log.duplicates.count).to eq(0)
+            end
+          end
+
+          context "and updating the answer creates a different set of duplicates" do
+            let!(:another_duplicate_log) { create(:lettings_log, :duplicate, created_by: user, age1_known: 1, age1: 20) }
+
+            before do
+              post "/lettings-logs/#{lettings_log.id}/lead-tenant-age", params:, headers: headers.merge({ "HTTP_REFERER" => referrer })
+            end
+
+            it "correctly assigs duplicate set IDs" do
+              expect(lettings_log.reload.duplicates.count).to eq(1)
+              expect(lettings_log.duplicate_set_id).to eq(another_duplicate_log.reload.duplicate_set_id)
+              expect(duplicate_log.reload.duplicates.count).to eq(0)
+            end
           end
 
           context "and the answer didn't change" do
@@ -916,15 +934,21 @@ RSpec.describe FormController, type: :request do
               }
             end
 
+            before do
+              post "/lettings-logs/#{lettings_log.id}/lead-tenant-age", params:, headers: headers.merge({ "HTTP_REFERER" => referrer })
+            end
+
             it "redirects back to the duplicates page for remaining duplicates" do
               expect(response).to redirect_to("/lettings-logs/#{lettings_log.id}/duplicate-logs?original_log_id=#{lettings_log.id}")
+              expect(lettings_log.duplicates.count).to eq(1)
+              expect(duplicate_log.duplicates.count).to eq(1)
             end
           end
         end
 
         context "when the sales question was accessed from a duplicate logs screen" do
-          let(:sales_log) { create(:sales_log, :duplicate, created_by: user) }
+          let!(:sales_log) { create(:sales_log, :duplicate, created_by: user, duplicate_set_id: 1) }
-          let(:duplicate_log) { create(:sales_log, :duplicate, created_by: user) }
+          let!(:duplicate_log) { create(:sales_log, :duplicate, created_by: user, duplicate_set_id: 1) }
           let(:referrer) { "/sales-logs/#{sales_log.id}/buyer-1-age?referrer=duplicate_logs&first_remaining_duplicate_id=#{duplicate_log.id}&original_log_id=#{sales_log.id}&referrer=duplicate_logs" }
           let(:params) do
             {
@@ -943,6 +967,10 @@ RSpec.describe FormController, type: :request do
 
           it "redirects back to the duplicates page for remaining duplicates" do
             expect(response).to redirect_to("/sales-logs/#{duplicate_log.id}/duplicate-logs?original_log_id=#{sales_log.id}")
+            sales_log.reload
+            duplicate_log.reload
+            expect(sales_log.duplicates.count).to eq(0)
+            expect(duplicate_log.duplicates.count).to eq(0)
           end
 
           context "and the answer didn't change" do
@@ -959,6 +987,8 @@ RSpec.describe FormController, type: :request do
 
             it "redirects back to the duplicates page for remaining duplicates" do
               expect(response).to redirect_to("/sales-logs/#{sales_log.id}/duplicate-logs?original_log_id=#{sales_log.id}")
+              expect(sales_log.duplicates.count).to eq(1)
+              expect(duplicate_log.duplicates.count).to eq(1)
             end
           end
         end

spec/services/csv/lettings_log_csv_service_spec.rb

@@ -116,7 +116,7 @@ RSpec.describe Csv::LettingsLogCsvService do
     end
 
     it "adds log attributes not related to questions to the headers" do
-      expect(headers.first(5)).to eq %w[id status created_by is_dpo created_at]
+      expect(headers.first(5)).to eq %w[id status duplicate_set_id created_by is_dpo]
     end
 
     it "adds attributes related to associated schemes and locations to the headers" do
@@ -170,6 +170,18 @@ RSpec.describe Csv::LettingsLogCsvService do
         end
         expect(csv).to eq expected_content
       end
+
+      context "when the log has a duplicate log reference" do
+        before do
+          log.update!(duplicate_set_id: 12_312)
+        end
+
+        it "exports the id for under the heading 'duplicate_set_id'" do
+          duplicate_set_id_column_index = csv.first.index("duplicate_set_id")
+          duplicate_set_id_value = csv.second[duplicate_set_id_column_index]
+          expect(duplicate_set_id_value).to eq "12312"
+        end
+      end
     end
 
     context "when exporting as codes" do
@@ -208,6 +220,18 @@ RSpec.describe Csv::LettingsLogCsvService do
         end
         expect(csv).to eq expected_content
       end
+
+      context "when the log has a duplicate log reference" do
+        before do
+          log.update!(duplicate_set_id: 12_312)
+        end
+
+        it "exports the id for under the heading 'duplicate_set_id'" do
+          duplicate_set_id_column_index = csv.first.index("duplicate_set_id")
+          duplicate_set_id_value = csv.second[duplicate_set_id_column_index]
+          expect(duplicate_set_id_value).to eq "12312"
+        end
+      end
     end
 
     context "when the user is not a support user" do

spec/services/csv/sales_log_csv_service_spec.rb

@@ -163,6 +163,18 @@ RSpec.describe Csv::SalesLogCsvService do
       end
       expect(csv).to eq expected_content
     end
+
+    context "when the log has a duplicate log reference" do
+      before do
+        log.update!(duplicate_set_id: 12_312)
+      end
+
+      it "exports the id for under the heading 'duplicate_set_id'" do
+        duplicate_set_id_column_index = csv.first.index("duplicate_set_id")
+        duplicate_set_id_value = csv.second[duplicate_set_id_column_index]
+        expect(duplicate_set_id_value).to eq "12312"
+      end
+    end
   end
 
   context "when exporting values as codes" do
@@ -211,5 +223,17 @@ RSpec.describe Csv::SalesLogCsvService do
       end
       expect(csv).to eq expected_content
     end
+
+    context "when the log has a duplicate log reference" do
+      before do
+        log.update!(duplicate_set_id: 12_312)
+      end
+
+      it "exports the id for under the heading 'duplicate_set_id'" do
+        duplicate_set_id_column_index = csv.first.index("duplicate_set_id")
+        duplicate_set_id_value = csv.second[duplicate_set_id_column_index]
+        expect(duplicate_set_id_value).to eq "12312"
+      end
+    end
   end
 end

spec/services/exports/lettings_log_export_service_spec.rb

@@ -413,6 +413,26 @@ RSpec.describe Exports::LettingsLogExportService do
         export_service.export_xml_lettings_logs
       end
     end
+
+    context "and one lettings log with duplicate reference is available for export" do
+      let!(:lettings_log) { FactoryBot.create(:lettings_log, :completed, created_by: user, propcode: "123", ppostcode_full: "SE2 6RT", postcode_full: "NW1 5TY", tenancycode: "BZ737", startdate: Time.zone.local(2022, 2, 2, 10, 36, 49), voiddate: Time.zone.local(2019, 11, 3), mrcdate: Time.zone.local(2020, 5, 5, 10, 36, 49), tenancylength: 5, underoccupation_benefitcap: 4, duplicate_set_id: 123) }
+
+      def replace_duplicate_set_id(export_file)
+        export_file.sub!("<duplicate_set_id/>", "<duplicate_set_id>123</duplicate_set_id>")
+      end
+
+      it "generates an XML export file with the expected content within the ZIP file" do
+        expected_content = replace_entity_ids(lettings_log, xml_export_file.read)
+        expected_content = replace_duplicate_set_id(expected_content)
+        expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) do |_, content|
+          entry = Zip::File.open_buffer(content).find_entry(expected_data_filename)
+          expect(entry).not_to be_nil
+          expect(entry.get_input_stream.read).to eq(expected_content)
+        end
+
+        export_service.export_xml_lettings_logs
+      end
+    end
   end
 
   context "when exporting a supported housing lettings logs in XML" do