CLDC-3928++ Update logs with manually entered address showing the property information as incomplete (#3004)

* set renewal ppostcode on postcode change

* update test

* Update factory bot and set uprn_selection to uprn as in derived fields

* Revert "update test"

This reverts commit 3b34cd0086.

* Revert "set renewal ppostcode on postcode change"

This reverts commit d311c8e8ad.

* Rake task and tests

* Lint

* Update tests

* Remove flaky behaviour

* Update rake task with error logging

* Revert "Remove flaky behaviour"

This reverts commit d51a03f88e.

* Try to prevent flaky test behaviour

* Check status before and after change

* Do not update logs without any address fields yet entered

* Rename trait

* Update query

* Empty line

* Implement postcode fix and log ids of status changed

* Add rake task to fix affected logs after tasks first run

* Simplify query

* Update task and use bulk upload value, additional logging

* Update to handle failing save

* Update tests

* Add bu created log test

* Add task reenable

* Additional logging

* rename

---------

Co-authored-by: Carolyn <carolyn.barker@softwire.com>

lib/tasks/update_manual_address_entry_selected_prexisting_logs.rake

@@ -2,7 +2,21 @@ namespace :bulk_update do
   desc "Update logs with specific criteria and set manual_address_entry_selected to true"
   task update_manual_address_entry_selected: :environment do
     updated_lettings_logs_count = 0
+    lettings_postcode_fixed_count = 0
+    lettings_postcode_fixed_status_changed_count = 0
+    lettings_postcode_not_fixed_status_changed_count = 0
+    lettings_postcode_fixed_status_changed_ids = []
+    lettings_postcode_not_fixed_status_changed_ids = []
+    lettings_updated_without_issue = 0
+
     updated_sales_logs_count = 0
+    sales_postcode_fixed_count = 0
+    sales_postcode_fixed_status_changed_count = 0
+    sales_postcode_not_fixed_status_changed_count = 0
+    sales_postcode_fixed_status_changed_ids = []
+    sales_postcode_not_fixed_status_changed_ids = []
+    sales_updated_without_issue = 0
+
     lettings_logs = LettingsLog.filter_by_year(2024)
                                .where(status: %w[in_progress completed])
                                .where(needstype: 1, manual_address_entry_selected: false, uprn: nil)
@@ -13,16 +27,47 @@ namespace :bulk_update do
       log.manual_address_entry_selected = true
       if log.save
         updated_lettings_logs_count += 1
+        Rails.logger.info "manual_address_entry_selected updated for lettings log #{log.id}"
       else
-        Rails.logger.info "Could not save changes to lettings log #{log.id}"
+        Rails.logger.info "Could not save manual_address_entry_selected changes to lettings log #{log.id} : #{log.errors.full_messages.join(', ')}"
+      end
+
+      postcode_fixed = false
+      if log.postcode_full.nil? && log.address_line1 == log.address_line1_input
+        log.postcode_full = log.postcode_full_input
+        if log.save
+          lettings_postcode_fixed_count += 1
+          Rails.logger.info "postcode_full updated by address_line1_input for lettings log #{log.id}"
+          postcode_fixed = true
+        else
+          Rails.logger.info "Could not save postcode_full changes to lettings log #{log.id} : #{log.errors.full_messages.join(', ')}"
         end
-      status_post_change = log.status
+      end
-      unless status_pre_change == status_post_change
+
-        Rails.logger.info "Status changed from #{status_pre_change} to #{status_post_change} for lettings log #{log.id}"
+      if log.postcode_full.nil? && log.creation_method == "bulk upload" && log.address_line1 == log.address_line1_as_entered
+        log.postcode_full = log.postcode_full_as_entered
+        if log.save
+          lettings_postcode_fixed_count += 1
+          Rails.logger.info "postcode_full updated by address_line1_as_entered for lettings log #{log.id}"
+          postcode_fixed = true
+        else
+          Rails.logger.info "Could not save postcode_full changes to lettings log #{log.id} : #{log.errors.full_messages.join(', ')}"
         end
       end
 
-    puts "#{updated_lettings_logs_count} lettings logs updated."
+      status_post_change = log.status
+      if status_pre_change != status_post_change
+        if postcode_fixed
+          lettings_postcode_fixed_status_changed_count += 1
+          lettings_postcode_fixed_status_changed_ids << log.id
+        else
+          lettings_postcode_not_fixed_status_changed_count += 1
+          lettings_postcode_not_fixed_status_changed_ids << log.id
+        end
+      else
+        lettings_updated_without_issue += 1
+      end
+    end
 
     sales_logs = SalesLog.filter_by_year(2024)
                          .where(status: %w[in_progress completed])
@@ -34,15 +79,129 @@ namespace :bulk_update do
       log.manual_address_entry_selected = true
       if log.save
         updated_sales_logs_count += 1
+        Rails.logger.info "manual_address_entry_selected updated for sales log #{log.id}"
+      else
+        Rails.logger.info "Could not save manual_address_entry_selected changes to sales log #{log.id} : #{log.errors.full_messages.join(', ')}"
+      end
+
+      postcode_fixed = false
+      if log.postcode_full.nil? && log.address_line1 == log.address_line1_input
+        log.postcode_full = log.postcode_full_input
+        if log.save
+          sales_postcode_fixed_count += 1
+          Rails.logger.info "postcode_full updated by address_line1_input for sales log #{log.id}"
+          postcode_fixed = true
         else
-        Rails.logger.info "Could not save changes to sales log #{log.id}"
+          Rails.logger.info "Could not save postcode_full changes to sales log #{log.id} : #{log.errors.full_messages.join(', ')}"
         end
+      end
+
+      if log.postcode_full.nil? && log.creation_method == "bulk upload" && log.address_line1 == log.address_line1_as_entered
+        log.postcode_full = log.postcode_full_as_entered
+        if log.save
+          sales_postcode_fixed_count += 1
+          Rails.logger.info "postcode_full updated by address_line1_as_entered for sales log #{log.id}"
+          postcode_fixed = true
+        else
+          Rails.logger.info "Could not save postcode_full changes to sales log #{log.id} : #{log.errors.full_messages.join(', ')}"
+        end
+      end
+
       status_post_change = log.status
-      unless status_pre_change == status_post_change
+      if status_pre_change != status_post_change
-        Rails.logger.info "Status changed from #{status_pre_change} to #{status_post_change} for sales log #{log.id}"
+        if postcode_fixed
+          sales_postcode_fixed_status_changed_count += 1
+          sales_postcode_fixed_status_changed_ids << log.id
+        else
+          sales_postcode_not_fixed_status_changed_count += 1
+          sales_postcode_not_fixed_status_changed_ids << log.id
+        end
+      else
+        sales_updated_without_issue += 1
+      end
+    end
+
+    puts "#{updated_lettings_logs_count} lettings logs were updated."
+    puts "#{lettings_updated_without_issue} lettings logs were updated without issue."
+    puts "#{lettings_postcode_fixed_count} lettings logs where postcode fix was applied."
+    puts "#{lettings_postcode_fixed_status_changed_count} lettings logs with postcode fix and status changed."
+    puts "#{lettings_postcode_not_fixed_status_changed_count} lettings logs without postcode fix and status changed."
+    puts "IDs of lettings logs with postcode fix and status changed: [#{lettings_postcode_fixed_status_changed_ids.join(', ')}]"
+    puts "IDs of lettings logs without postcode fix and status changed: [#{lettings_postcode_not_fixed_status_changed_ids.join(', ')}]"
+
+    lettings_postcode_fixed_org_counts = LettingsLog.where(id: lettings_postcode_fixed_status_changed_ids).group(:owning_organisation_id).count
+    lettings_postcode_fixed_org_counts.each do |org_id, count|
+      puts "Org #{org_id}: #{count} logs with postcode fix and status changed."
+    end
+
+    lettings_postcode_not_fixed_org_counts = LettingsLog.where(id: lettings_postcode_not_fixed_status_changed_ids).group(:owning_organisation_id).count
+    lettings_postcode_not_fixed_org_counts.each do |org_id, count|
+      puts "Org #{org_id}: #{count} logs without postcode fix and status changed."
+    end
+
+    puts "#{updated_sales_logs_count} sales logs were updated."
+    puts "#{sales_updated_without_issue} sales logs were updated without issue."
+    puts "#{sales_postcode_fixed_count} sales logs where postcode fix was applied."
+    puts "#{sales_postcode_fixed_status_changed_count} sales logs with postcode fix and status changed."
+    puts "#{sales_postcode_not_fixed_status_changed_count} sales logs without postcode fix and status changed."
+    puts "IDs of sales logs with postcode fix and status changed: [#{sales_postcode_fixed_status_changed_ids.join(', ')}]"
+    puts "IDs of sales logs without postcode fix and status changed: [#{sales_postcode_not_fixed_status_changed_ids.join(', ')}]"
+
+    sales_postcode_fixed_org_counts = SalesLog.where(id: sales_postcode_fixed_status_changed_ids).group(:owning_organisation_id).count
+    sales_postcode_fixed_org_counts.each do |org_id, count|
+      puts "Org #{org_id}: #{count} logs with postcode fix and status changed."
+    end
+
+    sales_postcode_not_fixed_org_counts = SalesLog.where(id: sales_postcode_not_fixed_status_changed_ids).group(:owning_organisation_id).count
+    sales_postcode_not_fixed_org_counts.each do |org_id, count|
+      puts "Org #{org_id}: #{count} logs without postcode fix and status changed."
+    end
+  end
+
+  desc "Find logs to fix and update postcode_full if conditions are met"
+  task update_postcode_full_preexisting_manual_entry_logs: :environment do
+    updated_count = 0
+    fixed_count = 0
+    not_updated_count = 0
+    not_updated_ids = []
+    updated_but_not_fixed_ids = []
+
+    logs_to_fix = LettingsLog.filter_by_year(2024).where(manual_address_entry_selected: true, uprn: nil, status: "in_progress", postcode_full: nil, updated_at: Time.zone.parse("2025-03-19 16:00:00")..Time.zone.parse("2025-03-19 17:00:00"))
+
+    logs_to_fix.find_each do |log|
+      previous_version = log.versions[-2]
+      previous_status = previous_version&.reify&.status
+
+      if log.address_line1 == log.address_line1_input
+        log.postcode_full = log.postcode_full_input
+      elsif log.creation_method == "bulk upload" && log.address_line1 == log.address_line1_as_entered
+        log.postcode_full = log.postcode_full_as_entered
+      end
+
+      if log.postcode_full.present?
+        if log.save
+          Rails.logger.info "Updated postcode_full for lettings log #{log.id}"
+          updated_count += 1
+          if log.status == previous_status
+            fixed_count += 1
+          else
+            updated_but_not_fixed_ids << log.id
+          end
+        else
+          Rails.logger.info "Could not save changes to lettings log #{log.id}: #{log.errors.full_messages.join(', ')}"
+          not_updated_count += 1
+          not_updated_ids << log.id
+        end
+      else
+        not_updated_count += 1
+        not_updated_ids << log.id
       end
     end
 
-    puts "#{updated_sales_logs_count} sales logs updated."
+    puts "#{updated_count} logs updated."
+    puts "#{fixed_count} logs fixed."
+    puts "#{not_updated_count} logs not updated."
+    puts "IDs of logs not updated: [#{not_updated_ids.join(', ')}]"
+    puts "IDs of logs updated but not fixed: [#{updated_but_not_fixed_ids.join(', ')}]"
   end
 end

spec/lib/tasks/update_manual_address_entry_selected_prexisting_logs_spec.rb

@@ -1,7 +1,14 @@
 require "rails_helper"
 require "rake"
 
-RSpec.describe "bulk_update:update_manual_address_entry_selected", type: :task do
+RSpec.describe "update_manual_address_entry_selected_preexisting_logs_spec", type: :task do
+  before do
+    Rake.application.rake_require("tasks/update_manual_address_entry_selected_prexisting_logs")
+    Rake::Task.define_task(:environment)
+    task.reenable
+  end
+
+  describe "bulk_update:update_manual_address_entry_selected" do
     let(:task) { Rake::Task["bulk_update:update_manual_address_entry_selected"] }
 
     let(:lettings_log_uprn_entered) do
@@ -36,17 +43,13 @@ RSpec.describe "bulk_update:update_manual_address_entry_selected", type: :task d
       build(:sales_log, :completed_without_uprn, saledate: Time.zone.local(2024, 12, 30))
     end
 
-  before do
-    Rake.application.rake_require("tasks/update_manual_address_entry_selected_prexisting_logs")
-    Rake::Task.define_task(:environment)
-  end
-
     context "when running the task" do
       context "when logs do not meet the criteria" do
         before do
           lettings_log_uprn_found.save!(validate: false)
           lettings_log_uprn_entered.save!(validate: false)
           lettings_log_address_fields_not_entered.save!(validate: false)
+
           sales_log_uprn_found.save!(validate: false)
           sales_log_uprn_entered.save!(validate: false)
           sales_log_address_fields_not_entered.save!(validate: false)
@@ -54,8 +57,10 @@ RSpec.describe "bulk_update:update_manual_address_entry_selected", type: :task d
 
         it "does not update logs with a UPRN entered" do
           task.invoke
+
           lettings_log_uprn_entered.reload
           sales_log_uprn_entered.reload
+
           expect(lettings_log_uprn_entered.manual_address_entry_selected).to be false
           expect(lettings_log_uprn_entered.uprn).to eq("10033558653")
           expect(sales_log_uprn_entered.manual_address_entry_selected).to be false
@@ -64,8 +69,10 @@ RSpec.describe "bulk_update:update_manual_address_entry_selected", type: :task d
 
         it "does not update logs with a UPRN found" do
           task.invoke
+
           lettings_log_uprn_found.reload
           sales_log_uprn_found.reload
+
           expect(lettings_log_uprn_found.manual_address_entry_selected).to be false
           expect(lettings_log_uprn_found.uprn).to eq("10033558653")
           expect(sales_log_uprn_found.manual_address_entry_selected).to be false
@@ -74,8 +81,10 @@ RSpec.describe "bulk_update:update_manual_address_entry_selected", type: :task d
 
         it "does not update logs with no UPRN or address fields entered" do
           task.invoke
+
           lettings_log_address_fields_not_entered.reload
           sales_log_address_fields_not_entered.reload
+
           expect(lettings_log_address_fields_not_entered.manual_address_entry_selected).to be false
           expect(sales_log_address_fields_not_entered.manual_address_entry_selected).to be false
         end
@@ -85,6 +94,7 @@ RSpec.describe "bulk_update:update_manual_address_entry_selected", type: :task d
         before do
           lettings_log_address_manually_entered.manual_address_entry_selected = false
           lettings_log_address_manually_entered.save!(validate: false)
+
           sales_log_address_manually_entered.manual_address_entry_selected = false
           sales_log_address_manually_entered.save!(validate: false)
         end
@@ -95,6 +105,7 @@ RSpec.describe "bulk_update:update_manual_address_entry_selected", type: :task d
           expect(lettings_log_address_manually_entered.address_line2).to eq("Testville")
           expect(lettings_log_address_manually_entered.town_or_city).to eq("Testford")
           expect(lettings_log_address_manually_entered.postcode_full).to eq("SW1 1AA")
+
           expect(sales_log_address_manually_entered.manual_address_entry_selected).to be false
           expect(sales_log_address_manually_entered.address_line1).to eq("1 Test Street")
           expect(sales_log_address_manually_entered.address_line2).to eq("Testville")
@@ -102,6 +113,7 @@ RSpec.describe "bulk_update:update_manual_address_entry_selected", type: :task d
           expect(sales_log_address_manually_entered.postcode_full).to eq("SW1 1AA")
 
           task.invoke
+
           lettings_log_address_manually_entered.reload
           sales_log_address_manually_entered.reload
 
@@ -110,6 +122,7 @@ RSpec.describe "bulk_update:update_manual_address_entry_selected", type: :task d
           expect(lettings_log_address_manually_entered.address_line2).to eq("Testville")
           expect(lettings_log_address_manually_entered.town_or_city).to eq("Testford")
           expect(lettings_log_address_manually_entered.postcode_full).to eq("SW1 1AA")
+
           expect(sales_log_address_manually_entered.manual_address_entry_selected).to be true
           expect(sales_log_address_manually_entered.address_line1).to eq("1 Test Street")
           expect(sales_log_address_manually_entered.address_line2).to eq("Testville")
@@ -118,4 +131,81 @@ RSpec.describe "bulk_update:update_manual_address_entry_selected", type: :task d
         end
       end
     end
+  end
+
+  describe "bulk_update:update_postcode_full_preexisting_manual_entry_logs" do
+    let(:task) { Rake::Task["bulk_update:update_postcode_full_preexisting_manual_entry_logs"] }
+
+    let(:lettings_log_to_fix) do
+      build(:lettings_log, :inprogress_without_address_fields, startdate: Time.zone.local(2024, 6, 1), updated_at: Time.zone.parse("2025-03-19 16:30:00"))
+    end
+
+    let(:bu_lettings_log_to_fix) do
+      build(:lettings_log, :inprogress_without_address_fields, startdate: Time.zone.local(2024, 6, 1), creation_method: "bulk upload", updated_at: Time.zone.parse("2025-03-19 16:30:00"))
+    end
+
+    let(:lettings_log_not_to_fix) do
+      build(:lettings_log, :inprogress_without_address_fields, startdate: Time.zone.local(2024, 6, 1), updated_at: Time.zone.parse("2025-03-19 15:30:00"))
+    end
+
+    before do
+      lettings_log_to_fix.manual_address_entry_selected = true
+      lettings_log_to_fix.address_line1 = "1 Test Street"
+      lettings_log_to_fix.address_line2 = "Testville"
+      lettings_log_to_fix.town_or_city = "Testford"
+      lettings_log_to_fix.postcode_full = nil
+      lettings_log_to_fix.address_line1_input = "1 Test Street"
+      lettings_log_to_fix.postcode_full_input = "SW1 2BB"
+      lettings_log_to_fix.save!(validate: false)
+
+      bu_lettings_log_to_fix.manual_address_entry_selected = true
+      bu_lettings_log_to_fix.address_line1 = "1 Test Street"
+      bu_lettings_log_to_fix.address_line2 = "Testville"
+      bu_lettings_log_to_fix.town_or_city = "Testford"
+      bu_lettings_log_to_fix.postcode_full = nil
+      bu_lettings_log_to_fix.address_line1_as_entered = "1 Test Street"
+      bu_lettings_log_to_fix.postcode_full_as_entered = "SW1 2BB"
+      bu_lettings_log_to_fix.save!(validate: false)
+
+      lettings_log_not_to_fix.postcode_full = nil
+      lettings_log_not_to_fix.save!(validate: false)
+    end
+
+    context "when running the task" do
+      it "updates logs that meet the criteria" do
+        expect(lettings_log_to_fix.postcode_full).to be_nil
+        expect(lettings_log_to_fix.address_line1).to eq("1 Test Street")
+        expect(lettings_log_to_fix.address_line2).to eq("Testville")
+        expect(lettings_log_to_fix.town_or_city).to eq("Testford")
+        expect(lettings_log_to_fix.address_line1_input).to eq("1 Test Street")
+        expect(lettings_log_to_fix.postcode_full_input).to eq("SW1 2BB")
+
+        expect(bu_lettings_log_to_fix.postcode_full).to be_nil
+        expect(bu_lettings_log_to_fix.address_line1_input).to be_nil
+        expect(bu_lettings_log_to_fix.address_line1).to eq("1 Test Street")
+        expect(bu_lettings_log_to_fix.address_line2).to eq("Testville")
+        expect(bu_lettings_log_to_fix.town_or_city).to eq("Testford")
+        expect(bu_lettings_log_to_fix.address_line1_as_entered).to eq("1 Test Street")
+        expect(bu_lettings_log_to_fix.postcode_full_as_entered).to eq("SW1 2BB")
+
+        task.invoke
+
+        lettings_log_to_fix.reload
+        bu_lettings_log_to_fix.reload
+
+        expect(lettings_log_to_fix.postcode_full).to eq(lettings_log_to_fix.postcode_full_input)
+        expect(lettings_log_to_fix.postcode_full).to eq("SW1 2BB")
+        expect(bu_lettings_log_to_fix.postcode_full).to eq(bu_lettings_log_to_fix.postcode_full_as_entered)
+        expect(bu_lettings_log_to_fix.postcode_full).to eq("SW1 2BB")
+      end
+
+      it "does not update logs that do not meet the criteria" do
+        task.invoke
+
+        lettings_log_not_to_fix.reload
+
+        expect(lettings_log_not_to_fix.postcode_full).to be_nil
+      end
+    end
+  end
 end