CLDC-2696 Add reimport addresses task (#1891)

* Add update address service for lettings

* Only process collections 2023 onwards

* Update app/services/imports/lettings_logs_field_import_service.rb

Co-authored-by: natdeanlewissoftwire <94526761+natdeanlewissoftwire@users.noreply.github.com>

* Refactor postcode_known

* test

---------

Co-authored-by: natdeanlewissoftwire <94526761+natdeanlewissoftwire@users.noreply.github.com>

app/services/imports/lettings_logs_field_import_service.rb

@@ -12,6 +12,8 @@ module Imports
         import_from(folder, :update_offered)
       when "creation_method"
         import_from(folder, :update_creation_method)
+      when "address"
+        import_from(folder, :update_address)
       else
         raise "Updating #{field} is not supported by the field import service"
       end
@@ -132,5 +134,47 @@ module Imports
         @logger.warn("Could not find record matching legacy ID #{old_id}")
       end
     end
+
+    def update_address(xml_doc)
+      return if meta_field_value(xml_doc, "form-name").include?("Sales")
+
+      old_id = meta_field_value(xml_doc, "document-id")
+      record = LettingsLog.find_by(old_id:)
+      return @logger.info("lettings log #{record.id} is from previous collection year, skipping") if record.collection_start_year < 2023
+
+      if record.present?
+        if string_or_nil(xml_doc, "AddressLine1").present? && string_or_nil(xml_doc, "TownCity").present?
+          record.la = string_or_nil(xml_doc, "Q28ONS")
+          record.postcode_full = compose_postcode(xml_doc, "POSTCODE", "POSTCOD2")
+          record.postcode_known = postcode_known(record)
+          record.address_line1 = string_or_nil(xml_doc, "AddressLine1")
+          record.address_line2 = string_or_nil(xml_doc, "AddressLine2")
+          record.town_or_city = string_or_nil(xml_doc, "TownCity")
+          record.county = string_or_nil(xml_doc, "County")
+          record.uprn = nil
+          record.uprn_known = 0
+          record.uprn_confirmed = 0
+          record.values_updated_at = Time.zone.now
+          record.save!
+          @logger.info("lettings log #{record.id} address_line1 value has been set to #{record.address_line1}")
+          @logger.info("lettings log #{record.id} address_line2 value has been set to #{record.address_line2}")
+          @logger.info("lettings log #{record.id} town_or_city value has been set to #{record.town_or_city}")
+          @logger.info("lettings log #{record.id} county value has been set to #{record.county}")
+          @logger.info("lettings log #{record.id} postcode_full value has been set to #{record.postcode_full}")
+        else
+          @logger.info("lettings log #{record.id} is missing either or both of address_line1 and town or city, skipping")
+        end
+      else
+        @logger.warn("Could not find record matching legacy ID #{old_id}")
+      end
+    end
+
+    def postcode_known(record)
+      if record.postcode_full.nil?
+        record.la.nil? ? nil : 0 # Assumes we selected No in the form since the LA is present
+      else
+        1
+      end
+    end
   end
 end

lib/tasks/data_import_field.rake

@@ -7,7 +7,7 @@ namespace :core do
 
     # We only allow a reduced list of known fields to be updatable
     case field
-    when "tenancycode", "major_repairs", "lettings_allocation", "offered"
+    when "tenancycode", "major_repairs", "lettings_allocation", "offered", "address"
       s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
       archive_io = s3_service.get_file_io(path)
       archive_service = Storage::ArchiveService.new(archive_io)

spec/lib/tasks/data_import_field_spec.rb

@@ -92,6 +92,18 @@ describe "data_import_field imports" do
         end
       end
 
+      context "and we update the address fields" do
+        let(:field) { "address" }
+
+        it "updates the 2023 logs from the given XML file" do
+          expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name)
+          expect(storage_service).to receive(:get_file_io).with("spec/fixtures/imports/logs")
+          expect(Imports::LettingsLogsFieldImportService).to receive(:new).with(archive_service)
+          expect(import_service).to receive(:update_field).with(field, "logs")
+          task.invoke(field, fixture_path)
+        end
+      end
+
       it "raises an exception if no parameters are provided" do
         expect { task.invoke }.to raise_error(/Usage/)
       end

spec/services/imports/lettings_logs_field_import_service_spec.rb

@@ -323,4 +323,152 @@ RSpec.describe Imports::LettingsLogsFieldImportService do
       end
     end
   end
+
+  context "when updating address" do
+    let(:field) { "address" }
+
+    before do
+      WebMock.stub_request(:get, /api.postcodes.io\/postcodes\/B11BB/)
+      .to_return(status: 200, body: '{"status":200,"result":{"admin_district":"Westminster","codes":{"admin_district":"E08000035"}}}', headers: {})
+
+      stub_request(:get, "https://api.os.uk/search/places/v1/uprn?key=OS_DATA_KEY&uprn=123")
+        .to_return(status: 500, body: "{}", headers: {})
+
+      Timecop.freeze(2023, 5, 5)
+      Singleton.__init__(FormHandler)
+      Imports::LettingsLogsImportService.new(storage_service, logger).create_logs(fixture_directory)
+      lettings_log_file.rewind
+    end
+
+    after do
+      Timecop.unfreeze
+      Singleton.__init__(FormHandler)
+    end
+
+    context "when the lettings log has no address values" do
+      let(:lettings_log_id) { "00d2343e-d5fa-4c89-8400-ec3854b0f2b4" }
+      let(:lettings_log) { LettingsLog.find_by(old_id: lettings_log_id) }
+
+      before do
+        lettings_log.update!(uprn_known: nil,
+                             startdate: Time.zone.local(2023, 5, 5),
+                             uprn: nil,
+                             uprn_confirmed: nil,
+                             address_line1: nil,
+                             address_line2: nil,
+                             town_or_city: nil,
+                             county: nil,
+                             postcode_known: nil,
+                             postcode_full: nil,
+                             la: nil,
+                             is_la_inferred: nil)
+      end
+
+      context "and new address values include address" do
+        before do
+          lettings_log_xml.at_xpath("//xmlns:UPRN").content = "123456781234"
+          lettings_log_xml.at_xpath("//xmlns:AddressLine1").content = "address 1"
+          lettings_log_xml.at_xpath("//xmlns:AddressLine2").content = "address 2"
+          lettings_log_xml.at_xpath("//xmlns:TownCity").content = "towncity"
+          lettings_log_xml.at_xpath("//xmlns:County").content = "county"
+          lettings_log_xml.at_xpath("//xmlns:POSTCODE").content = "B1"
+          lettings_log_xml.at_xpath("//xmlns:POSTCOD2").content = "1BB"
+          lettings_log_xml.at_xpath("//xmlns:Q28ONS").content = nil
+        end
+
+        it "updates the lettings_log prioritising address values" do
+          expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} address_line1 value has been set to address 1/)
+          expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} address_line2 value has been set to address 2/)
+          expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} town_or_city value has been set to towncity/)
+          expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} county value has been set to county/)
+          expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} postcode_full value has been set to B1 1BB/)
+          import_service.send(:update_address, lettings_log_xml)
+          lettings_log.reload
+
+          expect(lettings_log.uprn_known).to eq(0)
+          expect(lettings_log.uprn).to eq(nil)
+          expect(lettings_log.uprn_confirmed).to eq(nil)
+          expect(lettings_log.address_line1).to eq("address 1")
+          expect(lettings_log.address_line2).to eq("address 2")
+          expect(lettings_log.town_or_city).to eq("towncity")
+          expect(lettings_log.county).to eq("county")
+          expect(lettings_log.postcode_known).to eq(1)
+          expect(lettings_log.postcode_full).to eq("B1 1BB")
+          expect(lettings_log.la).to eq("E08000035")
+          expect(lettings_log.is_la_inferred).to eq(true)
+        end
+      end
+
+      context "and new address values don't include address" do
+        it "skips the update" do
+          expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} is missing either or both of address_line1 and town or city, skipping/)
+          import_service.send(:update_address, lettings_log_xml)
+        end
+      end
+    end
+
+    context "when the lettings log has address values" do
+      let(:lettings_log_id) { "00d2343e-d5fa-4c89-8400-ec3854b0f2b4" }
+      let(:lettings_log) { LettingsLog.find_by(old_id: lettings_log_id) }
+
+      before do
+        lettings_log_xml.at_xpath("//xmlns:UPRN").content = "123456781234"
+        lettings_log_xml.at_xpath("//xmlns:AddressLine1").content = "address 1"
+        lettings_log_xml.at_xpath("//xmlns:AddressLine2").content = "address 2"
+        lettings_log_xml.at_xpath("//xmlns:TownCity").content = "towncity"
+        lettings_log_xml.at_xpath("//xmlns:County").content = "county"
+        lettings_log_xml.at_xpath("//xmlns:POSTCODE").content = "B1"
+        lettings_log_xml.at_xpath("//xmlns:POSTCOD2").content = "1BC"
+        lettings_log_xml.at_xpath("//xmlns:Q28ONS").content = nil
+        lettings_log.update!(uprn_known: 1,
+                             startdate: Time.zone.local(2023, 5, 5),
+                             uprn: "123",
+                             uprn_confirmed: 0,
+                             address_line1: "wrong address line1",
+                             address_line2: "wrong address 2",
+                             town_or_city: "wrong town",
+                             county: "wrong city",
+                             postcode_known: 1,
+                             postcode_full: "A11AA",
+                             la: "E06000064",
+                             is_la_inferred: true)
+      end
+
+      it "replaces the lettings_log address values" do
+        expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} address_line1 value has been set to address 1/)
+        expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} address_line2 value has been set to address 2/)
+        expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} town_or_city value has been set to towncity/)
+        expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} county value has been set to county/)
+        expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} postcode_full value has been set to B11BC/)
+        import_service.send(:update_address, lettings_log_xml)
+        lettings_log.reload
+
+        expect(lettings_log.uprn_known).to eq(0)
+        expect(lettings_log.uprn).to eq(nil)
+        expect(lettings_log.uprn_confirmed).to eq(nil)
+        expect(lettings_log.address_line1).to eq("address 1")
+        expect(lettings_log.address_line2).to eq("address 2")
+        expect(lettings_log.town_or_city).to eq("towncity")
+        expect(lettings_log.county).to eq("county")
+        expect(lettings_log.postcode_known).to eq(1)
+        expect(lettings_log.postcode_full).to eq("B11BC")
+        expect(lettings_log.la).to eq(nil)
+        expect(lettings_log.is_la_inferred).to eq(false)
+      end
+    end
+
+    context "when the lettings log is from before collection 23/24" do
+      let(:lettings_log_id) { "00d2343e-d5fa-4c89-8400-ec3854b0f2b4" }
+      let(:lettings_log) { LettingsLog.find_by(old_id: lettings_log_id) }
+
+      before do
+        lettings_log.update!(startdate: Time.zone.local(2022, 5, 5))
+      end
+
+      it "skips the update" do
+        expect(logger).to receive(:info).with(/lettings log #{lettings_log.id} is from previous collection year, skipping/)
+        import_service.send(:update_address, lettings_log_xml)
+      end
+    end
+  end
 end