Add 23/24 fields to import mappings (#1560)

* Import location fields for 2023/24 logs

* Map 23/24 location fields for sales logs

* Add remaining 23/24 import mappings

* Remove redundant mappings

* Add default mappings

* Remove year check

app/models/log.rb

@@ -48,7 +48,7 @@ class Log < ApplicationRecord
       service = UprnClient.new(uprn)
       service.call
 
-      return errors.add(:uprn, service.error) if service.error.present?
+      return errors.add(:uprn, :uprn_error, message: service.error) if service.error.present?
 
       presenter = UprnDataPresenter.new(service.result)
 

app/services/imports/lettings_logs_import_service.rb

@@ -202,6 +202,14 @@ module Imports
       attributes["first_time_property_let_as_social_housing"] = first_time_let(attributes["rsnvac"])
       attributes["declaration"] = declaration(xml_doc)
 
+      attributes["uprn"] = string_or_nil(xml_doc, "UPRN")
+      attributes["uprn_known"] = attributes["uprn"].present? ? 1 : 0
+      attributes["uprn_confirmed"] = attributes["uprn"].present? ? 1 : 0
+      attributes["address_line1"] = string_or_nil(xml_doc, "AddressLine1")
+      attributes["address_line2"] = string_or_nil(xml_doc, "AddressLine2")
+      attributes["town_or_city"] = string_or_nil(xml_doc, "TownCity")
+      attributes["county"] = string_or_nil(xml_doc, "County")
+
       set_partial_charges_to_zero(attributes)
 
       # Supported Housing fields
@@ -333,6 +341,13 @@ module Imports
         return save_lettings_log(attributes, previous_status)
       end
 
+      if lettings_log.errors.of_kind?(:uprn, :uprn_error)
+        @logger.warn("Log #{lettings_log.old_id}: Setting uprn_known to no with error: #{lettings_log.errors[:uprn].join(', ')}")
+        @logs_overridden << lettings_log.old_id
+        attributes["uprn_known"] = 0
+        return save_lettings_log(attributes, previous_status)
+      end
+
       @logger.error("Log #{lettings_log.old_id}: Failed to import")
       lettings_log.errors.each do |error|
         @logger.error("Validation error: Field #{error.attribute}:")
@@ -360,7 +375,7 @@ module Imports
     end
 
     def fields_not_present_in_softwire_data
-      %w[majorrepairs illness_type_0 tshortfall_known pregnancy_value_check retirement_value_check rent_value_check net_income_value_check major_repairs_date_value_check void_date_value_check carehome_charges_value_check housingneeds_type housingneeds_other created_by]
+      %w[majorrepairs illness_type_0 tshortfall_known pregnancy_value_check retirement_value_check rent_value_check net_income_value_check major_repairs_date_value_check void_date_value_check carehome_charges_value_check housingneeds_type housingneeds_other created_by uprn_known uprn_confirmed]
     end
 
     def check_status_completed(lettings_log, previous_status)

app/services/imports/sales_logs_import_service.rb

@@ -126,10 +126,6 @@ module Imports
       attributes["postcode_full"] = parse_postcode(string_or_nil(xml_doc, "Q14Postcode"))
       attributes["pcodenk"] = 0 if attributes["postcode_full"].present? # known if given
       attributes["soctenant"] = 0 if attributes["ownershipsch"] == 1
-      attributes["ethnic_group2"] = nil # 23/24 variable
-      attributes["ethnicbuy2"] = nil # 23/24 variable
-      attributes["prevshared"] = nil # 23/24 variable
-      attributes["staircasesale"] = nil # 23/24 variable
 
       attributes["previous_la_known"] = 1 if attributes["prevloc"].present?
       if attributes["la"].present?
@@ -158,6 +154,27 @@ module Imports
       attributes["buyer_livein_value_check"] = 0
       attributes["percentage_discount_value_check"] = 0
 
+      # 2023/34 attributes
+      attributes["uprn"] = string_or_nil(xml_doc, "UPRN")
+      attributes["uprn_known"] = attributes["uprn"].present? ? 1 : 0
+      attributes["uprn_confirmed"] = attributes["uprn"].present? ? 1 : 0
+      attributes["address_line1"] = string_or_nil(xml_doc, "AddressLine1")
+      attributes["address_line2"] = string_or_nil(xml_doc, "AddressLine2")
+      attributes["town_or_city"] = string_or_nil(xml_doc, "TownCity")
+      attributes["county"] = string_or_nil(xml_doc, "County")
+
+      attributes["proplen_asked"] = 0 if attributes["proplen"]&.positive?
+      attributes["proplen_asked"] = 1 if attributes["proplen"]&.zero?
+
+      attributes["prevshared"] = unsafe_string_as_integer(xml_doc, "PREVSHARED")
+      attributes["ethnicbuy2"] = unsafe_string_as_integer(xml_doc, "P2Eth")
+      attributes["ethnic_group2"] = ethnic_group(attributes["ethnicbuy2"])
+      attributes["nationalbuy2"] = unsafe_string_as_integer(xml_doc, "P2Nat")
+      attributes["buy2living"] = unsafe_string_as_integer(xml_doc, "buy2livein")
+
+      attributes["staircasesale"] = unsafe_string_as_integer(xml_doc, "STAIRCASESALE")
+      attributes["prevtenbuy2"] = unsafe_string_as_integer(xml_doc, "PREVTENBUY2")
+
       # Sets the log creator
       owner_id = meta_field_value(xml_doc, "owner-user-id").strip
       if owner_id.present?
@@ -233,6 +250,11 @@ module Imports
         @logs_overridden << sales_log.old_id
         attributes.delete("mortgage")
         save_sales_log(attributes, previous_status)
+      elsif sales_log.errors.of_kind?(:uprn, :uprn_error)
+        @logger.warn("Log #{sales_log.old_id}: Setting uprn_known to no with error: #{sales_log.errors[:uprn].join(', ')}")
+        @logs_overridden << sales_log.old_id
+        attributes["uprn_known"] = 0
+        save_sales_log(attributes, previous_status)
       else
         @logger.error("Log #{sales_log.old_id}: Failed to import")
         sales_log.errors.each do |error|
@@ -282,7 +304,9 @@ module Imports
          student_not_child_value_check
          discounted_sale_value_check
          buyer_livein_value_check
-         percentage_discount_value_check]
+         percentage_discount_value_check
+         uprn_known
+         uprn_confirmed]
     end
 
     def check_status_completed(sales_log, previous_status)
@@ -461,12 +485,14 @@ module Imports
         safe_string_as_decimal(xml_doc, "Q29MonthlyCharges")
       when 2
         safe_string_as_decimal(xml_doc, "Q37MonthlyCharges")
+      when 3
+        safe_string_as_decimal(xml_doc, "Q44MonthlyCharges")
       end
     end
 
     def ownership_from_type(attributes)
       case attributes["type"]
-      when 2, 24, 18, 16, 28, 31, 30
+      when 2, 24, 18, 16, 28, 31, 30, 32
         1 # shared ownership
       when 8, 14, 27, 9, 29, 21, 22
         2 # discounted ownership
@@ -562,6 +588,9 @@ module Imports
         attributes["relat2"] ||= "R"
         attributes["inc2mort"] ||= 3
         attributes["buy2livein"] ||= 1 unless attributes["ownershipsch"] == 3
+        attributes["ethnic_group2"] ||= 17
+        attributes["ethnicbuy2"] ||= 17
+        attributes["nationalbuy2"] ||= 13
       end
 
       # other household members characteristics

spec/fixtures/imports/logs/00d2343e-d5fa-4c89-8400-ec3854b0f2b4.xml

@@ -131,6 +131,11 @@
     <_2cYears/>
   </Group>
   <Group>
+    <UPRN>12345678</UPRN>
+    <AddressLine1/>
+    <AddressLine2/>
+    <TownCity/>
+    <County/>
     <Q28pc override-field="">SR8 3HF</Q28pc>
     <Q28Auth>Durham</Q28Auth>
     <Q28ONS>E06000047</Q28ONS>

spec/fixtures/imports/sales_logs/outright_sale_sales_log.xml

@@ -32,6 +32,11 @@
     <Q11Bedrooms override-field="">1</Q11Bedrooms>
     <Q12PropertyType>1 Flat or maisonette</Q12PropertyType>
     <Q13BuildingType>1 Purpose built</Q13BuildingType>
+    <UPRN/>
+    <AddressLine1/>
+    <AddressLine2/>
+    <TownCity/>
+    <County/>
     <Q14Postcode override-field="">SW1A 1AA</Q14Postcode>
     <Q14PropertyLocation>Westminster</Q14PropertyLocation>
     <Q14ONSLACode>E09000033</Q14ONSLACode>
@@ -275,6 +280,7 @@
     <Q41b/>
     <Q42Borrowing/>
     <Q43CashDeposit override-field="">300000</Q43CashDeposit>
+    <Q44MonthlyCharges/>
   </Group>
   <Group>
     <HHMEMB>1</HHMEMB>

spec/fixtures/imports/sales_logs/shared_ownership_sales_log.xml

@@ -24,6 +24,7 @@
     <Q38OtherSale/>
     <company>2 No</company>
     <LiveInBuyer>1 Yes</LiveInBuyer>
+    <buy2livein/>
     <joint>2 No</joint>
     <JointMore/>
     <PartAPurchaser>2 Yes</PartAPurchaser>
@@ -32,6 +33,12 @@
     <Q11Bedrooms override-field="">2</Q11Bedrooms>
     <Q12PropertyType>1 Flat or maisonette</Q12PropertyType>
     <Q13BuildingType>1 Purpose built</Q13BuildingType>
+    <UPRN/>
+    <AddressLine1/>
+    <AddressLine2/>
+    <TownCity/>
+    <County/>
+    <PREVSHARED/>
     <Q14Postcode override-field="">SW1A 1AA</Q14Postcode>
     <Q14PropertyLocation>Westminster</Q14PropertyLocation>
     <Q14ONSLACode>E09000033</Q14ONSLACode>
@@ -47,6 +54,8 @@
     <P2Sex override-field=""/>
     <P2Rel/>
     <P2Eco/>
+    <P2Eth/>
+    <P2Nat/>
     <P3Age/>
     <P3Sex override-field=""/>
     <P3Rel/>

spec/services/imports/lettings_logs_import_service_spec.rb

@@ -1151,5 +1151,96 @@ RSpec.describe Imports::LettingsLogsImportService do
         expect(lettings_log.hbrentshortfall).to be_nil
       end
     end
+
+    context "when setting location fields for 23/24 logs" do
+      let(:lettings_log_id) { "00d2343e-d5fa-4c89-8400-ec3854b0f2b4" }
+      let(:lettings_log_file) { open_file(fixture_directory, lettings_log_id) }
+      let(:lettings_log_xml) { Nokogiri::XML(lettings_log_file) }
+
+      around do |example|
+        Timecop.freeze(Time.zone.local(2023, 4, 1)) do
+          Singleton.__init__(FormHandler)
+          example.run
+        end
+        Timecop.return
+        Singleton.__init__(FormHandler)
+      end
+
+      before do
+        lettings_log_xml.at_xpath("//xmlns:DAY").content = "10"
+        lettings_log_xml.at_xpath("//xmlns:MONTH").content = "4"
+        lettings_log_xml.at_xpath("//xmlns:YEAR").content = "2023"
+        lettings_log_xml.at_xpath("//xmlns:UPRN").content = "123456781234"
+        lettings_log_xml.at_xpath("//xmlns:AddressLine1").content = "address 1"
+        lettings_log_xml.at_xpath("//xmlns:AddressLine2").content = "address 2"
+        lettings_log_xml.at_xpath("//xmlns:TownCity").content = "towncity"
+        lettings_log_xml.at_xpath("//xmlns:County").content = "county"
+        lettings_log_xml.at_xpath("//xmlns:POSTCODE").content = "A1"
+        lettings_log_xml.at_xpath("//xmlns:POSTCOD2").content = "1AA"
+
+        body = {
+          results: [
+            {
+              DPA: {
+                "POSTCODE": "LS16 6FT",
+                "POST_TOWN": "Westminster",
+                "PO_BOX_NUMBER": "321",
+                "DOUBLE_DEPENDENT_LOCALITY": "Double Dependent Locality",
+              },
+            },
+          ],
+        }.to_json
+
+        stub_request(:get, "https://api.os.uk/search/places/v1/uprn?key=OS_DATA_KEY&uprn=123456781234")
+          .to_return(status: 200, body:, headers: {})
+        stub_request(:get, "https://api.os.uk/search/places/v1/uprn?key=OS_DATA_KEY&uprn=123")
+          .to_return(status: 500, body: "{}", headers: {})
+
+        allow(logger).to receive(:warn).and_return(nil)
+      end
+
+      it "correctly sets address if uprn is not given" do
+        lettings_log_xml.at_xpath("//xmlns:UPRN").content = ""
+        lettings_log_service.send(:create_log, lettings_log_xml)
+
+        lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
+        expect(lettings_log&.uprn_known).to eq(0) # no
+        expect(lettings_log&.uprn).to be_nil
+        expect(lettings_log&.address_line1).to eq("address 1")
+        expect(lettings_log&.address_line2).to eq("address 2")
+        expect(lettings_log&.town_or_city).to eq("towncity")
+        expect(lettings_log&.county).to eq("county")
+        expect(lettings_log&.postcode_full).to eq("A1 1AA")
+      end
+
+      it "correctly sets address and uprn if uprn is given" do
+        lettings_log_service.send(:create_log, lettings_log_xml)
+
+        lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
+        expect(lettings_log&.uprn_known).to eq(1)
+        expect(lettings_log&.uprn).to eq("123456781234")
+        expect(lettings_log&.address_line1).to eq("321")
+        expect(lettings_log&.address_line2).to eq("Double Dependent Locality")
+        expect(lettings_log&.town_or_city).to eq("Westminster")
+        expect(lettings_log&.postcode_full).to eq("LS16 6FT")
+        expect(lettings_log&.la).to eq("E08000035")
+      end
+
+      it "correctly sets address and uprn if uprn is given but not recognised" do
+        lettings_log_xml.at_xpath("//xmlns:UPRN").content = "123"
+
+        lettings_log_service.send(:create_log, lettings_log_xml)
+
+        lettings_log = LettingsLog.find_by(old_id: lettings_log_id)
+        expect(lettings_log&.uprn_known).to eq(0)
+        expect(lettings_log&.uprn).to be_nil
+        expect(lettings_log&.address_line1).to eq("address 1")
+        expect(lettings_log&.address_line2).to eq("address 2")
+        expect(lettings_log&.town_or_city).to eq("towncity")
+        expect(lettings_log&.county).to eq("county")
+        expect(lettings_log&.postcode_full).to eq("A1 1AA")
+        expect(lettings_log&.la).to eq("E06000047")
+      end
+    end
   end
 end

spec/services/imports/sales_logs_import_service_spec.rb

@@ -220,6 +220,87 @@ RSpec.describe Imports::SalesLogsImportService do
       end
     end
 
+    context "with 23/24 logs" do
+      around do |example|
+        Timecop.freeze(Time.zone.local(2023, 4, 1)) do
+          Singleton.__init__(FormHandler)
+          example.run
+        end
+        Timecop.return
+        Singleton.__init__(FormHandler)
+      end
+
+      before do
+        sales_log_xml.at_xpath("//xmlns:DAY").content = "10"
+        sales_log_xml.at_xpath("//xmlns:MONTH").content = "4"
+        sales_log_xml.at_xpath("//xmlns:YEAR").content = "2023"
+        sales_log_xml.at_xpath("//xmlns:UPRN").content = ""
+        sales_log_xml.at_xpath("//xmlns:AddressLine1").content = "address 1"
+        sales_log_xml.at_xpath("//xmlns:AddressLine2").content = "address 2"
+        sales_log_xml.at_xpath("//xmlns:TownCity").content = "towncity"
+        sales_log_xml.at_xpath("//xmlns:County").content = "county"
+        sales_log_xml.at_xpath("//xmlns:Q14Postcode").content = "A1 1AA"
+      end
+
+      context "with outright sale type" do
+        let(:sales_log_id) { "outright_sale_sales_log" }
+
+        before do
+          sales_log_xml.at_xpath("//xmlns:Q44MonthlyCharges").content = "40"
+        end
+
+        it "successfully creates a completed outright sale log" do
+          expect(logger).not_to receive(:error)
+          expect(logger).not_to receive(:warn)
+          expect(logger).not_to receive(:info)
+          expect { sales_log_service.send(:create_log, sales_log_xml) }
+            .to change(SalesLog, :count).by(1)
+        end
+      end
+
+      context "with shared sale type" do
+        let(:sales_log_id) { "shared_ownership_sales_log" }
+
+        before do
+          sales_log_xml.at_xpath("//xmlns:joint").content = "1 Yes"
+          sales_log_xml.at_xpath("//xmlns:JointMore").content = "2 No"
+          sales_log_xml.at_xpath("//xmlns:PREVSHARED").content = "1"
+          sales_log_xml.at_xpath("//xmlns:Q16aProplen2").content = "0"
+          sales_log_xml.at_xpath("//xmlns:Q20Bedrooms").content = "2"
+          sales_log_xml.at_xpath("//xmlns:P2Eth").content = "2 White: Irish"
+          sales_log_xml.at_xpath("//xmlns:P2Nat").content = "18 United Kingdom"
+          sales_log_xml.at_xpath("//xmlns:buy2livein").content = "1"
+        end
+
+        it "successfully creates a completed shared sale log" do
+          expect(logger).not_to receive(:error)
+          expect(logger).not_to receive(:warn)
+          expect(logger).not_to receive(:info)
+          expect { sales_log_service.send(:create_log, sales_log_xml) }
+            .to change(SalesLog, :count).by(1)
+          sales_log = SalesLog.find_by(old_id: sales_log_id)
+          expect(sales_log.proplen_asked).to eq(1)
+        end
+      end
+
+      context "with discounted sale type" do
+        let(:sales_log_id) { "shared_ownership_sales_log" }
+
+        before do
+          sales_log_xml.at_xpath("//xmlns:PREVSHARED").content = "1"
+          sales_log_xml.at_xpath("//xmlns:Q20Bedrooms").content = "2"
+        end
+
+        it "successfully creates a completed shared sale log" do
+          expect(logger).not_to receive(:error)
+          expect(logger).not_to receive(:warn)
+          expect(logger).not_to receive(:info)
+          expect { sales_log_service.send(:create_log, sales_log_xml) }
+            .to change(SalesLog, :count).by(1)
+        end
+      end
+    end
+
     context "and the mortgage soft validation is triggered (mortgage_value_check)" do
       let(:sales_log_id) { "discounted_ownership_sales_log" }
 
@@ -1336,6 +1417,97 @@ RSpec.describe Imports::SalesLogsImportService do
         end
       end
 
+      context "when setting location fields for 23/24 logs" do
+        let(:sales_log_id) { "outright_sale_sales_log" }
+
+        around do |example|
+          Timecop.freeze(Time.zone.local(2023, 4, 1)) do
+            Singleton.__init__(FormHandler)
+            example.run
+          end
+          Timecop.return
+          Singleton.__init__(FormHandler)
+        end
+
+        before do
+          sales_log_xml.at_xpath("//xmlns:DAY").content = "10"
+          sales_log_xml.at_xpath("//xmlns:MONTH").content = "4"
+          sales_log_xml.at_xpath("//xmlns:YEAR").content = "2023"
+          sales_log_xml.at_xpath("//xmlns:UPRN").content = "123456781234"
+          sales_log_xml.at_xpath("//xmlns:AddressLine1").content = "address 1"
+          sales_log_xml.at_xpath("//xmlns:AddressLine2").content = "address 2"
+          sales_log_xml.at_xpath("//xmlns:TownCity").content = "towncity"
+          sales_log_xml.at_xpath("//xmlns:County").content = "county"
+          sales_log_xml.at_xpath("//xmlns:Q14Postcode").content = "A1 1AA"
+
+          body = {
+            results: [
+              {
+                DPA: {
+                  "POSTCODE": "LS16 6FT",
+                  "POST_TOWN": "Westminster",
+                  "PO_BOX_NUMBER": "321",
+                  "DOUBLE_DEPENDENT_LOCALITY": "Double Dependent Locality",
+                },
+              },
+            ],
+          }.to_json
+
+          stub_request(:get, "https://api.os.uk/search/places/v1/uprn?key=OS_DATA_KEY&uprn=123456781234")
+            .to_return(status: 200, body:, headers: {})
+          stub_request(:get, "https://api.os.uk/search/places/v1/uprn?key=OS_DATA_KEY&uprn=123")
+            .to_return(status: 500, body: "{}", headers: {})
+
+          WebMock.stub_request(:get, /api.postcodes.io\/postcodes\/LS166FT/)
+            .to_return(status: 200, body: '{"status":200,"result":{"admin_district":"Westminster","codes":{"admin_district":"E08000035"}}}', headers: {})
+
+          allow(logger).to receive(:warn).and_return(nil)
+        end
+
+        it "correctly sets address if uprn is not given" do
+          sales_log_xml.at_xpath("//xmlns:UPRN").content = ""
+          sales_log_service.send(:create_log, sales_log_xml)
+
+          sales_log = SalesLog.find_by(old_id: sales_log_id)
+          expect(sales_log&.uprn_known).to eq(0) # no
+          expect(sales_log&.uprn).to be_nil
+          expect(sales_log&.address_line1).to eq("address 1")
+          expect(sales_log&.address_line2).to eq("address 2")
+          expect(sales_log&.town_or_city).to eq("towncity")
+          expect(sales_log&.county).to eq("county")
+          expect(sales_log&.postcode_full).to eq("A1 1AA")
+        end
+
+        it "correctly sets address and uprn if uprn is given" do
+          sales_log_service.send(:create_log, sales_log_xml)
+
+          sales_log = SalesLog.find_by(old_id: sales_log_id)
+          expect(sales_log&.uprn_known).to eq(1)
+          expect(sales_log&.uprn).to eq("123456781234")
+          expect(sales_log&.address_line1).to eq("321")
+          expect(sales_log&.address_line2).to eq("Double Dependent Locality")
+          expect(sales_log&.town_or_city).to eq("Westminster")
+          expect(sales_log&.postcode_full).to eq("LS16 6FT")
+          expect(sales_log&.la).to eq("E08000035")
+        end
+
+        it "correctly sets address and uprn if uprn is given but not recognised" do
+          sales_log_xml.at_xpath("//xmlns:UPRN").content = "123"
+
+          sales_log_service.send(:create_log, sales_log_xml)
+
+          sales_log = SalesLog.find_by(old_id: sales_log_id)
+          expect(sales_log&.uprn_known).to eq(0)
+          expect(sales_log&.uprn).to be_nil
+          expect(sales_log&.address_line1).to eq("address 1")
+          expect(sales_log&.address_line2).to eq("address 2")
+          expect(sales_log&.town_or_city).to eq("towncity")
+          expect(sales_log&.county).to eq("county")
+          expect(sales_log&.postcode_full).to eq("A1 1AA")
+          expect(sales_log&.la).to eq("E09000033")
+        end
+      end
+
       context "when setting default buyer 1 previous tenancy" do
         let(:sales_log_id) { "outright_sale_sales_log" }