From b3cf6724f3f6ca517efee03542b673cc46b1e48f Mon Sep 17 00:00:00 2001 From: kosiakkatrina <54268893+kosiakkatrina@users.noreply.github.com> Date: Wed, 19 Apr 2023 10:36:23 +0100 Subject: [PATCH] Add 23/24 fields to import mappings (#1560) * Import location fields for 2023/24 logs * Map 23/24 location fields for sales logs * Add remaining 23/24 import mappings * Remove redundant mappings * Add default mappings * Remove year check --- app/models/log.rb | 2 +- .../imports/lettings_logs_import_service.rb | 17 +- .../imports/sales_logs_import_service.rb | 41 ++++- .../00d2343e-d5fa-4c89-8400-ec3854b0f2b4.xml | 5 + .../sales_logs/outright_sale_sales_log.xml | 6 + .../sales_logs/shared_ownership_sales_log.xml | 9 + .../lettings_logs_import_service_spec.rb | 91 +++++++++ .../imports/sales_logs_import_service_spec.rb | 172 ++++++++++++++++++ 8 files changed, 335 insertions(+), 8 deletions(-) diff --git a/app/models/log.rb b/app/models/log.rb index fc125fd46..4ee748eda 100644 --- a/app/models/log.rb +++ b/app/models/log.rb @@ -48,7 +48,7 @@ class Log < ApplicationRecord service = UprnClient.new(uprn) service.call - return errors.add(:uprn, service.error) if service.error.present? + return errors.add(:uprn, :uprn_error, message: service.error) if service.error.present? presenter = UprnDataPresenter.new(service.result) diff --git a/app/services/imports/lettings_logs_import_service.rb b/app/services/imports/lettings_logs_import_service.rb index 01c71faef..b5f2d02b9 100644 --- a/app/services/imports/lettings_logs_import_service.rb +++ b/app/services/imports/lettings_logs_import_service.rb @@ -202,6 +202,14 @@ module Imports attributes["first_time_property_let_as_social_housing"] = first_time_let(attributes["rsnvac"]) attributes["declaration"] = declaration(xml_doc) + attributes["uprn"] = string_or_nil(xml_doc, "UPRN") + attributes["uprn_known"] = attributes["uprn"].present? ? 1 : 0 + attributes["uprn_confirmed"] = attributes["uprn"].present? ? 1 : 0 + attributes["address_line1"] = string_or_nil(xml_doc, "AddressLine1") + attributes["address_line2"] = string_or_nil(xml_doc, "AddressLine2") + attributes["town_or_city"] = string_or_nil(xml_doc, "TownCity") + attributes["county"] = string_or_nil(xml_doc, "County") + set_partial_charges_to_zero(attributes) # Supported Housing fields @@ -333,6 +341,13 @@ module Imports return save_lettings_log(attributes, previous_status) end + if lettings_log.errors.of_kind?(:uprn, :uprn_error) + @logger.warn("Log #{lettings_log.old_id}: Setting uprn_known to no with error: #{lettings_log.errors[:uprn].join(', ')}") + @logs_overridden << lettings_log.old_id + attributes["uprn_known"] = 0 + return save_lettings_log(attributes, previous_status) + end + @logger.error("Log #{lettings_log.old_id}: Failed to import") lettings_log.errors.each do |error| @logger.error("Validation error: Field #{error.attribute}:") @@ -360,7 +375,7 @@ module Imports end def fields_not_present_in_softwire_data - %w[majorrepairs illness_type_0 tshortfall_known pregnancy_value_check retirement_value_check rent_value_check net_income_value_check major_repairs_date_value_check void_date_value_check carehome_charges_value_check housingneeds_type housingneeds_other created_by] + %w[majorrepairs illness_type_0 tshortfall_known pregnancy_value_check retirement_value_check rent_value_check net_income_value_check major_repairs_date_value_check void_date_value_check carehome_charges_value_check housingneeds_type housingneeds_other created_by uprn_known uprn_confirmed] end def check_status_completed(lettings_log, previous_status) diff --git a/app/services/imports/sales_logs_import_service.rb b/app/services/imports/sales_logs_import_service.rb index 2d6551189..0f97df89a 100644 --- a/app/services/imports/sales_logs_import_service.rb +++ b/app/services/imports/sales_logs_import_service.rb @@ -126,10 +126,6 @@ module Imports attributes["postcode_full"] = parse_postcode(string_or_nil(xml_doc, "Q14Postcode")) attributes["pcodenk"] = 0 if attributes["postcode_full"].present? # known if given attributes["soctenant"] = 0 if attributes["ownershipsch"] == 1 - attributes["ethnic_group2"] = nil # 23/24 variable - attributes["ethnicbuy2"] = nil # 23/24 variable - attributes["prevshared"] = nil # 23/24 variable - attributes["staircasesale"] = nil # 23/24 variable attributes["previous_la_known"] = 1 if attributes["prevloc"].present? if attributes["la"].present? @@ -158,6 +154,27 @@ module Imports attributes["buyer_livein_value_check"] = 0 attributes["percentage_discount_value_check"] = 0 + # 2023/34 attributes + attributes["uprn"] = string_or_nil(xml_doc, "UPRN") + attributes["uprn_known"] = attributes["uprn"].present? ? 1 : 0 + attributes["uprn_confirmed"] = attributes["uprn"].present? ? 1 : 0 + attributes["address_line1"] = string_or_nil(xml_doc, "AddressLine1") + attributes["address_line2"] = string_or_nil(xml_doc, "AddressLine2") + attributes["town_or_city"] = string_or_nil(xml_doc, "TownCity") + attributes["county"] = string_or_nil(xml_doc, "County") + + attributes["proplen_asked"] = 0 if attributes["proplen"]&.positive? + attributes["proplen_asked"] = 1 if attributes["proplen"]&.zero? + + attributes["prevshared"] = unsafe_string_as_integer(xml_doc, "PREVSHARED") + attributes["ethnicbuy2"] = unsafe_string_as_integer(xml_doc, "P2Eth") + attributes["ethnic_group2"] = ethnic_group(attributes["ethnicbuy2"]) + attributes["nationalbuy2"] = unsafe_string_as_integer(xml_doc, "P2Nat") + attributes["buy2living"] = unsafe_string_as_integer(xml_doc, "buy2livein") + + attributes["staircasesale"] = unsafe_string_as_integer(xml_doc, "STAIRCASESALE") + attributes["prevtenbuy2"] = unsafe_string_as_integer(xml_doc, "PREVTENBUY2") + # Sets the log creator owner_id = meta_field_value(xml_doc, "owner-user-id").strip if owner_id.present? @@ -233,6 +250,11 @@ module Imports @logs_overridden << sales_log.old_id attributes.delete("mortgage") save_sales_log(attributes, previous_status) + elsif sales_log.errors.of_kind?(:uprn, :uprn_error) + @logger.warn("Log #{sales_log.old_id}: Setting uprn_known to no with error: #{sales_log.errors[:uprn].join(', ')}") + @logs_overridden << sales_log.old_id + attributes["uprn_known"] = 0 + save_sales_log(attributes, previous_status) else @logger.error("Log #{sales_log.old_id}: Failed to import") sales_log.errors.each do |error| @@ -282,7 +304,9 @@ module Imports student_not_child_value_check discounted_sale_value_check buyer_livein_value_check - percentage_discount_value_check] + percentage_discount_value_check + uprn_known + uprn_confirmed] end def check_status_completed(sales_log, previous_status) @@ -461,12 +485,14 @@ module Imports safe_string_as_decimal(xml_doc, "Q29MonthlyCharges") when 2 safe_string_as_decimal(xml_doc, "Q37MonthlyCharges") + when 3 + safe_string_as_decimal(xml_doc, "Q44MonthlyCharges") end end def ownership_from_type(attributes) case attributes["type"] - when 2, 24, 18, 16, 28, 31, 30 + when 2, 24, 18, 16, 28, 31, 30, 32 1 # shared ownership when 8, 14, 27, 9, 29, 21, 22 2 # discounted ownership @@ -562,6 +588,9 @@ module Imports attributes["relat2"] ||= "R" attributes["inc2mort"] ||= 3 attributes["buy2livein"] ||= 1 unless attributes["ownershipsch"] == 3 + attributes["ethnic_group2"] ||= 17 + attributes["ethnicbuy2"] ||= 17 + attributes["nationalbuy2"] ||= 13 end # other household members characteristics diff --git a/spec/fixtures/imports/logs/00d2343e-d5fa-4c89-8400-ec3854b0f2b4.xml b/spec/fixtures/imports/logs/00d2343e-d5fa-4c89-8400-ec3854b0f2b4.xml index 8bc6a935c..cd0e03d74 100644 --- a/spec/fixtures/imports/logs/00d2343e-d5fa-4c89-8400-ec3854b0f2b4.xml +++ b/spec/fixtures/imports/logs/00d2343e-d5fa-4c89-8400-ec3854b0f2b4.xml @@ -131,6 +131,11 @@ <_2cYears/> + 12345678 + + + + SR8 3HF Durham E06000047 diff --git a/spec/fixtures/imports/sales_logs/outright_sale_sales_log.xml b/spec/fixtures/imports/sales_logs/outright_sale_sales_log.xml index 4bbdc9981..ce17ff3fc 100644 --- a/spec/fixtures/imports/sales_logs/outright_sale_sales_log.xml +++ b/spec/fixtures/imports/sales_logs/outright_sale_sales_log.xml @@ -32,6 +32,11 @@ 1 1 Flat or maisonette 1 Purpose built + + + + + SW1A 1AA Westminster E09000033 @@ -275,6 +280,7 @@ 300000 + 1 diff --git a/spec/fixtures/imports/sales_logs/shared_ownership_sales_log.xml b/spec/fixtures/imports/sales_logs/shared_ownership_sales_log.xml index 6e0c11174..b69cb22c2 100644 --- a/spec/fixtures/imports/sales_logs/shared_ownership_sales_log.xml +++ b/spec/fixtures/imports/sales_logs/shared_ownership_sales_log.xml @@ -24,6 +24,7 @@ 2 No 1 Yes + 2 No 2 Yes @@ -32,6 +33,12 @@ 2 1 Flat or maisonette 1 Purpose built + + + + + + SW1A 1AA Westminster E09000033 @@ -47,6 +54,8 @@ + + diff --git a/spec/services/imports/lettings_logs_import_service_spec.rb b/spec/services/imports/lettings_logs_import_service_spec.rb index 4621afb2f..4fbd485c0 100644 --- a/spec/services/imports/lettings_logs_import_service_spec.rb +++ b/spec/services/imports/lettings_logs_import_service_spec.rb @@ -1151,5 +1151,96 @@ RSpec.describe Imports::LettingsLogsImportService do expect(lettings_log.hbrentshortfall).to be_nil end end + + context "when setting location fields for 23/24 logs" do + let(:lettings_log_id) { "00d2343e-d5fa-4c89-8400-ec3854b0f2b4" } + let(:lettings_log_file) { open_file(fixture_directory, lettings_log_id) } + let(:lettings_log_xml) { Nokogiri::XML(lettings_log_file) } + + around do |example| + Timecop.freeze(Time.zone.local(2023, 4, 1)) do + Singleton.__init__(FormHandler) + example.run + end + Timecop.return + Singleton.__init__(FormHandler) + end + + before do + lettings_log_xml.at_xpath("//xmlns:DAY").content = "10" + lettings_log_xml.at_xpath("//xmlns:MONTH").content = "4" + lettings_log_xml.at_xpath("//xmlns:YEAR").content = "2023" + lettings_log_xml.at_xpath("//xmlns:UPRN").content = "123456781234" + lettings_log_xml.at_xpath("//xmlns:AddressLine1").content = "address 1" + lettings_log_xml.at_xpath("//xmlns:AddressLine2").content = "address 2" + lettings_log_xml.at_xpath("//xmlns:TownCity").content = "towncity" + lettings_log_xml.at_xpath("//xmlns:County").content = "county" + lettings_log_xml.at_xpath("//xmlns:POSTCODE").content = "A1" + lettings_log_xml.at_xpath("//xmlns:POSTCOD2").content = "1AA" + + body = { + results: [ + { + DPA: { + "POSTCODE": "LS16 6FT", + "POST_TOWN": "Westminster", + "PO_BOX_NUMBER": "321", + "DOUBLE_DEPENDENT_LOCALITY": "Double Dependent Locality", + }, + }, + ], + }.to_json + + stub_request(:get, "https://api.os.uk/search/places/v1/uprn?key=OS_DATA_KEY&uprn=123456781234") + .to_return(status: 200, body:, headers: {}) + stub_request(:get, "https://api.os.uk/search/places/v1/uprn?key=OS_DATA_KEY&uprn=123") + .to_return(status: 500, body: "{}", headers: {}) + + allow(logger).to receive(:warn).and_return(nil) + end + + it "correctly sets address if uprn is not given" do + lettings_log_xml.at_xpath("//xmlns:UPRN").content = "" + lettings_log_service.send(:create_log, lettings_log_xml) + + lettings_log = LettingsLog.find_by(old_id: lettings_log_id) + expect(lettings_log&.uprn_known).to eq(0) # no + expect(lettings_log&.uprn).to be_nil + expect(lettings_log&.address_line1).to eq("address 1") + expect(lettings_log&.address_line2).to eq("address 2") + expect(lettings_log&.town_or_city).to eq("towncity") + expect(lettings_log&.county).to eq("county") + expect(lettings_log&.postcode_full).to eq("A1 1AA") + end + + it "correctly sets address and uprn if uprn is given" do + lettings_log_service.send(:create_log, lettings_log_xml) + + lettings_log = LettingsLog.find_by(old_id: lettings_log_id) + expect(lettings_log&.uprn_known).to eq(1) + expect(lettings_log&.uprn).to eq("123456781234") + expect(lettings_log&.address_line1).to eq("321") + expect(lettings_log&.address_line2).to eq("Double Dependent Locality") + expect(lettings_log&.town_or_city).to eq("Westminster") + expect(lettings_log&.postcode_full).to eq("LS16 6FT") + expect(lettings_log&.la).to eq("E08000035") + end + + it "correctly sets address and uprn if uprn is given but not recognised" do + lettings_log_xml.at_xpath("//xmlns:UPRN").content = "123" + + lettings_log_service.send(:create_log, lettings_log_xml) + + lettings_log = LettingsLog.find_by(old_id: lettings_log_id) + expect(lettings_log&.uprn_known).to eq(0) + expect(lettings_log&.uprn).to be_nil + expect(lettings_log&.address_line1).to eq("address 1") + expect(lettings_log&.address_line2).to eq("address 2") + expect(lettings_log&.town_or_city).to eq("towncity") + expect(lettings_log&.county).to eq("county") + expect(lettings_log&.postcode_full).to eq("A1 1AA") + expect(lettings_log&.la).to eq("E06000047") + end + end end end diff --git a/spec/services/imports/sales_logs_import_service_spec.rb b/spec/services/imports/sales_logs_import_service_spec.rb index 5862efa23..2e7eea168 100644 --- a/spec/services/imports/sales_logs_import_service_spec.rb +++ b/spec/services/imports/sales_logs_import_service_spec.rb @@ -220,6 +220,87 @@ RSpec.describe Imports::SalesLogsImportService do end end + context "with 23/24 logs" do + around do |example| + Timecop.freeze(Time.zone.local(2023, 4, 1)) do + Singleton.__init__(FormHandler) + example.run + end + Timecop.return + Singleton.__init__(FormHandler) + end + + before do + sales_log_xml.at_xpath("//xmlns:DAY").content = "10" + sales_log_xml.at_xpath("//xmlns:MONTH").content = "4" + sales_log_xml.at_xpath("//xmlns:YEAR").content = "2023" + sales_log_xml.at_xpath("//xmlns:UPRN").content = "" + sales_log_xml.at_xpath("//xmlns:AddressLine1").content = "address 1" + sales_log_xml.at_xpath("//xmlns:AddressLine2").content = "address 2" + sales_log_xml.at_xpath("//xmlns:TownCity").content = "towncity" + sales_log_xml.at_xpath("//xmlns:County").content = "county" + sales_log_xml.at_xpath("//xmlns:Q14Postcode").content = "A1 1AA" + end + + context "with outright sale type" do + let(:sales_log_id) { "outright_sale_sales_log" } + + before do + sales_log_xml.at_xpath("//xmlns:Q44MonthlyCharges").content = "40" + end + + it "successfully creates a completed outright sale log" do + expect(logger).not_to receive(:error) + expect(logger).not_to receive(:warn) + expect(logger).not_to receive(:info) + expect { sales_log_service.send(:create_log, sales_log_xml) } + .to change(SalesLog, :count).by(1) + end + end + + context "with shared sale type" do + let(:sales_log_id) { "shared_ownership_sales_log" } + + before do + sales_log_xml.at_xpath("//xmlns:joint").content = "1 Yes" + sales_log_xml.at_xpath("//xmlns:JointMore").content = "2 No" + sales_log_xml.at_xpath("//xmlns:PREVSHARED").content = "1" + sales_log_xml.at_xpath("//xmlns:Q16aProplen2").content = "0" + sales_log_xml.at_xpath("//xmlns:Q20Bedrooms").content = "2" + sales_log_xml.at_xpath("//xmlns:P2Eth").content = "2 White: Irish" + sales_log_xml.at_xpath("//xmlns:P2Nat").content = "18 United Kingdom" + sales_log_xml.at_xpath("//xmlns:buy2livein").content = "1" + end + + it "successfully creates a completed shared sale log" do + expect(logger).not_to receive(:error) + expect(logger).not_to receive(:warn) + expect(logger).not_to receive(:info) + expect { sales_log_service.send(:create_log, sales_log_xml) } + .to change(SalesLog, :count).by(1) + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log.proplen_asked).to eq(1) + end + end + + context "with discounted sale type" do + let(:sales_log_id) { "shared_ownership_sales_log" } + + before do + sales_log_xml.at_xpath("//xmlns:PREVSHARED").content = "1" + sales_log_xml.at_xpath("//xmlns:Q20Bedrooms").content = "2" + end + + it "successfully creates a completed shared sale log" do + expect(logger).not_to receive(:error) + expect(logger).not_to receive(:warn) + expect(logger).not_to receive(:info) + expect { sales_log_service.send(:create_log, sales_log_xml) } + .to change(SalesLog, :count).by(1) + end + end + end + context "and the mortgage soft validation is triggered (mortgage_value_check)" do let(:sales_log_id) { "discounted_ownership_sales_log" } @@ -1336,6 +1417,97 @@ RSpec.describe Imports::SalesLogsImportService do end end + context "when setting location fields for 23/24 logs" do + let(:sales_log_id) { "outright_sale_sales_log" } + + around do |example| + Timecop.freeze(Time.zone.local(2023, 4, 1)) do + Singleton.__init__(FormHandler) + example.run + end + Timecop.return + Singleton.__init__(FormHandler) + end + + before do + sales_log_xml.at_xpath("//xmlns:DAY").content = "10" + sales_log_xml.at_xpath("//xmlns:MONTH").content = "4" + sales_log_xml.at_xpath("//xmlns:YEAR").content = "2023" + sales_log_xml.at_xpath("//xmlns:UPRN").content = "123456781234" + sales_log_xml.at_xpath("//xmlns:AddressLine1").content = "address 1" + sales_log_xml.at_xpath("//xmlns:AddressLine2").content = "address 2" + sales_log_xml.at_xpath("//xmlns:TownCity").content = "towncity" + sales_log_xml.at_xpath("//xmlns:County").content = "county" + sales_log_xml.at_xpath("//xmlns:Q14Postcode").content = "A1 1AA" + + body = { + results: [ + { + DPA: { + "POSTCODE": "LS16 6FT", + "POST_TOWN": "Westminster", + "PO_BOX_NUMBER": "321", + "DOUBLE_DEPENDENT_LOCALITY": "Double Dependent Locality", + }, + }, + ], + }.to_json + + stub_request(:get, "https://api.os.uk/search/places/v1/uprn?key=OS_DATA_KEY&uprn=123456781234") + .to_return(status: 200, body:, headers: {}) + stub_request(:get, "https://api.os.uk/search/places/v1/uprn?key=OS_DATA_KEY&uprn=123") + .to_return(status: 500, body: "{}", headers: {}) + + WebMock.stub_request(:get, /api.postcodes.io\/postcodes\/LS166FT/) + .to_return(status: 200, body: '{"status":200,"result":{"admin_district":"Westminster","codes":{"admin_district":"E08000035"}}}', headers: {}) + + allow(logger).to receive(:warn).and_return(nil) + end + + it "correctly sets address if uprn is not given" do + sales_log_xml.at_xpath("//xmlns:UPRN").content = "" + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.uprn_known).to eq(0) # no + expect(sales_log&.uprn).to be_nil + expect(sales_log&.address_line1).to eq("address 1") + expect(sales_log&.address_line2).to eq("address 2") + expect(sales_log&.town_or_city).to eq("towncity") + expect(sales_log&.county).to eq("county") + expect(sales_log&.postcode_full).to eq("A1 1AA") + end + + it "correctly sets address and uprn if uprn is given" do + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.uprn_known).to eq(1) + expect(sales_log&.uprn).to eq("123456781234") + expect(sales_log&.address_line1).to eq("321") + expect(sales_log&.address_line2).to eq("Double Dependent Locality") + expect(sales_log&.town_or_city).to eq("Westminster") + expect(sales_log&.postcode_full).to eq("LS16 6FT") + expect(sales_log&.la).to eq("E08000035") + end + + it "correctly sets address and uprn if uprn is given but not recognised" do + sales_log_xml.at_xpath("//xmlns:UPRN").content = "123" + + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.uprn_known).to eq(0) + expect(sales_log&.uprn).to be_nil + expect(sales_log&.address_line1).to eq("address 1") + expect(sales_log&.address_line2).to eq("address 2") + expect(sales_log&.town_or_city).to eq("towncity") + expect(sales_log&.county).to eq("county") + expect(sales_log&.postcode_full).to eq("A1 1AA") + expect(sales_log&.la).to eq("E09000033") + end + end + context "when setting default buyer 1 previous tenancy" do let(:sales_log_id) { "outright_sale_sales_log" }