diff --git a/app/models/lettings_log.rb b/app/models/lettings_log.rb index 3d01c5284..45d13a792 100644 --- a/app/models/lettings_log.rb +++ b/app/models/lettings_log.rb @@ -572,6 +572,14 @@ class LettingsLog < Log is_general_needs? && referral == 4 end + def has_any_person_details?(person_index) + ["sex#{person_index}", "relat#{person_index}", "ecstat#{person_index}"].any? { |field| public_send(field).present? } || public_send("age#{person_index}_known") == 1 + end + + def details_not_known_for_person?(person_index) + public_send("details_known_#{person_index}") == 1 + end + private def reset_invalid_unresolved_log_fields! diff --git a/app/services/imports/lettings_logs_field_import_service.rb b/app/services/imports/lettings_logs_field_import_service.rb index d63cc3354..75acef8fd 100644 --- a/app/services/imports/lettings_logs_field_import_service.rb +++ b/app/services/imports/lettings_logs_field_import_service.rb @@ -24,6 +24,8 @@ module Imports import_from(folder, :update_sex_and_relat) when "general_needs_referral" import_from(folder, :update_general_needs_referral) + when "person_details" + import_from(folder, :update_person_details) else raise "Updating #{field} is not supported by the field import service" end @@ -265,7 +267,7 @@ module Imports (2..record.hhmemb).each do |i| next @logger.info("lettings log #{record.id} has values for sex#{i} and relat#{i}, skipping update") if record["sex#{i}"] && record["relat#{i}"] - next @logger.info("lettings log #{record.id} has value 'no' for details_known_#{i}, skipping update") if record["details_known_#{i}"] == 1 + next @logger.info("lettings log #{record.id} has value 'no' for details_known_#{i}, skipping update") if record.details_not_known_for_person?(i) if record["sex#{i}"].blank? record["sex#{i}"] = sex(xml_doc, i) @@ -299,5 +301,113 @@ module Imports record.update!(referral: 4, referral_value_check: 0, values_updated_at: Time.zone.now) @logger.info("lettings log #{record.id}'s referral value has been set to 4") end + + def update_person_details(xml_doc) + return if meta_field_value(xml_doc, "form-name").include?("Sales") + + old_id = meta_field_value(xml_doc, "document-id") + record = LettingsLog.find_by(old_id:) + + return @logger.warn("lettings log with old id #{old_id} not found") unless record + return @logger.info("lettings log #{record.id} has no hhmemb, skipping update") unless record.hhmemb + + if (2..record.hhmemb).all? { |person_index| record.has_any_person_details?(person_index) || record.details_not_known_for_person?(person_index) } + return @logger.info("lettings log #{record.id} has all household member details, skipping update") + end + if record.hhmemb == 8 || ((record.hhmemb + 1)..8).none? { |person_index| file_contains_person_details?(xml_doc, person_index) } + return @logger.info("lettings log #{record.id} has no additional household member details, skipping update") + end + + person_index = 2 + next_person_index = person_index + 1 + + while person_exists_on_the_log?(record, person_index) + if person_exists_on_the_log?(record, next_person_index) + if record.has_any_person_details?(person_index) || record.details_not_known_for_person?(person_index) + @logger.info("lettings log #{record.id} has details for person #{person_index}, skipping person") + person_index += 1 + next_person_index += 1 + next + end + + if !record.has_any_person_details?(next_person_index) && !record.details_not_known_for_person?(next_person_index) + next_person_index += 1 + next + end + + move_person_details(record, person_index, next_person_index) + else + reimport_person_details(record, xml_doc, person_index, next_person_index) + end + + person_index += 1 + next_person_index += 1 + end + + record.values_updated_at = Time.zone.now + record.save! + end + + def age_known(xml_doc, person_index) + age_refused = string_or_nil(xml_doc, "P#{person_index}AR") + if age_refused.present? + if age_refused.casecmp?("AGE_REFUSED") || age_refused.casecmp?("No") + return 1 # No + else + return 0 # Yes + end + end + 0 + end + + def details_known(index, record) + if record["age#{index}_known"] == 1 && + record["sex#{index}"] == "R" && + record["relat#{index}"] == "R" && + record["ecstat#{index}"] == 10 + 1 # No + else + 0 # Yes + end + end + + def file_contains_person_details?(xml_doc, person_index) + safe_string_as_integer(xml_doc, "P#{person_index}Age").present? || + string_or_nil(xml_doc, "P#{person_index}Sex").present? || + unsafe_string_as_integer(xml_doc, "P#{person_index}Eco").present? || + string_or_nil(xml_doc, "P#{person_index}Rel").present? + end + + def move_person_details(record, person_index, next_person_index) + record["details_known_#{person_index}"] = record["details_known_#{next_person_index}"] + record["age#{person_index}"] = record["age#{next_person_index}"] + record["age#{person_index}_known"] = record["age#{next_person_index}_known"] + record["sex#{person_index}"] = record["sex#{next_person_index}"] + record["ecstat#{person_index}"] = record["ecstat#{next_person_index}"] + record["relat#{person_index}"] = record["relat#{next_person_index}"] + + record["details_known_#{next_person_index}"] = nil + record["age#{next_person_index}"] = nil + record["age#{next_person_index}_known"] = nil + record["sex#{next_person_index}"] = nil + record["ecstat#{next_person_index}"] = nil + record["relat#{next_person_index}"] = nil + + @logger.info("lettings log #{record.id}'s person #{next_person_index} details moved to person #{person_index} details") + end + + def reimport_person_details(record, xml_doc, person_index, next_person_index) + record["age#{person_index}"] = safe_string_as_integer(xml_doc, "P#{next_person_index}Age") + record["age#{person_index}_known"] = age_known(xml_doc, next_person_index) + record["sex#{person_index}"] = sex(xml_doc, next_person_index) + record["ecstat#{person_index}"] = unsafe_string_as_integer(xml_doc, "P#{next_person_index}Eco") + record["relat#{person_index}"] = relat(xml_doc, next_person_index) + record["details_known_#{person_index}"] = details_known(person_index, record) + @logger.info("lettings log #{record.id}, reimported person #{person_index} details") + end + + def person_exists_on_the_log?(record, person_index) + person_index <= record.hhmemb + end end end diff --git a/lib/tasks/data_import_field.rake b/lib/tasks/data_import_field.rake index f9879b4c6..3c3be4d2f 100644 --- a/lib/tasks/data_import_field.rake +++ b/lib/tasks/data_import_field.rake @@ -7,7 +7,7 @@ namespace :core do # We only allow a reduced list of known fields to be updatable case field - when "tenancycode", "major_repairs", "lettings_allocation", "offered", "address", "reason", "homeless", "created_by", "sex_and_relat", "general_needs_referral" + when "tenancycode", "major_repairs", "lettings_allocation", "offered", "address", "reason", "homeless", "created_by", "sex_and_relat", "general_needs_referral", "person_details" s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"]) archive_io = s3_service.get_file_io(path) archive_service = Storage::ArchiveService.new(archive_io) diff --git a/spec/lib/tasks/data_import_field_spec.rb b/spec/lib/tasks/data_import_field_spec.rb index 6973e0762..b84cb9d23 100644 --- a/spec/lib/tasks/data_import_field_spec.rb +++ b/spec/lib/tasks/data_import_field_spec.rb @@ -164,6 +164,18 @@ describe "data_import_field imports" do end end + context "and we update the person_details field" do + let(:field) { "person_details" } + + it "updates the 2023 logs from the given XML file" do + expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name) + expect(storage_service).to receive(:get_file_io).with("spec/fixtures/imports/logs") + expect(Imports::LettingsLogsFieldImportService).to receive(:new).with(archive_service) + expect(import_service).to receive(:update_field).with(field, "logs") + task.invoke(field, fixture_path) + end + end + it "raises an exception if no parameters are provided" do expect { task.invoke }.to raise_error(/Usage/) end diff --git a/spec/services/imports/lettings_logs_field_import_service_spec.rb b/spec/services/imports/lettings_logs_field_import_service_spec.rb index 9cc3ddfe5..7f7c8bf3c 100644 --- a/spec/services/imports/lettings_logs_field_import_service_spec.rb +++ b/spec/services/imports/lettings_logs_field_import_service_spec.rb @@ -1021,4 +1021,336 @@ RSpec.describe Imports::LettingsLogsFieldImportService do end end end + + context "when updating person_details" do + let(:field) { "person_details" } + let(:lettings_log) { LettingsLog.find_by(old_id: lettings_log_id) } + + before do + Imports::LettingsLogsImportService.new(storage_service, logger).create_logs(fixture_directory) + lettings_log_file.rewind + lettings_log.update!(values_updated_at: nil) + lettings_log_xml.at_xpath("//xmlns:P4Age").content = 7 + lettings_log_xml.at_xpath("//xmlns:P4Sex").content = "Male" + lettings_log_xml.at_xpath("//xmlns:P4Rel").content = "Child" + lettings_log_xml.at_xpath("//xmlns:P4Eco").content = "9) Child under 16" + + lettings_log_xml.at_xpath("//xmlns:P7Age").content = 7 + lettings_log_xml.at_xpath("//xmlns:P7Sex").content = "Male" + lettings_log_xml.at_xpath("//xmlns:P7Rel").content = "Child" + lettings_log_xml.at_xpath("//xmlns:P7Eco").content = "9) Child under 16" + + lettings_log_xml.at_xpath("//xmlns:P8Age").content = 21 + end + + context "when the lettings log has no details for person 2" do + before do + lettings_log.update(details_known_2: 0, age2_known: nil, age2: nil, sex2: nil, ecstat2: nil, relat2: nil, hhmemb: 3, + details_known_3: 0, age3_known: 0, age3: 19, sex3: "F", ecstat3: 10, relat3: "X") + end + + it "moves the details of person 3 to person 2 and imports missing person 3 values" do + expect(logger).to receive(:info).with(/lettings log \d+'s person 3 details moved to person 2 details/) + expect(logger).to receive(:info).with(/lettings log \d+, reimported person 3 details/) + import_service.send(:update_person_details, lettings_log_xml) + lettings_log.reload + expect(lettings_log.age2_known).to eq(0) + expect(lettings_log.age2).to eq(19) + expect(lettings_log.sex2).to eq("F") + expect(lettings_log.ecstat2).to eq(10) + expect(lettings_log.relat2).to eq("X") + + expect(lettings_log.age3_known).to eq(0) + expect(lettings_log.age3).to eq(7) + expect(lettings_log.sex3).to eq("M") + expect(lettings_log.ecstat3).to eq(9) + expect(lettings_log.relat3).to eq("C") + expect(lettings_log.values_updated_at).not_to be_nil + end + end + + context "when the lettings log has no details for person 2 and there are 6 household members" do + before do + lettings_log.update(details_known_2: 0, age2_known: nil, age2: nil, sex2: nil, ecstat2: nil, relat2: nil, hhmemb: 6, + details_known_3: 0, age3_known: 0, age3: 19, sex3: "F", ecstat3: 10, relat3: "X", + details_known_4: 1, age4_known: nil, age4: nil, sex4: nil, ecstat4: nil, relat4: nil, + details_known_5: 0, age5_known: 0, age5: 21, sex5: "M", ecstat5: 1, relat5: "P", + details_known_6: 0, age6_known: 0, age6: 22, sex6: "R", ecstat6: 10, relat6: "R") + end + + it "moves the details of all the household members" do + expect(logger).to receive(:info).with(/lettings log \d+'s person 3 details moved to person 2 details/) + expect(logger).to receive(:info).with(/lettings log \d+'s person 4 details moved to person 3 details/) + expect(logger).to receive(:info).with(/lettings log \d+'s person 5 details moved to person 4 details/) + expect(logger).to receive(:info).with(/lettings log \d+'s person 6 details moved to person 5 details/) + expect(logger).to receive(:info).with(/lettings log \d+, reimported person 6 details/) + import_service.send(:update_person_details, lettings_log_xml) + lettings_log.reload + expect(lettings_log.age2_known).to eq(0) + expect(lettings_log.age2).to eq(19) + expect(lettings_log.sex2).to eq("F") + expect(lettings_log.ecstat2).to eq(10) + expect(lettings_log.relat2).to eq("X") + + expect(lettings_log.details_known_3).to eq(1) + expect(lettings_log.age3_known).to eq(nil) + expect(lettings_log.age3).to eq(nil) + expect(lettings_log.sex3).to eq(nil) + expect(lettings_log.ecstat3).to eq(nil) + expect(lettings_log.relat3).to eq(nil) + + expect(lettings_log.details_known_4).to eq(0) + expect(lettings_log.age4_known).to eq(0) + expect(lettings_log.age4).to eq(21) + expect(lettings_log.sex4).to eq("M") + expect(lettings_log.ecstat4).to eq(1) + expect(lettings_log.relat4).to eq("P") + + expect(lettings_log.details_known_5).to eq(0) + expect(lettings_log.age5_known).to eq(0) + expect(lettings_log.age5).to eq(22) + expect(lettings_log.sex5).to eq("R") + expect(lettings_log.ecstat5).to eq(10) + expect(lettings_log.relat5).to eq("R") + + expect(lettings_log.details_known_6).to eq(0) + expect(lettings_log.age6_known).to eq(0) + expect(lettings_log.age6).to eq(7) + expect(lettings_log.sex6).to eq("M") + expect(lettings_log.ecstat6).to eq(9) + expect(lettings_log.relat6).to eq("C") + + expect(lettings_log.values_updated_at).not_to be_nil + end + end + + context "when the lettings log has no details for person 4 and there are 6 household members" do + before do + lettings_log.update(details_known_2: 1, age2_known: nil, age2: nil, sex2: nil, ecstat2: nil, relat2: nil, hhmemb: 6, + details_known_3: 0, age3_known: 0, age3: 19, sex3: "F", ecstat3: 10, relat3: "X", + details_known_4: 0, age4_known: nil, age4: nil, sex4: nil, ecstat4: nil, relat4: nil, + details_known_5: 0, age5_known: 0, age5: 21, sex5: "M", ecstat5: 1, relat5: "P", + details_known_6: 0, age6_known: 0, age6: 22, sex6: "R", ecstat6: 10, relat6: "R") + end + + it "moves the details of all the household members" do + expect(logger).to receive(:info).with(/lettings log \d+ has details for person 2, skipping person/) + expect(logger).to receive(:info).with(/lettings log \d+ has details for person 3, skipping person/) + expect(logger).to receive(:info).with(/lettings log \d+'s person 5 details moved to person 4 details/) + expect(logger).to receive(:info).with(/lettings log \d+'s person 6 details moved to person 5 details/) + expect(logger).to receive(:info).with(/lettings log \d+, reimported person 6 details/) + import_service.send(:update_person_details, lettings_log_xml) + lettings_log.reload + expect(lettings_log.details_known_2).to eq(1) + expect(lettings_log.age2_known).to eq(nil) + expect(lettings_log.age2).to eq(nil) + expect(lettings_log.sex2).to eq(nil) + expect(lettings_log.ecstat2).to eq(nil) + expect(lettings_log.relat2).to eq(nil) + + expect(lettings_log.age3_known).to eq(0) + expect(lettings_log.age3).to eq(19) + expect(lettings_log.sex3).to eq("F") + expect(lettings_log.ecstat3).to eq(10) + expect(lettings_log.relat3).to eq("X") + + expect(lettings_log.details_known_4).to eq(0) + expect(lettings_log.age4_known).to eq(0) + expect(lettings_log.age4).to eq(21) + expect(lettings_log.sex4).to eq("M") + expect(lettings_log.ecstat4).to eq(1) + expect(lettings_log.relat4).to eq("P") + + expect(lettings_log.details_known_5).to eq(0) + expect(lettings_log.age5_known).to eq(0) + expect(lettings_log.age5).to eq(22) + expect(lettings_log.sex5).to eq("R") + expect(lettings_log.ecstat5).to eq(10) + expect(lettings_log.relat5).to eq("R") + + expect(lettings_log.details_known_6).to eq(0) + expect(lettings_log.age6_known).to eq(0) + expect(lettings_log.age6).to eq(7) + expect(lettings_log.sex6).to eq("M") + expect(lettings_log.ecstat6).to eq(9) + expect(lettings_log.relat6).to eq("C") + + expect(lettings_log.values_updated_at).not_to be_nil + end + end + + context "when the lettings log has no details for several consecutive household members" do + before do + lettings_log.update(details_known_2: 1, age2_known: nil, age2: nil, sex2: nil, ecstat2: nil, relat2: nil, hhmemb: 6, + details_known_3: 0, age3_known: 0, age3: 19, sex3: "F", ecstat3: 10, relat3: "X", + details_known_4: 0, age4_known: nil, age4: nil, sex4: nil, ecstat4: nil, relat4: nil, + details_known_5: 0, age5_known: nil, age5: nil, sex5: nil, ecstat5: nil, relat5: nil, + details_known_6: 0, age6_known: 0, age6: 22, sex6: "R", ecstat6: 10, relat6: "R") + end + + it "moves the details of all the relevant household members" do + expect(logger).to receive(:info).with(/lettings log \d+ has details for person 2, skipping person/) + expect(logger).to receive(:info).with(/lettings log \d+ has details for person 3, skipping person/) + expect(logger).to receive(:info).with(/lettings log \d+'s person 6 details moved to person 4 details/) + expect(logger).to receive(:info).with(/lettings log \d+, reimported person 5 details/) + expect(logger).to receive(:info).with(/lettings log \d+, reimported person 6 details/) + import_service.send(:update_person_details, lettings_log_xml) + lettings_log.reload + expect(lettings_log.details_known_2).to eq(1) + expect(lettings_log.age2_known).to eq(nil) + expect(lettings_log.age2).to eq(nil) + expect(lettings_log.sex2).to eq(nil) + expect(lettings_log.ecstat2).to eq(nil) + expect(lettings_log.relat2).to eq(nil) + + expect(lettings_log.age3_known).to eq(0) + expect(lettings_log.age3).to eq(19) + expect(lettings_log.sex3).to eq("F") + expect(lettings_log.ecstat3).to eq(10) + expect(lettings_log.relat3).to eq("X") + + expect(lettings_log.details_known_4).to eq(0) + expect(lettings_log.age4_known).to eq(0) + expect(lettings_log.age4).to eq(22) + expect(lettings_log.sex4).to eq("R") + expect(lettings_log.ecstat4).to eq(10) + expect(lettings_log.relat4).to eq("R") + + expect(lettings_log.details_known_5).to eq(0) + expect(lettings_log.age5_known).to eq(0) + expect(lettings_log.age5).to eq(7) + expect(lettings_log.sex5).to eq("M") + expect(lettings_log.ecstat5).to eq(9) + expect(lettings_log.relat5).to eq("C") + + expect(lettings_log.details_known_6).to eq(0) + expect(lettings_log.age6_known).to eq(0) + expect(lettings_log.age6).to eq(21) + expect(lettings_log.sex6).to eq(nil) + expect(lettings_log.ecstat6).to eq(nil) + expect(lettings_log.relat6).to eq(nil) + + expect(lettings_log.values_updated_at).not_to be_nil + end + end + + context "when the lettings log has no details for several non consecutive household members" do + before do + lettings_log.update(details_known_2: 0, age2_known: nil, age2: nil, sex2: nil, ecstat2: nil, relat2: nil, hhmemb: 6, + details_known_3: 0, age3_known: 0, age3: 19, sex3: "F", ecstat3: 10, relat3: "X", + details_known_4: 1, age4_known: nil, age4: nil, sex4: nil, ecstat4: nil, relat4: nil, + details_known_5: 0, age5_known: nil, age5: nil, sex5: nil, ecstat5: nil, relat5: nil, + details_known_6: 0, age6_known: 0, age6: 22, sex6: "R", ecstat6: 10, relat6: "R") + end + + it "moves the details of all the relevant household members" do + expect(logger).to receive(:info).with(/lettings log \d+'s person 3 details moved to person 2 details/) + expect(logger).to receive(:info).with(/lettings log \d+'s person 4 details moved to person 3 details/) + expect(logger).to receive(:info).with(/lettings log \d+'s person 6 details moved to person 4 details/) + expect(logger).to receive(:info).with(/lettings log \d+, reimported person 5 details/) + expect(logger).to receive(:info).with(/lettings log \d+, reimported person 6 details/) + import_service.send(:update_person_details, lettings_log_xml) + lettings_log.reload + expect(lettings_log.age2_known).to eq(0) + expect(lettings_log.age2).to eq(19) + expect(lettings_log.sex2).to eq("F") + expect(lettings_log.ecstat2).to eq(10) + expect(lettings_log.relat2).to eq("X") + + expect(lettings_log.details_known_3).to eq(1) + expect(lettings_log.age3_known).to eq(nil) + expect(lettings_log.age3).to eq(nil) + expect(lettings_log.sex3).to eq(nil) + expect(lettings_log.ecstat3).to eq(nil) + expect(lettings_log.relat3).to eq(nil) + + expect(lettings_log.details_known_4).to eq(0) + expect(lettings_log.age4_known).to eq(0) + expect(lettings_log.age4).to eq(22) + expect(lettings_log.sex4).to eq("R") + expect(lettings_log.ecstat4).to eq(10) + expect(lettings_log.relat4).to eq("R") + + expect(lettings_log.details_known_5).to eq(0) + expect(lettings_log.age5_known).to eq(0) + expect(lettings_log.age5).to eq(7) + expect(lettings_log.sex5).to eq("M") + expect(lettings_log.ecstat5).to eq(9) + expect(lettings_log.relat5).to eq("C") + + expect(lettings_log.details_known_6).to eq(0) + expect(lettings_log.age6_known).to eq(0) + expect(lettings_log.age6).to eq(21) + expect(lettings_log.sex6).to eq(nil) + expect(lettings_log.ecstat6).to eq(nil) + expect(lettings_log.relat6).to eq(nil) + + expect(lettings_log.values_updated_at).not_to be_nil + end + end + + context "when the lettings log has details for person 2" do + let(:lettings_log) { LettingsLog.find_by(old_id: lettings_log_id) } + + before do + lettings_log.update(details_known_2: 0, age2: 22) + end + + it "does not update the lettings_log person details" do + expect(logger).to receive(:info).with(/lettings log \d+ has all household member details, skipping update/) + import_service.send(:update_person_details, lettings_log_xml) + lettings_log.reload + expect(lettings_log.values_updated_at).to be_nil + end + end + + context "when the person 2 details are not known" do + let(:lettings_log) { LettingsLog.find_by(old_id: lettings_log_id) } + + before do + lettings_log.update(details_known_2: 1) + end + + it "does not update the lettings_log person details" do + expect(logger).to receive(:info).with(/lettings log \d+ has all household member details, skipping update/) + import_service.send(:update_person_details, lettings_log_xml) + lettings_log.reload + expect(lettings_log.values_updated_at).to be_nil + end + end + + context "when none of the details past hhmemb are given in the reimport" do + let(:lettings_log) { LettingsLog.find_by(old_id: lettings_log_id) } + + before do + lettings_log.update!(hhmemb: 7) + lettings_log_xml.at_xpath("//xmlns:P8Age").content = "" + lettings_log_xml.at_xpath("//xmlns:P8Sex").content = "" + lettings_log_xml.at_xpath("//xmlns:P8Rel").content = "" + lettings_log_xml.at_xpath("//xmlns:P8Eco").content = "" + end + + it "does not update the lettings_log person details" do + expect(logger).to receive(:info).with(/lettings log \d+ has no additional household member details, skipping update/) + import_service.send(:update_person_details, lettings_log_xml) + lettings_log.reload + expect(lettings_log.values_updated_at).to be_nil + end + end + + context "when the record has no hhmemb" do + let(:lettings_log) { LettingsLog.find_by(old_id: lettings_log_id) } + + before do + lettings_log.update!(hhmemb: nil) + end + + it "does not update the lettings_log person details" do + expect(logger).to receive(:info).with(/lettings log \d+ has no hhmemb, skipping update/) + import_service.send(:update_person_details, lettings_log_xml) + lettings_log.reload + expect(lettings_log.values_updated_at).to be_nil + end + end + end end