From e0e09a22d27465d174187eea84a26da46d36d942 Mon Sep 17 00:00:00 2001 From: kosiakkatrina <54268893+kosiakkatrina@users.noreply.github.com> Date: Fri, 24 Feb 2023 15:16:28 +0000 Subject: [PATCH 1/8] CLDS-1938 Add sales logs importer (#1331) * Create test fixtures * Add old id column * Add sales logs importer * Save and update completed discounted ownership log without postcode * Update fixtures to be incomplete * Import a completed shared ownership example * Test with a non homebuy shared ownership example * Add privacynotice * Update hholdcount and confirmed fields * Add buyer stilll serving mapping * Add totadult/totchild and outright sale examples * Update log fixtures * Extract shared methods into logs import service and lint * Add sales logs import rake task * Update noint, xml examples and some mappings * Add tests for checking that all required questions are answered * Update tests, clean up import * Map mortgage lender and mortgage lender other * Infer Mscharge known as no for outright sale * refactor setting default values * when the armedforcesspouse is not answered set is as don't know * Refactor tests: change log id names * set savings to not known if not given * Refactor tests: change nesting * Backfill default household characteristics for completed log * Add more default mapping * Typo * Improve logging and refactor tests * Adjust test data to fit with the mappings that are known so far * Rename fixture files --- .../imports/lettings_logs_import_service.rb | 113 +--- app/services/imports/logs_import_service.rb | 127 ++++ .../imports/sales_logs_import_service.rb | 450 +++++++++++++ ...20230215112932_add_old_id_to_sales_logs.rb | 8 + db/schema.rb | 10 +- lib/tasks/data_import.rake | 2 + .../discounted_ownership_sales_log.xml | 351 +++++++++++ .../sales_logs/outright_sale_sales_log.xml | 333 ++++++++++ .../sales_logs/shared_ownership_sales_log.xml | 333 ++++++++++ .../shared_ownership_sales_log2.xml | 333 ++++++++++ .../shared_ownership_sales_log3.xml | 333 ++++++++++ .../shared_ownership_sales_log4.xml | 333 ++++++++++ spec/lib/tasks/data_import_spec.rb | 18 + .../imports/sales_logs_import_service_spec.rb | 595 ++++++++++++++++++ 14 files changed, 3223 insertions(+), 116 deletions(-) create mode 100644 app/services/imports/logs_import_service.rb create mode 100644 app/services/imports/sales_logs_import_service.rb create mode 100644 db/migrate/20230215112932_add_old_id_to_sales_logs.rb create mode 100644 spec/fixtures/imports/sales_logs/discounted_ownership_sales_log.xml create mode 100644 spec/fixtures/imports/sales_logs/outright_sale_sales_log.xml create mode 100644 spec/fixtures/imports/sales_logs/shared_ownership_sales_log.xml create mode 100644 spec/fixtures/imports/sales_logs/shared_ownership_sales_log2.xml create mode 100644 spec/fixtures/imports/sales_logs/shared_ownership_sales_log3.xml create mode 100644 spec/fixtures/imports/sales_logs/shared_ownership_sales_log4.xml create mode 100644 spec/services/imports/sales_logs_import_service_spec.rb diff --git a/app/services/imports/lettings_logs_import_service.rb b/app/services/imports/lettings_logs_import_service.rb index 8fe71d3dc..a215e8f5a 100644 --- a/app/services/imports/lettings_logs_import_service.rb +++ b/app/services/imports/lettings_logs_import_service.rb @@ -1,5 +1,5 @@ module Imports - class LettingsLogsImportService < ImportService + class LettingsLogsImportService < LogsImportService def initialize(storage_service, logger = Rails.logger) @logs_with_discrepancies = Set.new @logs_overridden = Set.new @@ -318,43 +318,6 @@ module Imports end end - # Safe: A string that represents only an integer (or empty/nil) - def safe_string_as_integer(xml_doc, attribute) - str = field_value(xml_doc, "xmlns", attribute) - Integer(str, exception: false) - end - - # Safe: A string that represents only a decimal (or empty/nil) - def safe_string_as_decimal(xml_doc, attribute) - str = string_or_nil(xml_doc, attribute) - if str.nil? - nil - else - BigDecimal(str, exception: false) - end - end - - # Unsafe: A string that has more than just the integer value - def unsafe_string_as_integer(xml_doc, attribute) - str = string_or_nil(xml_doc, attribute) - if str.nil? - nil - else - str.to_i - end - end - - def compose_date(xml_doc, day_str, month_str, year_str) - day = Integer(field_value(xml_doc, "xmlns", day_str), exception: false) - month = Integer(field_value(xml_doc, "xmlns", month_str), exception: false) - year = Integer(field_value(xml_doc, "xmlns", year_str), exception: false) - if day.nil? || month.nil? || year.nil? - nil - else - Time.zone.local(year, month, day) - end - end - def get_form_name_component(xml_doc, index) form_name = meta_field_value(xml_doc, "form-name") form_type_components = form_name.split("-") @@ -399,42 +362,6 @@ module Imports end end - def find_organisation_id(xml_doc, id_field) - old_visible_id = string_or_nil(xml_doc, id_field) - organisation = Organisation.find_by(old_visible_id:) - raise "Organisation not found with legacy ID #{old_visible_id}" if organisation.nil? - - organisation.id - end - - def sex(xml_doc, index) - sex = string_or_nil(xml_doc, "P#{index}Sex") - case sex - when "Male" - "M" - when "Female" - "F" - when "Other", "Non-binary" - "X" - when "Refused" - "R" - end - end - - def relat(xml_doc, index) - relat = string_or_nil(xml_doc, "P#{index}Rel") - case relat - when "Child" - "C" - when "Partner" - "P" - when "Other", "Non-binary" - "X" - when "Refused" - "R" - end - end - def age_known(xml_doc, index, hhmemb) return nil if hhmemb.present? && index > hhmemb @@ -473,16 +400,6 @@ module Imports end end - def compose_postcode(xml_doc, outcode, incode) - outcode_value = string_or_nil(xml_doc, outcode) - incode_value = string_or_nil(xml_doc, incode) - if outcode_value.nil? || incode_value.nil? || !"#{outcode_value} #{incode_value}".match(POSTCODE_REGEXP) - nil - else - "#{outcode_value} #{incode_value}" - end - end - def london_affordable_rent(xml_doc) lar = unsafe_string_as_integer(xml_doc, "LAR") if lar == 1 @@ -502,34 +419,6 @@ module Imports end end - def string_or_nil(xml_doc, attribute) - str = field_value(xml_doc, "xmlns", attribute) - str.presence - end - - def ethnic_group(ethnic) - case ethnic - when 1, 2, 3, 18 - # White - 0 - when 4, 5, 6, 7 - # Mixed - 1 - when 8, 9, 10, 11, 15 - # Asian - 2 - when 12, 13, 14 - # Black - 3 - when 16, 19 - # Others - 4 - when 17 - # Refused - 17 - end - end - # Letters should be lowercase to match case def housing_needs(xml_doc, letter) housing_need = string_or_nil(xml_doc, "Q10-#{letter}") diff --git a/app/services/imports/logs_import_service.rb b/app/services/imports/logs_import_service.rb new file mode 100644 index 000000000..0a3d2df87 --- /dev/null +++ b/app/services/imports/logs_import_service.rb @@ -0,0 +1,127 @@ +module Imports + class LogsImportService < ImportService + private + + # Safe: A string that represents only an integer (or empty/nil) + def safe_string_as_integer(xml_doc, attribute) + str = field_value(xml_doc, "xmlns", attribute) + Integer(str, exception: false) + end + + # Unsafe: A string that has more than just the integer value + def unsafe_string_as_integer(xml_doc, attribute) + str = string_or_nil(xml_doc, attribute) + if str.nil? + nil + else + str.to_i + end + end + + def compose_date(xml_doc, day_str, month_str, year_str) + day = Integer(field_value(xml_doc, "xmlns", day_str), exception: false) + month = Integer(field_value(xml_doc, "xmlns", month_str), exception: false) + year = Integer(field_value(xml_doc, "xmlns", year_str), exception: false) + if day.nil? || month.nil? || year.nil? + nil + else + Time.zone.local(year, month, day) + end + end + + def find_organisation_id(xml_doc, id_field) + old_visible_id = string_or_nil(xml_doc, id_field) + organisation = Organisation.find_by(old_visible_id:) + raise "Organisation not found with legacy ID #{old_visible_id}" if organisation.nil? + + organisation.id + end + + def string_or_nil(xml_doc, attribute) + str = field_value(xml_doc, "xmlns", attribute) + str.presence + end + + def ethnic_group(ethnic) + case ethnic + when 1, 2, 3, 18 + # White + 0 + when 4, 5, 6, 7 + # Mixed + 1 + when 8, 9, 10, 11, 15 + # Asian + 2 + when 12, 13, 14 + # Black + 3 + when 16, 19 + # Others + 4 + when 17 + # Refused + 17 + end + end + + # Safe: A string that represents only a decimal (or empty/nil) + def safe_string_as_decimal(xml_doc, attribute) + str = string_or_nil(xml_doc, attribute) + if str.nil? + nil + else + BigDecimal(str, exception: false) + end + end + + def compose_postcode(xml_doc, outcode, incode) + outcode_value = string_or_nil(xml_doc, outcode) + incode_value = string_or_nil(xml_doc, incode) + if outcode_value.nil? || incode_value.nil? || !"#{outcode_value} #{incode_value}".match(POSTCODE_REGEXP) + nil + else + "#{outcode_value} #{incode_value}" + end + end + + def previous_postcode_known(xml_doc, previous_postcode, prevloc) + previous_postcode_known = string_or_nil(xml_doc, "Q7UnknownPostcode") + if previous_postcode_known == "If postcode not known tick" || (previous_postcode.nil? && prevloc.present?) + 1 + elsif previous_postcode.nil? + nil + else + 0 + end + end + + def sex(xml_doc, index) + sex = string_or_nil(xml_doc, "P#{index}Sex") + case sex + when "Male" + "M" + when "Female" + "F" + when "Other", "Non-binary" + "X" + when "Refused" + "R" + end + end + + def relat(xml_doc, index) + relat = string_or_nil(xml_doc, "P#{index}Rel") + case relat + when "Child" + "C" + when "Partner" + "P" + when "Other", "Non-binary" + "X" + when "Refused", "Buyer prefers not to say" + "R" + end + end + end +end diff --git a/app/services/imports/sales_logs_import_service.rb b/app/services/imports/sales_logs_import_service.rb new file mode 100644 index 000000000..25f8148b8 --- /dev/null +++ b/app/services/imports/sales_logs_import_service.rb @@ -0,0 +1,450 @@ +module Imports + class SalesLogsImportService < LogsImportService + def initialize(storage_service, logger = Rails.logger) + @logs_with_discrepancies = Set.new + @logs_overridden = Set.new + super + end + + def create_logs(folder) + import_from(folder, :create_log) + if @logs_with_discrepancies.count.positive? + @logger.warn("The following sales logs had status discrepancies: [#{@logs_with_discrepancies.join(', ')}]") + end + end + + private + + def create_log(xml_doc) + attributes = {} + + previous_status = meta_field_value(xml_doc, "status") + + # Required fields for status complete or logic to work + # Note: order matters when we derive from previous values (attributes parameter) + + attributes["saledate"] = compose_date(xml_doc, "DAY", "MONTH", "YEAR") + attributes["owning_organisation_id"] = find_organisation_id(xml_doc, "OWNINGORGID") + attributes["type"] = unsafe_string_as_integer(xml_doc, "DerSaleType") + attributes["old_id"] = meta_field_value(xml_doc, "document-id") + attributes["created_at"] = Time.zone.parse(meta_field_value(xml_doc, "created-date")) + attributes["updated_at"] = Time.zone.parse(meta_field_value(xml_doc, "modified-date")) + attributes["purchid"] = string_or_nil(xml_doc, "PurchaserCode") + attributes["ownershipsch"] = unsafe_string_as_integer(xml_doc, "Ownership") + attributes["othtype"] = string_or_nil(xml_doc, "Q38OtherSale") + attributes["jointmore"] = unsafe_string_as_integer(xml_doc, "JointMore") + attributes["jointpur"] = unsafe_string_as_integer(xml_doc, "joint") + attributes["beds"] = safe_string_as_integer(xml_doc, "Q11Bedrooms") + attributes["companybuy"] = unsafe_string_as_integer(xml_doc, "company") if attributes["ownershipsch"] == 3 + attributes["hhmemb"] = safe_string_as_integer(xml_doc, "HHMEMB") + (1..6).each do |index| + attributes["age#{index}"] = safe_string_as_integer(xml_doc, "P#{index}Age") + attributes["sex#{index}"] = sex(xml_doc, index) + attributes["ecstat#{index}"] = unsafe_string_as_integer(xml_doc, "P#{index}Eco") + attributes["age#{index}_known"] = age_known(xml_doc, index, attributes["hhmemb"], attributes["age#{index}"]) + end + (2..6).each do |index| + attributes["relat#{index}"] = relat(xml_doc, index) + attributes["details_known_#{index}"] = details_known(index, attributes) + end + attributes["national"] = unsafe_string_as_integer(xml_doc, "P1Nat") + attributes["othernational"] = nil + attributes["ethnic"] = unsafe_string_as_integer(xml_doc, "P1Eth") + attributes["ethnic_group"] = ethnic_group(attributes["ethnic"]) + attributes["buy1livein"] = unsafe_string_as_integer(xml_doc, "LiveInBuyer1") + attributes["buylivein"] = unsafe_string_as_integer(xml_doc, "LiveInBuyer") if attributes["ownershipsch"] == 3 + attributes["builtype"] = unsafe_string_as_integer(xml_doc, "Q13BuildingType") + attributes["proptype"] = unsafe_string_as_integer(xml_doc, "Q12PropertyType") + attributes["privacynotice"] = 1 if string_or_nil(xml_doc, "Qdp") == "Yes" + attributes["noint"] = unsafe_string_as_integer(xml_doc, "PartAPurchaser") + attributes["buy2livein"] = unsafe_string_as_integer(xml_doc, "LiveInBuyer2") + attributes["wheel"] = unsafe_string_as_integer(xml_doc, "Q10Wheelchair") + attributes["hholdcount"] = safe_string_as_integer(xml_doc, "LiveInOther") + attributes["la"] = string_or_nil(xml_doc, "Q14ONSLACode") + attributes["income1"] = safe_string_as_integer(xml_doc, "Q2Person1Income") + attributes["income1nk"] = income_known(unsafe_string_as_integer(xml_doc, "P1IncKnown")) + attributes["inc1mort"] = unsafe_string_as_integer(xml_doc, "Q2Person1Mortgage") + attributes["income2"] = safe_string_as_integer(xml_doc, "Q2Person2Income") + attributes["income2nk"] = income_known(unsafe_string_as_integer(xml_doc, "P2IncKnown")) + attributes["savings"] = safe_string_as_integer(xml_doc, "Q3Savings") + attributes["savingsnk"] = savings_known(xml_doc) + attributes["prevown"] = unsafe_string_as_integer(xml_doc, "Q4PrevOwnedProperty") + attributes["mortgage"] = safe_string_as_decimal(xml_doc, "CALCMORT") + attributes["inc2mort"] = unsafe_string_as_integer(xml_doc, "Q2Person2MortApplication") + attributes["hb"] = unsafe_string_as_integer(xml_doc, "Q2a") + attributes["frombeds"] = safe_string_as_integer(xml_doc, "Q20Bedrooms") + attributes["staircase"] = unsafe_string_as_integer(xml_doc, "Q17aStaircase") + attributes["stairbought"] = safe_string_as_integer(xml_doc, "PercentBought") + attributes["stairowned"] = safe_string_as_integer(xml_doc, "PercentOwns") if attributes["staircase"] == 1 + attributes["mrent"] = safe_string_as_decimal(xml_doc, "Q28MonthlyRent") + attributes["exdate"] = compose_date(xml_doc, "EXDAY", "EXMONTH", "EXYEAR") + attributes["exday"] = safe_string_as_integer(xml_doc, "EXDAY") + attributes["exmonth"] = safe_string_as_integer(xml_doc, "EXMONTH") + attributes["exyear"] = safe_string_as_integer(xml_doc, "EXYEAR") + attributes["resale"] = unsafe_string_as_integer(xml_doc, "Q17Resale") + attributes["deposit"] = deposit(xml_doc, attributes) + attributes["cashdis"] = safe_string_as_decimal(xml_doc, "Q27SocialHomeBuy") + attributes["disabled"] = unsafe_string_as_integer(xml_doc, "Disability") + attributes["lanomagr"] = unsafe_string_as_integer(xml_doc, "Q19Rehoused") + attributes["value"] = purchase_price(xml_doc, attributes) + attributes["equity"] = safe_string_as_decimal(xml_doc, "Q23Equity") + attributes["discount"] = safe_string_as_decimal(xml_doc, "Q33Discount") + attributes["grant"] = safe_string_as_decimal(xml_doc, "Q32Reductions") + attributes["pregyrha"] = 1 if string_or_nil(xml_doc, "PREGYRHA") == "Yes" + attributes["pregla"] = 1 if string_or_nil(xml_doc, "PREGLA") == "Yes" + attributes["pregghb"] = 1 if string_or_nil(xml_doc, "PREGHBA") == "Yes" + attributes["pregother"] = 1 if string_or_nil(xml_doc, "PREGOTHER") == "Yes" + attributes["ppostcode_full"] = compose_postcode(xml_doc, "PPOSTC1", "PPOSTC2") + attributes["prevloc"] = string_or_nil(xml_doc, "Q7ONSLACode") + attributes["ppcodenk"] = previous_postcode_known(xml_doc, attributes["ppostcode_full"], attributes["prevloc"]) # Q7UNKNOWNPOSTCODE check mapping + attributes["ppostc1"] = string_or_nil(xml_doc, "PPOSTC1") + attributes["ppostc2"] = string_or_nil(xml_doc, "PPOSTC2") + attributes["previous_la_known"] = nil + attributes["hhregres"] = unsafe_string_as_integer(xml_doc, "ArmedF") + attributes["hhregresstill"] = still_serving(xml_doc) + attributes["proplen"] = safe_string_as_integer(xml_doc, "Q16aProplen2") + attributes["mscharge"] = monthly_charges(xml_doc, attributes) + attributes["mscharge_known"] = 1 if attributes["mscharge"].present? + attributes["prevten"] = unsafe_string_as_integer(xml_doc, "Q6PrevTenure") + attributes["mortgageused"] = unsafe_string_as_integer(xml_doc, "MORTGAGEUSED") + attributes["wchair"] = unsafe_string_as_integer(xml_doc, "Q15Wheelchair") + attributes["armedforcesspouse"] = unsafe_string_as_integer(xml_doc, "ARMEDFORCESSPOUSE") + attributes["hodate"] = compose_date(xml_doc, "HODAY", "HOMONTH", "HOYEAR") + attributes["hoday"] = safe_string_as_integer(xml_doc, "HODAY") + attributes["homonth"] = safe_string_as_integer(xml_doc, "HOMONTH") + attributes["hoyear"] = safe_string_as_integer(xml_doc, "HOYEAR") + attributes["fromprop"] = unsafe_string_as_integer(xml_doc, "Q21PropertyType") + attributes["socprevten"] = unsafe_string_as_integer(xml_doc, "PrevRentType") + attributes["mortgagelender"] = mortgage_lender(xml_doc, attributes) + attributes["mortgagelenderother"] = mortgage_lender_other(xml_doc, attributes) + attributes["mortlen"] = mortgage_length(xml_doc, attributes) + attributes["extrabor"] = borrowing(xml_doc, attributes) + attributes["totadult"] = safe_string_as_integer(xml_doc, "TOTADULT") # would get overridden + attributes["totchild"] = safe_string_as_integer(xml_doc, "TOTCHILD") # would get overridden + attributes["hhtype"] = unsafe_string_as_integer(xml_doc, "HHTYPE") + attributes["pcode1"] = string_or_nil(xml_doc, "PCODE1") + attributes["pcode2"] = string_or_nil(xml_doc, "PCODE2") + attributes["postcode_full"] = compose_postcode(xml_doc, "PCODE1", "PCODE2") + attributes["pcodenk"] = 0 if attributes["postcode_full"].present? # known if given + attributes["soctenant"] = soctenant(attributes) + attributes["ethnic_group2"] = nil # 23/24 variable + attributes["ethnicbuy2"] = nil # 23/24 variable + attributes["prevshared"] = nil # 23/24 variable + attributes["staircasesale"] = nil # 23/24 variable + + # Required for our form invalidated questions (not present in import) + attributes["previous_la_known"] = 1 if attributes["prevloc"].present? && attributes["ppostcode_full"].blank? + attributes["la_known"] = 1 if attributes["la"].present? && attributes["postcode_full"].blank? + + # Sets the log creator + owner_id = meta_field_value(xml_doc, "owner-user-id").strip + if owner_id.present? + user = LegacyUser.find_by(old_user_id: owner_id)&.user + @logger.warn "Missing user! We expected to find a legacy user with old_user_id #{owner_id}" unless user + + attributes["created_by"] = user + end + + set_default_values(attributes) if previous_status.include?("submitted") + sales_log = save_sales_log(attributes, previous_status) + compute_differences(sales_log, attributes) + check_status_completed(sales_log, previous_status) unless @logs_overridden.include?(sales_log.old_id) + end + + def save_sales_log(attributes, previous_status) + sales_log = SalesLog.new(attributes) + begin + sales_log.save! + sales_log + rescue ActiveRecord::RecordNotUnique + legacy_id = attributes["old_id"] + record = SalesLog.find_by(old_id: legacy_id) + @logger.info "Updating sales log #{record.id} with legacy ID #{legacy_id}" + record.update!(attributes) + record + rescue ActiveRecord::RecordInvalid => e + rescue_validation_or_raise(sales_log, attributes, previous_status, e) + end + end + + def rescue_validation_or_raise(sales_log, _attributes, _previous_status, exception) + @logger.error("Log #{sales_log.old_id}: Failed to import") + raise exception + end + + def compute_differences(sales_log, attributes) + differences = [] + attributes.each do |key, value| + sales_log_value = sales_log.send(key.to_sym) + next if fields_not_present_in_softwire_data.include?(key) + + if value != sales_log_value + differences.push("#{key} #{value.inspect} #{sales_log_value.inspect}") + end + end + @logger.warn "Differences found when saving log #{sales_log.old_id}: #{differences}" unless differences.empty? + end + + def fields_not_present_in_softwire_data + %w[created_by + income1_value_check + mortgage_value_check + savings_value_check + deposit_value_check + wheel_value_check + retirement_value_check + extrabor_value_check + deposit_and_mortgage_value_check + shared_ownership_deposit_value_check + grant_value_check + value_value_check + old_persons_shared_ownership_value_check + staircase_bought_value_check + monthly_charges_value_check + hodate_check + saledate_check] + end + + def check_status_completed(sales_log, previous_status) + if previous_status.include?("submitted") && sales_log.status != "completed" + @logger.warn "sales log #{sales_log.id} is not completed. The following answers are missing: #{missing_answers(sales_log).join(', ')}" + @logger.warn "sales log with old id:#{sales_log.old_id} is incomplete but status should be complete" + @logs_with_discrepancies << sales_log.old_id + end + end + + def age_known(_xml_doc, index, hhmemb, age) + return nil if hhmemb.present? && index > hhmemb + + return 0 if age.present? + end + + def details_known(index, attributes) + return nil if attributes["hhmemb"].nil? || index > attributes["hhmemb"] + return nil if attributes["jointpur"] == 1 && index == 2 + + if attributes["age#{index}_known"] != 0 && + attributes["sex#{index}"] == "R" && + attributes["relat#{index}"] == "R" && + attributes["ecstat#{index}"] == 10 + 2 # No + else + 1 # Yes + end + end + + MORTGAGE_LENDER_OPTIONS = { + "atom bank" => 1, + "barclays bank plc" => 2, + "bath building society" => 3, + "buckinghamshire building society" => 4, + "cambridge building society" => 5, + "coventry building society" => 6, + "cumberland building society" => 7, + "darlington building society" => 8, + "dudley building society" => 9, + "ecology building society" => 10, + "halifax" => 11, + "hanley economic building society" => 12, + "hinckley and rugby building society" => 13, + "holmesdale building society" => 14, + "ipswich building society" => 15, + "leeds building society" => 16, + "lloyds bank" => 17, + "mansfield building society" => 18, + "market harborough building society" => 19, + "melton mowbray building society" => 20, + "nationwide building society" => 21, + "natwest" => 22, + "nedbank private wealth" => 23, + "newbury building society" => 24, + "oneSavings bank" => 25, + "parity trust" => 26, + "penrith building society" => 27, + "pepper homeloans" => 28, + "royal bank of scotland" => 29, + "santander" => 30, + "skipton building society" => 31, + "teachers building society" => 32, + "the co-operative bank" => 33, + "tipton & coseley building society" => 34, + "tss" => 35, + "ulster bank" => 36, + "virgin money" => 37, + "west bromwich building society" => 38, + "yorkshire building society" => 39, + "other" => 40, + }.freeze + + # this comes through as a string, need to map to a corresponding integer + def mortgage_lender(xml_doc, attributes) + lender = case attributes["ownershipsch"] + when 1 + string_or_nil(xml_doc, "Q24aMortgageLender") + when 2 + string_or_nil(xml_doc, "Q34a") + when 3 + string_or_nil(xml_doc, "Q41aMortgageLender") + end + return if lender.blank? + + MORTGAGE_LENDER_OPTIONS[lender.downcase] || MORTGAGE_LENDER_OPTIONS["other"] + end + + def mortgage_lender_other(xml_doc, attributes) + return unless attributes["mortgagelender"] == MORTGAGE_LENDER_OPTIONS["other"] + + case attributes["ownershipsch"] + when 1 + string_or_nil(xml_doc, "Q24aMortgageLender") + when 2 + string_or_nil(xml_doc, "Q34a") + when 3 + string_or_nil(xml_doc, "Q41aMortgageLender") + end + end + + def mortgage_length(xml_doc, attributes) + case attributes["ownershipsch"] + when 1 + unsafe_string_as_integer(xml_doc, "Q24b") + when 2 + unsafe_string_as_integer(xml_doc, "Q34b") + when 3 + unsafe_string_as_integer(xml_doc, "Q41b") + end + end + + def savings_known(xml_doc) + case unsafe_string_as_integer(xml_doc, "savingsKnown") + when 1 # known + 0 + when 2 # unknown + 1 + end + end + + def soctenant(attributes) + return nil unless attributes["ownershipsch"] == 1 + + if attributes["frombeds"].blank? && attributes["fromprop"].blank? && attributes["socprevten"].blank? + 2 + else + 1 + end + # NO (2) if FROMBEDS, FROMPROP and socprevten are blank, and YES(1) if they are completed + end + + def still_serving(xml_doc) + case unsafe_string_as_integer(xml_doc, "LeftArmedF") + when 4 + 4 + when 5, 6 + 5 + end + end + + def income_known(value) + case value + when 1 # known + 0 + when 2 # unknown + 1 + end + end + + def borrowing(xml_doc, attributes) + case attributes["ownershipsch"] + when 1 + unsafe_string_as_integer(xml_doc, "Q25Borrowing") + when 2 + unsafe_string_as_integer(xml_doc, "Q35Borrowing") + when 3 + unsafe_string_as_integer(xml_doc, "Q42Borrowing") + end + end + + def purchase_price(xml_doc, attributes) + case attributes["ownershipsch"] + when 1 + safe_string_as_decimal(xml_doc, "Q22PurchasePrice") + when 2 + safe_string_as_decimal(xml_doc, "Q31PurchasePrice") + when 3 + safe_string_as_decimal(xml_doc, "Q40PurchasePrice") + end + end + + def deposit(xml_doc, attributes) + case attributes["ownershipsch"] + when 1 + safe_string_as_decimal(xml_doc, "Q26CashDeposit") + when 2 + safe_string_as_decimal(xml_doc, "Q36CashDeposit") + when 3 + safe_string_as_decimal(xml_doc, "Q43CashDeposit") + end + end + + def monthly_charges(xml_doc, attributes) + safe_string_as_decimal(xml_doc, "Q29MonthlyCharges") + case attributes["ownershipsch"] + when 1 + safe_string_as_decimal(xml_doc, "Q29MonthlyCharges") + when 2 + safe_string_as_decimal(xml_doc, "Q37MonthlyCharges") + end + end + + def set_default_values(attributes) + attributes["mscharge_known"] ||= 0 if attributes["ownershipsch"] == 3 + attributes["mscharge"] ||= 0 if attributes["ownershipsch"] == 3 + attributes["armedforcesspouse"] ||= 7 + attributes["hhregres"] ||= 8 + attributes["disabled"] ||= 3 + attributes["wheel"] ||= 3 + attributes["hb"] ||= 4 + attributes["prevown"] ||= 3 + attributes["savingsnk"] ||= attributes["savings"].present? ? 0 : 1 + # attributes["noint"] = 1 # not interviewed + + # buyer 1 characteristics + attributes["age1_known"] ||= 1 + attributes["sex1"] ||= "R" + attributes["ethnic_group"] ||= 17 + attributes["ethnic"] ||= 17 + attributes["national"] ||= 13 + attributes["ecstat1"] ||= 10 + attributes["income1nk"] ||= attributes["income1"].present? ? 0 : 1 + attributes["hholdcount"] ||= default_household_count(attributes) # just for testing, might need to change + + # buyer 2 characteristics + if attributes["jointpur"] == 1 + attributes["age2_known"] ||= 1 + attributes["sex2"] ||= "R" + attributes["ecstat2"] ||= 10 + attributes["income2nk"] ||= attributes["income2"].present? ? 0 : 1 + end + + # other household members characteristics + (2..attributes["hhmemb"]).each do |index| + attributes["age#{index}_known"] ||= 1 + attributes["sex#{index}"] ||= "R" + attributes["ecstat#{index}"] ||= 10 + attributes["relat#{index}"] ||= "R" + end + end + + def missing_answers(sales_log) + applicable_questions = sales_log.form.subsections.map { |s| s.applicable_questions(sales_log) }.flatten + applicable_questions.filter { |q| q.unanswered?(sales_log) }.map(&:id) + end + + # just for testing, logic might need to change + def default_household_count(attributes) + return 0 if attributes["hhmemb"].zero? || attributes["hhmemb"].blank? + + attributes["jointpur"] == 1 ? attributes["hhmemb"] - 2 : attributes["hhmemb"] - 1 + end + end +end diff --git a/db/migrate/20230215112932_add_old_id_to_sales_logs.rb b/db/migrate/20230215112932_add_old_id_to_sales_logs.rb new file mode 100644 index 000000000..cf61a968c --- /dev/null +++ b/db/migrate/20230215112932_add_old_id_to_sales_logs.rb @@ -0,0 +1,8 @@ +class AddOldIdToSalesLogs < ActiveRecord::Migration[7.0] + def change + change_table :sales_logs, bulk: true do |t| + t.column :old_id, :string + end + add_index :sales_logs, :old_id, unique: true + end +end diff --git a/db/schema.rb b/db/schema.rb index 1747fa6c5..d1668b738 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2023_02_13_140932) do +ActiveRecord::Schema[7.0].define(version: 2023_02_15_112932) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -522,16 +522,18 @@ ActiveRecord::Schema[7.0].define(version: 2023_02_13_140932) do t.integer "old_persons_shared_ownership_value_check" t.integer "staircase_bought_value_check" t.integer "monthly_charges_value_check" + t.integer "saledate_check" t.integer "details_known_5" t.integer "details_known_6" - t.integer "saledate_check" - t.integer "prevshared" - t.integer "staircasesale" t.integer "ethnic_group2" t.integer "ethnicbuy2" t.integer "proplen_asked" + t.integer "prevshared" + t.integer "staircasesale" + t.string "old_id" t.index ["bulk_upload_id"], name: "index_sales_logs_on_bulk_upload_id" t.index ["created_by_id"], name: "index_sales_logs_on_created_by_id" + t.index ["old_id"], name: "index_sales_logs_on_old_id", unique: true t.index ["owning_organisation_id"], name: "index_sales_logs_on_owning_organisation_id" t.index ["updated_by_id"], name: "index_sales_logs_on_updated_by_id" end diff --git a/lib/tasks/data_import.rake b/lib/tasks/data_import.rake index 0b3881388..0dede82a3 100644 --- a/lib/tasks/data_import.rake +++ b/lib/tasks/data_import.rake @@ -22,6 +22,8 @@ namespace :core do Imports::OrganisationRentPeriodImportService.new(storage_service).create_organisation_rent_periods(path) when "lettings-logs" Imports::LettingsLogsImportService.new(storage_service).create_logs(path) + when "sales-logs" + Imports::SalesLogsImportService.new(storage_service).create_logs(path) else raise "Type #{type} is not supported by data_import" end diff --git a/spec/fixtures/imports/sales_logs/discounted_ownership_sales_log.xml b/spec/fixtures/imports/sales_logs/discounted_ownership_sales_log.xml new file mode 100644 index 000000000..575f93887 --- /dev/null +++ b/spec/fixtures/imports/sales_logs/discounted_ownership_sales_log.xml @@ -0,0 +1,351 @@ + + + 2022-CORE-Sales + discounted_ownership_sales_log + c3061a2e6ea0b702e6f6210d5c52d2a92612d2aa + 7c5bd5fb549c09z2c55d9cb90d7ba84927e64618 + 7c5bd5fb549c09z2c55d9cb90d7ba84927e64618 + 2023-02-21T11:54:51.786722Z + 2023-02-22T10:59:45.88188Z + submitted-valid + 2022 + Manual Entry + + + + + + + + 2023-02-01 + Discount ownership example + 2 Yes - a discount ownership scheme + + 14 Preserved Right to Buy (PRTB) + + + + + 2 No + + 1 No + + + 3 + 3 House + 1 Purpose built + SW1A 1AA + + + Cheltenham + E07000078 + 3 Don’t know + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 Yes + + + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 134750 + 1 + 0 + 0 + 0 + 1 +
300203
+ + + + + + + + + + + + + + + + + + + + + + + + + 0 + 1 + 0 + 1 + 0 + 0 + 0 + 1 + 2 + 2023 + + + + + + + GL51 + 9EX + + + GL51 + 9EX + 1 + + + + + 3 Private tenant + GL51 9EX + + Cheltenham + E07000078 + + + Yes + + + + + + + + + + + + + + + + 1 Yes + + + + + + + + + + + + + + 1 + + + + + + + + + + + + + + + + + + + + + + + + + 275000 + + 51 + 1 Yes + 134750 + Halifax + + + 33 + 2 No + 0 + 0.00 + + + + + + + + + + + + 0 + 0 + 0 + + + 9 = other + + + + + 14 Preserved Right to Buy (PRTB) + + + 1 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 1 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + + + E12000007 + 1 + 1 Test + 655 + + diff --git a/spec/fixtures/imports/sales_logs/outright_sale_sales_log.xml b/spec/fixtures/imports/sales_logs/outright_sale_sales_log.xml new file mode 100644 index 000000000..4bbdc9981 --- /dev/null +++ b/spec/fixtures/imports/sales_logs/outright_sale_sales_log.xml @@ -0,0 +1,333 @@ + + + 2022-CORE-Sales + outright_sale_sales_log + c3061a2e6ea0b702e6f6210d5c52d2a92612d2aa + 7c5bd5fb549c09a2c55d7cb90d7ba84927e64618 + 7c5bd5fb549c09a2c55d7cb90d7ba84927e64618 + 2023-02-21T12:09:45.809134Z + 2023-02-22T10:59:01.709949Z + submitted-valid + 2022 + Manual Entry + + + + + Yes + 2023-01-16 + Outright ownership example + 3 No - this is an outright or other sale + + + 10 Outright + + 2 No + 1 Yes + 2 No + + 2 Yes + + + 1 + 1 Flat or maisonette + 1 Purpose built + SW1A 1AA + Westminster + E09000033 + 2 No + + + 75 + Female + 5 Retired + 1 White: English/Scottish/Welsh/Northern Irish/British + 18 United Kingdom + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 Yes + + + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 2 + 0 + 0 + 0 + 0 +
300202
+ + + + + + + B + + + + B + + + + + + + + + + + + + + 0 + 0 + 1 + 1 + 0 + 0 + 0 + 16 + 1 + 2023 + + + + + + + B95 + 5HZ + SW1A + 1AA + + + + 3 Private tenant + B95 5HZ + + Stratford-on-Avon + E07000221 + + + Yes + + + + + 7 No + + + 2 No + 2 No + + + 2 No + + + + 2 No + + + + 4 Don’t know + + + 2 No + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 300000 + 2 No + + + + + 300000 + + + 1 + 1 + 0 + + + 1 = 1 elder + + + 10 Outright + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 1 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 1 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 1 + 0 + 0 + 1 + + + + + E12000007 + 1 + 1 Test + 655 + + diff --git a/spec/fixtures/imports/sales_logs/shared_ownership_sales_log.xml b/spec/fixtures/imports/sales_logs/shared_ownership_sales_log.xml new file mode 100644 index 000000000..6e0c11174 --- /dev/null +++ b/spec/fixtures/imports/sales_logs/shared_ownership_sales_log.xml @@ -0,0 +1,333 @@ + + + 2022-CORE-Sales + shared_ownership_sales_log + c3061a2e6ea0b702e6f6210d5c52d2a92612d2aa + 7c5bd5fb549c09a2c55d7cb90d7ba84927e64618 + 7c5bd5fb549c09a2c55d7cb90d7ba84927e64618 + 2023-02-21T11:48:28.255968Z + 2023-02-22T11:00:06.575832Z + submitted-valid + 2022 + Manual Entry + + + + + Yes + 2023-01-17 + Shared ownership example + 1 Yes - a shared ownership scheme + 2 Shared Ownership + + + + 2 No + 1 Yes + 2 No + + 2 Yes + + + 2 + 1 Flat or maisonette + 1 Purpose built + SW1A 1AA + Westminster + E09000033 + 3 Don’t know + + + 30 + Male + 1 Full Time - 30 hours or more a week + 2 White: Irish + 18 United Kingdom + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 Yes + + + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 76000 + 1 + 47000 + 0 + 235000 + 0 +
300204
+ + + + + + + + + + + + + + + + + + + + + + + + + 1 + 0 + 0 + 1 + 1 + 2 + 3 + 17 + 1 + 2023 + 6 + 9 + 2022 + 8 + 1 + 2023 + SW14 + 7QP + SW1A + 1AA + + + + 2 Private registered provider (PRP) or housing association tenant + SW14 7QP + + Richmond-upon-Thames + E09000027 + Yes + + + + + + + 8 Don’t know + + + 2 No + 2 No + + + 1 Yes + 47000 + 1 Yes + + + + 4 Don’t know + 1 Yes + 89000 + 1 Yes + + + + + 1 + 2 No + + 30 + 2 No + 2023-01-08 + 2022-09-06 + 2 No + + + + 550000 + 30 + 1 Yes + 76000 + Nationwide + 33 + 2 No + 89000 + + 912.00 + 134.24 + + + + + + + + + + + + + + + + + + + + + + + + + 1 + 1 + 0 + + + 3 = 1 adult + + + 2 Shared Ownership + + + 1 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 1 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 1 + 0 + 0 + 0 + 0 + 1 + + + + + E12000007 + 1 + 1 Test + 655 + + diff --git a/spec/fixtures/imports/sales_logs/shared_ownership_sales_log2.xml b/spec/fixtures/imports/sales_logs/shared_ownership_sales_log2.xml new file mode 100644 index 000000000..6334674a2 --- /dev/null +++ b/spec/fixtures/imports/sales_logs/shared_ownership_sales_log2.xml @@ -0,0 +1,333 @@ + + + 2022-CORE-Sales + shared_ownership_sales_log2 + c3061a2e6ea0b702e6f6210d5c52d2a92612d2aa + 7c5bd5fb549c09a2c55d7cb90d7ba84927e64618 + 7c5bd5fb549c09a2c55d7cb90d7ba84927e64618 + 2023-02-21T11:48:28.255968Z + 2023-02-22T11:00:06.575832Z + submitted-valid + 2022 + Manual Entry + + + + + Yes + 2023-01-17 + Shared ownership example + 1 Yes - a shared ownership scheme + 2 Shared Ownership + + + + 2 No + 1 Yes + 2 No + + 2 Yes + + + 2 + 1 Flat or maisonette + 1 Purpose built + SW1A 1AA + Westminster + E09000033 + 3 Don’t know + + + 30 + Male + 1 Full Time - 30 hours or more a week + 2 White: Irish + 18 United Kingdom + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 Yes + + + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 76000 + 1 + 47000 + 0 + 235000 + 0 +
300204
+ + + + + + + + + + + + + + + + + + + + + + + + + 1 + 0 + 0 + 1 + 1 + 2 + 3 + 17 + 1 + 2023 + 6 + 9 + 2022 + 8 + 1 + 2023 + SW14 + 7QP + SW1A + 1AA + + + + 2 Private registered provider (PRP) or housing association tenant + SW14 7QP + + Richmond-upon-Thames + E09000027 + Yes + + + + + + + 8 Don’t know + + + 2 No + 2 No + + + 1 Yes + 47000 + 1 Yes + + + + 4 Don’t know + 1 Yes + 89000 + 1 Yes + + + + + 1 + 2 No + + 30 + 2 No + 2023-01-08 + 2022-09-06 + 2 No + + + + 550000 + 30 + 1 Yes + 76000 + Nationwide + 33 + 2 No + 89000 + + 912.00 + 134.24 + + + + + + + + + + + + + + + + + + + + + + + + + 1 + 1 + 0 + + + 3 = 1 adult + + + 2 Shared Ownership + + + 1 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 1 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 1 + 0 + 0 + 0 + 0 + 1 + + + + + E12000007 + 1 + 1 Test + 655 + + diff --git a/spec/fixtures/imports/sales_logs/shared_ownership_sales_log3.xml b/spec/fixtures/imports/sales_logs/shared_ownership_sales_log3.xml new file mode 100644 index 000000000..b5dabde97 --- /dev/null +++ b/spec/fixtures/imports/sales_logs/shared_ownership_sales_log3.xml @@ -0,0 +1,333 @@ + + + 2022-CORE-Sales + shared_ownership_sales_log3 + c3061a2e6ea0b702e6f6210d5c52d2a92612d2aa + 7c5bd5fb549c09a2c55d7cb90d7ba84927e64618 + 7c5bd5fb549c09a2c55d7cb90d7ba84927e64618 + 2023-02-21T11:48:28.255968Z + 2023-02-22T11:00:06.575832Z + submitted-valid + 2022 + Manual Entry + + + + + Yes + 2023-01-17 + Shared ownership example + 1 Yes - a shared ownership scheme + 2 Shared Ownership + + + + 2 No + 1 Yes + 2 No + + 2 Yes + + + 2 + 1 Flat or maisonette + 1 Purpose built + SW1A 1AA + Westminster + E09000033 + 3 Don’t know + + + 30 + Male + 1 Full Time - 30 hours or more a week + 2 White: Irish + 18 United Kingdom + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 Yes + + + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 76000 + 1 + 47000 + 0 + 235000 + 0 +
300204
+ + + + + + + + + + + + + + + + + + + + + + + + + 1 + 0 + 0 + 1 + 1 + 2 + 3 + 17 + 1 + 2023 + 6 + 9 + 2022 + 8 + 1 + 2023 + SW14 + 7QP + SW1A + 1AA + + + + 2 Private registered provider (PRP) or housing association tenant + SW14 7QP + + Richmond-upon-Thames + E09000027 + Yes + + + + + + + 8 Don’t know + + + 2 No + 2 No + + + 1 Yes + 47000 + + + + + 4 Don’t know + 1 Yes + 89000 + 1 Yes + + + + + 1 + 2 No + + 30 + 2 No + 2023-01-08 + 2022-09-06 + 2 No + + + + 550000 + 30 + 1 Yes + 76000 + Nationwide + 33 + 2 No + 89000 + + 912.00 + 134.24 + + + + + + + + + + + + + + + + + + + + + + + + + 1 + 1 + 0 + + + 3 = 1 adult + + + 2 Shared Ownership + + + 1 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 1 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 1 + 0 + 0 + 0 + 0 + 1 + + + + + E12000007 + 1 + 1 Test + 655 + + diff --git a/spec/fixtures/imports/sales_logs/shared_ownership_sales_log4.xml b/spec/fixtures/imports/sales_logs/shared_ownership_sales_log4.xml new file mode 100644 index 000000000..98342cf3c --- /dev/null +++ b/spec/fixtures/imports/sales_logs/shared_ownership_sales_log4.xml @@ -0,0 +1,333 @@ + + + 2022-CORE-Sales + shared_ownership_sales_log4 + e29c492473446dca4d50224f2bb7cf965a261d6f + 7c5bd5fb549c09a2c55d7cb90d7ba84927e64618 + 7c5bd5fb549c09a2c55d7cb90d7ba84927e64618 + 2023-02-21T11:48:28.255968Z + 2023-02-22T11:00:06.575832Z + submitted-valid + 2022 + Manual Entry + + + + + Yes + 2023-01-17 + Shared ownership example + 1 Yes - a shared ownership scheme + 2 Shared Ownership + + + + 2 No + 1 Yes + 2 No + + 2 Yes + + + 2 + 1 Flat or maisonette + 1 Purpose built + SW1A 1AA + Westminster + E09000033 + 3 Don’t know + + + 30 + Male + 1 Full Time - 30 hours or more a week + 2 White: Irish + 18 United Kingdom + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 Yes + + + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 76000 + 1 + 47000 + 0 + 235000 + 0 +
300204
+ + + + + + + + + + + + + + + + + + + + + + + + + 1 + 0 + 0 + 1 + 1 + 2 + 3 + 17 + 1 + 2023 + 6 + 9 + 2022 + 8 + 1 + 2023 + SW14 + 7QP + SW1A + 1AA + + + + 2 Private registered provider (PRP) or housing association tenant + SW14 7QP + + Richmond-upon-Thames + E09000027 + Yes + + + + + + + 8 Don’t know + + + 2 No + 2 No + + + 1 Yes + 47000 + 1 Yes + + + + 4 Don’t know + 1 Yes + 89000 + 1 Yes + + + + + 1 + 2 No + + 30 + 2 No + 2023-01-08 + 2022-09-06 + 2 No + + + + 550000 + 30 + 1 Yes + 76000 + Nationwide + 33 + 2 No + 89000 + + 912.00 + + + + + + + + + + + + + + + + + + + + + + + + + + 1 + 1 + 0 + + + 3 = 1 adult + + + 2 Shared Ownership + + + 1 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 1 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 1 + 0 + 0 + 0 + 0 + 1 + + + + + E12000007 + 1 + 1 Test + 655 + + diff --git a/spec/lib/tasks/data_import_spec.rb b/spec/lib/tasks/data_import_spec.rb index ad1887d16..462f2740e 100644 --- a/spec/lib/tasks/data_import_spec.rb +++ b/spec/lib/tasks/data_import_spec.rb @@ -109,6 +109,24 @@ describe "rake core:data_import", type: :task do end end + context "when importing sales logs" do + let(:type) { "sales-logs" } + let(:import_service) { instance_double(Imports::SalesLogsImportService) } + let(:fixture_path) { "spec/fixtures/imports/sales_logs" } + + before do + allow(Imports::SalesLogsImportService).to receive(:new).and_return(import_service) + end + + it "creates sales logs from the given XML file" do + expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name) + expect(Imports::SalesLogsImportService).to receive(:new).with(storage_service) + expect(import_service).to receive(:create_logs).with(fixture_path) + + task.invoke(type, fixture_path) + end + end + context "when importing scheme data" do let(:type) { "scheme" } let(:import_service) { instance_double(Imports::SchemeImportService) } diff --git a/spec/services/imports/sales_logs_import_service_spec.rb b/spec/services/imports/sales_logs_import_service_spec.rb new file mode 100644 index 000000000..b556d3e68 --- /dev/null +++ b/spec/services/imports/sales_logs_import_service_spec.rb @@ -0,0 +1,595 @@ +require "rails_helper" + +RSpec.describe Imports::SalesLogsImportService do + subject(:sales_log_service) { described_class.new(storage_service, logger) } + + let(:storage_service) { instance_double(Storage::S3Service) } + let(:logger) { instance_double(ActiveSupport::Logger) } + + let(:fixture_directory) { "spec/fixtures/imports/sales_logs" } + + let(:organisation) { FactoryBot.create(:organisation, old_visible_id: "1", provider_type: "PRP") } + let(:managing_organisation) { FactoryBot.create(:organisation, old_visible_id: "2", provider_type: "PRP") } + let(:remote_folder) { "sales_logs" } + + def open_file(directory, filename) + File.open("#{directory}/#{filename}.xml") + end + + before do + { "GL519EX" => "E07000078", + "SW1A2AA" => "E09000033", + "SW1A1AA" => "E09000033", + "SW147QP" => "E09000027", + "B955HZ" => "E07000221" }.each do |postcode, district_code| + WebMock.stub_request(:get, /api.postcodes.io\/postcodes\/#{postcode}/).to_return(status: 200, body: "{\"status\":200,\"result\":{\"admin_district\":\"#{district_code}\",\"codes\":{\"admin_district\":\"#{district_code}\"}}}", headers: {}) + end + + allow(Organisation).to receive(:find_by).and_return(nil) + allow(Organisation).to receive(:find_by).with(old_visible_id: organisation.old_visible_id).and_return(organisation) + allow(Organisation).to receive(:find_by).with(old_visible_id: managing_organisation.old_visible_id).and_return(managing_organisation) + + # Created by users + FactoryBot.create(:user, old_user_id: "c3061a2e6ea0b702e6f6210d5c52d2a92612d2aa", organisation:) + FactoryBot.create(:user, old_user_id: "e29c492473446dca4d50224f2bb7cf965a261d6f", organisation:) + end + + context "when importing sales logs" do + before do + # Stub the S3 file listing and download + allow(storage_service).to receive(:list_files) + .and_return(%W[#{remote_folder}/shared_ownership_sales_log.xml #{remote_folder}/shared_ownership_sales_log2.xml #{remote_folder}/outright_sale_sales_log.xml #{remote_folder}/discounted_ownership_sales_log.xml]) + allow(storage_service).to receive(:get_file_io) + .with("#{remote_folder}/shared_ownership_sales_log.xml") + .and_return(open_file(fixture_directory, "shared_ownership_sales_log"), open_file(fixture_directory, "shared_ownership_sales_log")) + allow(storage_service).to receive(:get_file_io) + .with("#{remote_folder}/shared_ownership_sales_log2.xml") + .and_return(open_file(fixture_directory, "shared_ownership_sales_log2"), open_file(fixture_directory, "shared_ownership_sales_log2")) + allow(storage_service).to receive(:get_file_io) + .with("#{remote_folder}/outright_sale_sales_log.xml") + .and_return(open_file(fixture_directory, "outright_sale_sales_log"), open_file(fixture_directory, "outright_sale_sales_log")) + allow(storage_service).to receive(:get_file_io) + .with("#{remote_folder}/discounted_ownership_sales_log.xml") + .and_return(open_file(fixture_directory, "discounted_ownership_sales_log"), open_file(fixture_directory, "discounted_ownership_sales_log")) + end + + it "successfully creates all sales logs" do + expect(logger).not_to receive(:error) + expect(logger).not_to receive(:warn) + expect(logger).not_to receive(:info) + expect { sales_log_service.create_logs(remote_folder) } + .to change(SalesLog, :count).by(4) + end + + it "only updates existing sales logs" do + expect(logger).not_to receive(:error) + expect(logger).not_to receive(:warn) + expect(logger).to receive(:info).with(/Updating sales log/).exactly(4).times + expect { 2.times { sales_log_service.create_logs(remote_folder) } } + .to change(SalesLog, :count).by(4) + end + + context "when there are status discrepancies" do + let(:sales_log_file) { open_file(fixture_directory, "shared_ownership_sales_log3") } + let(:sales_log_xml) { Nokogiri::XML(sales_log_file) } + + before do + allow(storage_service).to receive(:get_file_io) + .with("#{remote_folder}/shared_ownership_sales_log3.xml") + .and_return(open_file(fixture_directory, "shared_ownership_sales_log3"), open_file(fixture_directory, "shared_ownership_sales_log3")) + allow(storage_service).to receive(:get_file_io) + .with("#{remote_folder}/shared_ownership_sales_log4.xml") + .and_return(open_file(fixture_directory, "shared_ownership_sales_log4"), open_file(fixture_directory, "shared_ownership_sales_log4")) + end + + it "the logger logs a warning with the sales log's old id/filename" do + expect(logger).to receive(:warn).with(/is not completed/).once + expect(logger).to receive(:warn).with(/sales log with old id:shared_ownership_sales_log3 is incomplete but status should be complete/).once + + sales_log_service.send(:create_log, sales_log_xml) + end + + it "on completion the ids of all logs with status discrepancies are logged in a warning" do + allow(storage_service).to receive(:list_files) + .and_return(%W[#{remote_folder}/shared_ownership_sales_log3.xml #{remote_folder}/shared_ownership_sales_log4.xml]) + expect(logger).to receive(:warn).with(/is not completed/).twice + expect(logger).to receive(:warn).with(/is incomplete but status should be complete/).twice + expect(logger).to receive(:warn).with(/The following sales logs had status discrepancies: \[shared_ownership_sales_log3, shared_ownership_sales_log4\]/) + + sales_log_service.create_logs(remote_folder) + end + end + end + + context "when importing a specific log" do + let(:sales_log_file) { open_file(fixture_directory, sales_log_id) } + let(:sales_log_xml) { Nokogiri::XML(sales_log_file) } + + context "and the organisation legacy ID does not exist" do + let(:sales_log_id) { "shared_ownership_sales_log" } + + before { sales_log_xml.at_xpath("//xmlns:OWNINGORGID").content = 99_999 } + + it "raises an exception" do + expect { sales_log_service.send(:create_log, sales_log_xml) } + .to raise_error(RuntimeError, "Organisation not found with legacy ID 99999") + end + end + + context "when the mortgage lender is set to an existing option" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + before do + sales_log_xml.at_xpath("//xmlns:Q34a").content = "halifax" + allow(logger).to receive(:warn).and_return(nil) + end + + it "correctly sets mortgage lender" do + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.mortgagelender).to be(11) + end + end + + context "when the mortgage lender is set to a non existing option" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + before do + sales_log_xml.at_xpath("//xmlns:Q34a").content = "something else" + allow(logger).to receive(:warn).and_return(nil) + end + + it "correctly sets mortgage lender and mortgage lender other" do + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.mortgagelender).to be(40) + expect(sales_log&.mortgagelenderother).to eq("something else") + end + end + + context "with shared ownership type" do + let(:sales_log_id) { "shared_ownership_sales_log" } + + it "successfully creates a completed shared ownership log" do + expect(logger).not_to receive(:error) + expect(logger).not_to receive(:warn) + expect(logger).not_to receive(:info) + expect { sales_log_service.send(:create_log, sales_log_xml) } + .to change(SalesLog, :count).by(1) + end + end + + context "with discounted ownership type" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + it "successfully creates a completed discounted ownership log" do + expect(logger).not_to receive(:error) + expect(logger).not_to receive(:warn) + expect(logger).not_to receive(:info) + expect { sales_log_service.send(:create_log, sales_log_xml) } + .to change(SalesLog, :count).by(1) + end + end + + context "with outright sale type" do + let(:sales_log_id) { "outright_sale_sales_log" } + + it "successfully creates a completed outright sale log" do + expect(logger).not_to receive(:error) + expect(logger).not_to receive(:warn) + expect(logger).not_to receive(:info) + expect { sales_log_service.send(:create_log, sales_log_xml) } + .to change(SalesLog, :count).by(1) + end + end + + context "when inferring default answers for completed sales logs" do + context "when the armedforcesspouse is not answered" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + before do + sales_log_xml.at_xpath("//xmlns:ARMEDFORCESSPOUSE").content = "" + allow(logger).to receive(:warn).and_return(nil) + end + + it "sets armedforcesspouse to don't know" do + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.armedforcesspouse).to be(7) + end + end + + context "when the savings not known is not answered and savings is not given" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + before do + sales_log_xml.at_xpath("//xmlns:savingsKnown").content = "" + allow(logger).to receive(:warn).and_return(nil) + end + + it "sets savingsnk to not know" do + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.savingsnk).to be(1) + end + end + + context "when the savings not known is not answered and savings is given" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + before do + sales_log_xml.at_xpath("//xmlns:Q3Savings").content = "10000" + sales_log_xml.at_xpath("//xmlns:savingsKnown").content = "" + allow(logger).to receive(:warn).and_return(nil) + end + + it "sets savingsnk to know" do + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.savingsnk).to be(0) + end + end + + context "and it's an outright sale" do + let(:sales_log_id) { "outright_sale_sales_log" } + + before do + allow(logger).to receive(:warn).and_return(nil) + end + + it "infers mscharge_known as no" do + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log.mscharge_known).to eq(0) + end + end + + context "when inferring age known" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + before do + sales_log_xml.at_xpath("//xmlns:HHMEMB").content = "3" + sales_log_xml.at_xpath("//xmlns:P1Age").content = "" + sales_log_xml.at_xpath("//xmlns:P2Age").content = "" + sales_log_xml.at_xpath("//xmlns:P3Age").content = "22" + allow(logger).to receive(:warn).and_return(nil) + + sales_log_service.send(:create_log, sales_log_xml) + end + + it "sets age known to no if age not answered" do + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.age1_known).to be(1) # unknown + expect(sales_log&.age2_known).to be(1) # unknown + end + + it "sets age known to yes if age answered" do + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.age3_known).to be(0) # known + end + end + + context "when inferring gender" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + before do + sales_log_xml.at_xpath("//xmlns:HHMEMB").content = "3" + sales_log_xml.at_xpath("//xmlns:P1Sex").content = "" + sales_log_xml.at_xpath("//xmlns:P2Sex").content = "" + sales_log_xml.at_xpath("//xmlns:P3Sex").content = "Female" + allow(logger).to receive(:warn).and_return(nil) + + sales_log_service.send(:create_log, sales_log_xml) + end + + it "sets gender to prefers not to say if not answered" do + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.sex1).to eq("R") + expect(sales_log&.sex2).to eq("R") + end + + it "sets the gender correctly if answered" do + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.sex3).to eq("F") + end + end + + context "when inferring ethnic group" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + before do + sales_log_xml.at_xpath("//xmlns:HHMEMB").content = "1" + sales_log_xml.at_xpath("//xmlns:P1Eth").content = "" + allow(logger).to receive(:warn).and_return(nil) + + sales_log_service.send(:create_log, sales_log_xml) + end + + it "sets ethnic group to prefers not to say if not answered" do + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.ethnic_group).to eq(17) + end + end + + context "when inferring nationality" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + before do + sales_log_xml.at_xpath("//xmlns:HHMEMB").content = "1" + sales_log_xml.at_xpath("//xmlns:P1Nat").content = "" + allow(logger).to receive(:warn).and_return(nil) + + sales_log_service.send(:create_log, sales_log_xml) + end + + it "sets nationality to prefers not to say if not answered" do + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.national).to eq(13) + end + end + + context "when inferring economic status" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + before do + sales_log_xml.at_xpath("//xmlns:HHMEMB").content = "3" + sales_log_xml.at_xpath("//xmlns:P1Eco").content = "" + sales_log_xml.at_xpath("//xmlns:P2Eco").content = "" + sales_log_xml.at_xpath("//xmlns:P3Eco").content = "3" + allow(logger).to receive(:warn).and_return(nil) + + sales_log_service.send(:create_log, sales_log_xml) + end + + it "sets economic status to prefers not to say if not answered" do + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.ecstat1).to eq(10) + expect(sales_log&.ecstat2).to eq(10) + end + + it "sets the economic status correctly if answered" do + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.ecstat3).to eq(3) + end + end + + context "when inferring relationship" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + before do + sales_log_xml.at_xpath("//xmlns:HHMEMB").content = "3" + sales_log_xml.at_xpath("//xmlns:P2Rel").content = "" + sales_log_xml.at_xpath("//xmlns:P3Rel").content = "Partner" + allow(logger).to receive(:warn).and_return(nil) + + sales_log_service.send(:create_log, sales_log_xml) + end + + it "sets relationship to prefers not to say if not answered" do + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.relat2).to eq("R") + end + + it "sets the relationship correctly if answered" do + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.relat3).to eq("P") + end + end + + context "when inferring armed forces" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + before do + allow(logger).to receive(:warn).and_return(nil) + end + + it "sets hhregres to don't know if not answered" do + sales_log_xml.at_xpath("//xmlns:ArmedF").content = "" + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.hhregres).to eq(8) + end + + it "sets hhregres correctly if answered" do + sales_log_xml.at_xpath("//xmlns:ArmedF").content = "7 No" + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.hhregres).to eq(7) + end + end + + context "when inferring disability" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + before do + allow(logger).to receive(:warn).and_return(nil) + end + + it "sets disabled to don't know if not answered" do + sales_log_xml.at_xpath("//xmlns:Disability").content = "" + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.disabled).to eq(3) + end + + it "sets disabled correctly if answered" do + sales_log_xml.at_xpath("//xmlns:Disability").content = "2 No" + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.disabled).to eq(2) + end + end + + context "when inferring wheelchair" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + before do + allow(logger).to receive(:warn).and_return(nil) + end + + it "sets wheel to don't know if not answered" do + sales_log_xml.at_xpath("//xmlns:Q10Wheelchair").content = "" + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.wheel).to eq(3) + end + + it "sets wheel correctly if answered" do + sales_log_xml.at_xpath("//xmlns:Q10Wheelchair").content = "2 No" + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.wheel).to eq(2) + end + end + + context "when inferring housing benefit" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + before do + allow(logger).to receive(:warn).and_return(nil) + end + + it "sets hb to don't know if not answered" do + sales_log_xml.at_xpath("//xmlns:Q2a").content = "" + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.hb).to eq(4) + end + + it "sets hb correctly if answered" do + sales_log_xml.at_xpath("//xmlns:Q2a").content = "2 Housing Benefit" + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.hb).to eq(2) + end + end + + context "when inferring income not known" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + before do + sales_log_xml.at_xpath("//xmlns:joint").content = "1 Yes" + sales_log_xml.at_xpath("//xmlns:JointMore").content = "2 No" + allow(logger).to receive(:warn).and_return(nil) + end + + it "sets income to not known if not answered and income is not given" do + sales_log_xml.at_xpath("//xmlns:P1IncKnown").content = "" + sales_log_xml.at_xpath("//xmlns:Q2Person1Income").content = "" + sales_log_xml.at_xpath("//xmlns:P2IncKnown").content = "" + sales_log_xml.at_xpath("//xmlns:Q2Person2Income").content = "" + + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.income1nk).to eq(1) + expect(sales_log&.income2nk).to eq(1) + end + + it "sets income to known if not answered but the income is given" do + sales_log_xml.at_xpath("//xmlns:P1IncKnown").content = "" + sales_log_xml.at_xpath("//xmlns:Q2Person1Income").content = "30000" + sales_log_xml.at_xpath("//xmlns:P2IncKnown").content = "" + sales_log_xml.at_xpath("//xmlns:Q2Person2Income").content = "40000" + + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.income1nk).to eq(0) + expect(sales_log&.income2nk).to eq(0) + end + + it "sets income known correctly if answered" do + sales_log_xml.at_xpath("//xmlns:P1IncKnown").content = "1 Yes" + sales_log_xml.at_xpath("//xmlns:P2IncKnown").content = "2 No" + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.income1nk).to eq(0) + expect(sales_log&.income2nk).to eq(1) + end + end + + context "when inferring prevown" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + before do + allow(logger).to receive(:warn).and_return(nil) + end + + it "sets prevown to don't know if not answered" do + sales_log_xml.at_xpath("//xmlns:Q4PrevOwnedProperty").content = "" + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.prevown).to eq(3) + end + + it "sets prevown correctly if answered" do + sales_log_xml.at_xpath("//xmlns:Q4PrevOwnedProperty").content = "2 No" + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.prevown).to eq(2) + end + end + + context "when inferring household count" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + before do + allow(logger).to receive(:warn).and_return(nil) + end + + it "sets hholdcount to hhmemb - 1 if not answered and not joint purchase" do + sales_log_xml.at_xpath("//xmlns:HHMEMB").content = "3" + sales_log_xml.at_xpath("//xmlns:joint").content = "2 No" + sales_log_xml.at_xpath("//xmlns:LiveInOther").content = "" + + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.hholdcount).to eq(2) + end + + it "sets hholdcount to hhmemb - 2 if not answered and joint purchase" do + sales_log_xml.at_xpath("//xmlns:joint").content = "1 Yes" + sales_log_xml.at_xpath("//xmlns:JointMore").content = "2 No" + sales_log_xml.at_xpath("//xmlns:HHMEMB").content = "3" + sales_log_xml.at_xpath("//xmlns:LiveInOther").content = "" + + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.hholdcount).to eq(1) + end + + it "sets hholdcount to 0 if HHMEMB is 0" do + sales_log_xml.at_xpath("//xmlns:joint").content = "1 Yes" + sales_log_xml.at_xpath("//xmlns:JointMore").content = "2 No" + sales_log_xml.at_xpath("//xmlns:HHMEMB").content = "0" + sales_log_xml.at_xpath("//xmlns:LiveInOther").content = "" + + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.hholdcount).to eq(0) + end + end + end + end +end From 9fc705cc37fa6084c0837290f1643bbea7d794b7 Mon Sep 17 00:00:00 2001 From: Arthur Campbell <51094020+arfacamble@users.noreply.github.com> Date: Mon, 27 Feb 2023 09:31:44 +0000 Subject: [PATCH 2/8] CLDC-1905 infer vacancy reason when renewal (#1321) * add test for derivation of relet when renewal * derive rsnvac when renewal set to yes, reset rsnvac when renewal set to no * add test to ensure vacancy reasion is reset if letting is not a renewal --- .../lettings_log_variables.rb | 1 + app/models/lettings_log.rb | 1 + spec/models/lettings_log_spec.rb | 21 +++++++++++++++++++ 3 files changed, 23 insertions(+) diff --git a/app/models/derived_variables/lettings_log_variables.rb b/app/models/derived_variables/lettings_log_variables.rb index 7d664e48f..37e659652 100644 --- a/app/models/derived_variables/lettings_log_variables.rb +++ b/app/models/derived_variables/lettings_log_variables.rb @@ -48,6 +48,7 @@ module DerivedVariables::LettingsLogVariables self.offered = 0 self.voiddate = startdate self.first_time_property_let_as_social_housing = 0 + self.rsnvac = 14 if is_general_needs? # fixed term self.prevten = 32 if managing_organisation&.provider_type == "PRP" diff --git a/app/models/lettings_log.rb b/app/models/lettings_log.rb index 02236f70b..d9fe89509 100644 --- a/app/models/lettings_log.rb +++ b/app/models/lettings_log.rb @@ -518,6 +518,7 @@ private def reset_derived_questions dependent_questions = { waityear: [{ key: :renewal, value: 0 }], referral: [{ key: :renewal, value: 0 }], + rsnvac: [{ key: :renewal, value: 0 }], underoccupation_benefitcap: [{ key: :renewal, value: 0 }], wchair: [{ key: :needstype, value: 1 }], location_id: [{ key: :needstype, value: 1 }] } diff --git a/spec/models/lettings_log_spec.rb b/spec/models/lettings_log_spec.rb index d4bbd8378..b28f8bfea 100644 --- a/spec/models/lettings_log_spec.rb +++ b/spec/models/lettings_log_spec.rb @@ -1468,6 +1468,12 @@ RSpec.describe LettingsLog do expect(record_from_db["first_time_property_let_as_social_housing"]).to eq(0) expect(lettings_log["first_time_property_let_as_social_housing"]).to eq(0) end + + it "derives vacancy reason as relet" do + record_from_db = ActiveRecord::Base.connection.execute("select rsnvac from lettings_logs where id=#{lettings_log.id}").to_a[0] + expect(record_from_db["rsnvac"]).to eq(14) + expect(lettings_log["rsnvac"]).to eq(14) + end end context "when answering the household characteristics questions" do @@ -1992,6 +1998,21 @@ RSpec.describe LettingsLog do expect(record_from_db["waityear"]).to eq(nil) expect(lettings_log["waityear"]).to eq(nil) end + + it "resets inferred vacancy reason value" do + vacancy_reason = "rsnvac" + + lettings_log.update!({ renewal: 1 }) + + record_from_db = ActiveRecord::Base.connection.execute("select #{vacancy_reason} from lettings_logs where id=#{lettings_log.id}").to_a[0] + expect(record_from_db[vacancy_reason]).to eq(14) + expect(lettings_log[vacancy_reason]).to eq(14) + + lettings_log.update!({ renewal: 0 }) + record_from_db = ActiveRecord::Base.connection.execute("select #{vacancy_reason} from lettings_logs where id=#{lettings_log.id}").to_a[0] + expect(record_from_db[vacancy_reason]).to eq(nil) + expect(lettings_log[vacancy_reason]).to eq(nil) + end end context "when it changes from a supported housing to not a supported housing" do From f9d91581fcfc6f72c924c7895f2054c8da34885c Mon Sep 17 00:00:00 2001 From: Arthur Campbell <51094020+arfacamble@users.noreply.github.com> Date: Mon, 27 Feb 2023 09:32:36 +0000 Subject: [PATCH 3/8] CLDC-1913 Change date question hint text (#1293) * change hardcoded date text to today's date formatted correctly * update to show an example date in the tax year of the relevant log where that year is known * move helper method and reuse some existing logic, adjust tests accordingly --- app/helpers/collection_time_helper.rb | 5 +++ app/views/form/_date_question.html.erb | 2 +- config/initializers/date_formats.rb | 1 + spec/helpers/collection_time_helper_spec.rb | 41 ++++++++++++++++++--- 4 files changed, 43 insertions(+), 6 deletions(-) diff --git a/app/helpers/collection_time_helper.rb b/app/helpers/collection_time_helper.rb index 23014b8d1..b9dee7dcc 100644 --- a/app/helpers/collection_time_helper.rb +++ b/app/helpers/collection_time_helper.rb @@ -12,6 +12,11 @@ module CollectionTimeHelper Time.zone.local(collection_start_year(date), 4, 1) end + def date_mid_collection_year_formatted(date) + example_date = date.nil? ? Time.zone.today : collection_start_date(date).to_date + 5.months + example_date.to_formatted_s(:govuk_date_number_month) + end + def current_collection_start_date Time.zone.local(current_collection_start_year, 4, 1) end diff --git a/app/views/form/_date_question.html.erb b/app/views/form/_date_question.html.erb index 153679ace..167e02a31 100644 --- a/app/views/form/_date_question.html.erb +++ b/app/views/form/_date_question.html.erb @@ -3,7 +3,7 @@ <%= f.govuk_date_field question.id.to_sym, caption: caption(caption_text, page_header, conditional), legend: legend(question, page_header, conditional), - hint: { text: question.hint_text&.html_safe || "For example, 1 9 2022." }, + hint: { text: question.hint_text&.html_safe || "For example, #{date_mid_collection_year_formatted(lettings_log.startdate)}" }, width: 20, **stimulus_html_attributes(question) do %> <%= govuk_inset_text(text: question.unresolved_hint_text) if question.unresolved_hint_text.present? && @log.unresolved %> diff --git a/config/initializers/date_formats.rb b/config/initializers/date_formats.rb index 59bc215ce..71179f6d2 100644 --- a/config/initializers/date_formats.rb +++ b/config/initializers/date_formats.rb @@ -2,6 +2,7 @@ Time::DATE_FORMATS[:govuk_date] = "%-d %B %Y" Time::DATE_FORMATS[:govuk_date_short_month] = "%-d %b %Y" Date::DATE_FORMATS[:govuk_date] = "%-d %B %Y" Date::DATE_FORMATS[:govuk_date_short_month] = "%-d %b %Y" +Date::DATE_FORMATS[:govuk_date_number_month] = "%-d %-m %Y" Time::DATE_FORMATS[:month_and_year] = "%B %Y" Date::DATE_FORMATS[:month_and_year] = "%B %Y" diff --git a/spec/helpers/collection_time_helper_spec.rb b/spec/helpers/collection_time_helper_spec.rb index c2eb2fedb..0c187020e 100644 --- a/spec/helpers/collection_time_helper_spec.rb +++ b/spec/helpers/collection_time_helper_spec.rb @@ -4,13 +4,13 @@ RSpec.describe CollectionTimeHelper do let(:current_user) { create(:user, :data_coordinator) } let(:user) { create(:user, :data_coordinator) } - around do |example| - Timecop.freeze(now) do - example.run + describe "Current collection start year" do + around do |example| + Timecop.freeze(now) do + example.run + end end - end - describe "Current collection start year" do context "when the date is after 1st of April" do let(:now) { Time.utc(2022, 8, 3) } @@ -77,4 +77,35 @@ RSpec.describe CollectionTimeHelper do end end end + + describe "#date_mid_collection_year_formatted" do + subject(:result) { date_mid_collection_year_formatted(input) } + + context "when called with nil" do + let(:input) { nil } + + it "returns the current date" do + today = Time.zone.today + expect(result).to eq("#{today.day} #{today.month} #{today.year}") + end + end + + context "when called with a date after the first of April" do + calendar_year = 2030 + let(:input) { Date.new(calendar_year, 7, 7) } + + it "returns the first of September from that year" do + expect(result).to eq("1 9 #{calendar_year}") + end + end + + context "when called with a date before April" do + calendar_year = 2040 + let(:input) { Date.new(calendar_year, 2, 7) } + + it "returns the first of September from the previous year" do + expect(result).to eq("1 9 #{calendar_year - 1}") + end + end + end end From 80f04a8908f485dc4e2de4dd4debebab9d2220a2 Mon Sep 17 00:00:00 2001 From: Arthur Campbell <51094020+arfacamble@users.noreply.github.com> Date: Mon, 27 Feb 2023 10:30:42 +0000 Subject: [PATCH 4/8] CLDC-1807 pluralise copy in housing benefits if joint purchase (#1320) * alter copy conditionally for plurality depending on whether a purchase is a joint purchase * correct linting oversight * resolve rebase conflicts and alter annoying test to prevent future conflicts --- app/controllers/form_controller.rb | 2 +- .../form/sales/pages/housing_benefits.rb | 9 ++++---- .../form/sales/questions/housing_benefits.rb | 6 +++--- .../income_benefits_and_savings.rb | 3 ++- .../form/sales/pages/housing_benefits_spec.rb | 21 ++++++++++++++----- .../sales/questions/housing_benefits_spec.rb | 17 ++++++++++++--- .../income_benefits_and_savings_spec.rb | 6 ++++-- spec/models/form_handler_spec.rb | 4 ++-- 8 files changed, 47 insertions(+), 21 deletions(-) diff --git a/app/controllers/form_controller.rb b/app/controllers/form_controller.rb index 0a2d1ad65..467b049df 100644 --- a/app/controllers/form_controller.rb +++ b/app/controllers/form_controller.rb @@ -54,7 +54,7 @@ class FormController < ApplicationController if @page.routed_to?(@log, current_user) render "form/page" else - redirect_to lettings_log_path(@log) + redirect_to @log.lettings? ? lettings_log_path(@log) : sales_log_path(@log) end else render_not_found diff --git a/app/models/form/sales/pages/housing_benefits.rb b/app/models/form/sales/pages/housing_benefits.rb index 5de83cbfd..670673bd8 100644 --- a/app/models/form/sales/pages/housing_benefits.rb +++ b/app/models/form/sales/pages/housing_benefits.rb @@ -1,12 +1,13 @@ class Form::Sales::Pages::HousingBenefits < ::Form::Page - def initialize(id, hsh, subsection) - super - @id = "housing_benefits" + def initialize(id, hsh, subsection, joint_purchase:) + super(id, hsh, subsection) + @joint_purchase = joint_purchase + @depends_on = [{ "jointpur" => @joint_purchase ? 1 : 2 }] end def questions @questions ||= [ - Form::Sales::Questions::HousingBenefits.new(nil, nil, self), + Form::Sales::Questions::HousingBenefits.new(nil, nil, self, joint_purchase: @joint_purchase), ] end end diff --git a/app/models/form/sales/questions/housing_benefits.rb b/app/models/form/sales/questions/housing_benefits.rb index db3e14c88..a4af620e6 100644 --- a/app/models/form/sales/questions/housing_benefits.rb +++ b/app/models/form/sales/questions/housing_benefits.rb @@ -1,9 +1,9 @@ class Form::Sales::Questions::HousingBenefits < ::Form::Question - def initialize(id, hsh, page) - super + def initialize(id, hsh, page, joint_purchase:) + super(id, hsh, page) @id = "hb" @check_answer_label = "Housing-related benefits buyer received before buying this property" - @header = "Was the buyer receiving any of these housing-related benefits immediately before buying this property?" + @header = "#{joint_purchase ? 'Were the buyers' : 'Was the buyer'} receiving any of these housing-related benefits immediately before buying this property?" @type = "radio" @answer_options = ANSWER_OPTIONS end diff --git a/app/models/form/sales/subsections/income_benefits_and_savings.rb b/app/models/form/sales/subsections/income_benefits_and_savings.rb index 7642e84bd..9ff584d47 100644 --- a/app/models/form/sales/subsections/income_benefits_and_savings.rb +++ b/app/models/form/sales/subsections/income_benefits_and_savings.rb @@ -18,7 +18,8 @@ class Form::Sales::Subsections::IncomeBenefitsAndSavings < ::Form::Subsection Form::Sales::Pages::Buyer2IncomeValueCheck.new("buyer_2_income_value_check", nil, self), Form::Sales::Pages::Buyer2Mortgage.new(nil, nil, self), Form::Sales::Pages::MortgageValueCheck.new("buyer_2_mortgage_value_check", nil, self, 2), - Form::Sales::Pages::HousingBenefits.new(nil, nil, self), + Form::Sales::Pages::HousingBenefits.new("housing_benefits_joint_purchase", nil, self, joint_purchase: true), + Form::Sales::Pages::HousingBenefits.new("housing_benefits_not_joint_purchase", nil, self, joint_purchase: false), Form::Sales::Pages::Savings.new(nil, nil, self), Form::Sales::Pages::SavingsValueCheck.new("savings_value_check", nil, self), Form::Sales::Pages::DepositValueCheck.new("savings_deposit_value_check", nil, self), diff --git a/spec/models/form/sales/pages/housing_benefits_spec.rb b/spec/models/form/sales/pages/housing_benefits_spec.rb index 9cf017bdc..f4a591d40 100644 --- a/spec/models/form/sales/pages/housing_benefits_spec.rb +++ b/spec/models/form/sales/pages/housing_benefits_spec.rb @@ -1,11 +1,12 @@ require "rails_helper" RSpec.describe Form::Sales::Pages::HousingBenefits, type: :model do - subject(:page) { described_class.new(page_id, page_definition, subsection) } + subject(:page) { described_class.new(page_id, page_definition, subsection, joint_purchase:) } - let(:page_id) { nil } + let(:page_id) { "provided_id" } let(:page_definition) { nil } let(:subsection) { instance_double(Form::Subsection) } + let(:joint_purchase) { false } it "has correct subsection" do expect(page.subsection).to eq(subsection) @@ -16,7 +17,7 @@ RSpec.describe Form::Sales::Pages::HousingBenefits, type: :model do end it "has the correct id" do - expect(page.id).to eq("housing_benefits") + expect(page.id).to eq(page_id) end it "has the correct header" do @@ -27,7 +28,17 @@ RSpec.describe Form::Sales::Pages::HousingBenefits, type: :model do expect(page.description).to be_nil end - it "has correct depends_on" do - expect(page.depends_on).to be_nil + context "when joint_purchase is false" do + it "has correct depends_on" do + expect(page.depends_on).to eq([{ "jointpur" => 2 }]) + end + end + + context "when joint_purchase is true" do + let(:joint_purchase) { true } + + it "has correct depends_on" do + expect(page.depends_on).to eq([{ "jointpur" => 1 }]) + end end end diff --git a/spec/models/form/sales/questions/housing_benefits_spec.rb b/spec/models/form/sales/questions/housing_benefits_spec.rb index 8c3ae57d3..12ae84774 100644 --- a/spec/models/form/sales/questions/housing_benefits_spec.rb +++ b/spec/models/form/sales/questions/housing_benefits_spec.rb @@ -1,11 +1,12 @@ require "rails_helper" RSpec.describe Form::Sales::Questions::HousingBenefits, type: :model do - subject(:question) { described_class.new(question_id, question_definition, page) } + subject(:question) { described_class.new(question_id, question_definition, page, joint_purchase:) } let(:question_id) { nil } let(:question_definition) { nil } let(:page) { instance_double(Form::Page) } + let(:joint_purchase) { false } it "has correct page" do expect(question.page).to eq(page) @@ -15,8 +16,18 @@ RSpec.describe Form::Sales::Questions::HousingBenefits, type: :model do expect(question.id).to eq("hb") end - it "has the correct header" do - expect(question.header).to eq("Was the buyer receiving any of these housing-related benefits immediately before buying this property?") + context "when joint purchase is false" do + it "has the correct header" do + expect(question.header).to eq("Was the buyer receiving any of these housing-related benefits immediately before buying this property?") + end + end + + context "when joint purchase is true" do + let(:joint_purchase) { true } + + it "has the correct header" do + expect(question.header).to eq("Were the buyers receiving any of these housing-related benefits immediately before buying this property?") + end end it "has the correct check_answer_label" do diff --git a/spec/models/form/sales/subsections/income_benefits_and_savings_spec.rb b/spec/models/form/sales/subsections/income_benefits_and_savings_spec.rb index cfc677233..6c0997fd2 100644 --- a/spec/models/form/sales/subsections/income_benefits_and_savings_spec.rb +++ b/spec/models/form/sales/subsections/income_benefits_and_savings_spec.rb @@ -30,7 +30,8 @@ RSpec.describe Form::Sales::Subsections::IncomeBenefitsAndSavings, type: :model buyer_2_income_value_check buyer_2_mortgage buyer_2_mortgage_value_check - housing_benefits + housing_benefits_joint_purchase + housing_benefits_not_joint_purchase savings savings_value_check savings_deposit_value_check @@ -56,7 +57,8 @@ RSpec.describe Form::Sales::Subsections::IncomeBenefitsAndSavings, type: :model buyer_2_income_value_check buyer_2_mortgage buyer_2_mortgage_value_check - housing_benefits + housing_benefits_joint_purchase + housing_benefits_not_joint_purchase savings savings_value_check savings_deposit_value_check diff --git a/spec/models/form_handler_spec.rb b/spec/models/form_handler_spec.rb index fb7bad4b2..a6b5b181a 100644 --- a/spec/models/form_handler_spec.rb +++ b/spec/models/form_handler_spec.rb @@ -54,14 +54,14 @@ RSpec.describe FormHandler do it "is able to load a current sales form" do form = form_handler.get_form("current_sales") expect(form).to be_a(Form) - expect(form.pages.count).to eq(182) + expect(form.pages.count).to be_positive expect(form.name).to eq("2022_2023_sales") end it "is able to load a previous sales form" do form = form_handler.get_form("previous_sales") expect(form).to be_a(Form) - expect(form.pages.count).to eq(182) + expect(form.pages.count).to be_positive expect(form.name).to eq("2021_2022_sales") end end From 21341bdac14b653c7e01ae85170929dfc3b95978 Mon Sep 17 00:00:00 2001 From: kosiakkatrina <54268893+kosiakkatrina@users.noreply.github.com> Date: Mon, 27 Feb 2023 11:09:33 +0000 Subject: [PATCH 5/8] Make privacy notice mandatory (#1351) --- app/models/form/sales/pages/privacy_notice.rb | 3 --- .../imports/sales_logs/discounted_ownership_sales_log.xml | 3 +-- spec/models/form/sales/pages/privacy_notice_spec.rb | 2 +- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/app/models/form/sales/pages/privacy_notice.rb b/app/models/form/sales/pages/privacy_notice.rb index a41800194..dc4bfa70d 100644 --- a/app/models/form/sales/pages/privacy_notice.rb +++ b/app/models/form/sales/pages/privacy_notice.rb @@ -3,9 +3,6 @@ class Form::Sales::Pages::PrivacyNotice < ::Form::Page super @id = "privacy_notice" @header = "Department for Levelling Up, Housing and Communities privacy notice" - @depends_on = [{ - "noint" => 2, - }] end def questions diff --git a/spec/fixtures/imports/sales_logs/discounted_ownership_sales_log.xml b/spec/fixtures/imports/sales_logs/discounted_ownership_sales_log.xml index 575f93887..bc1d5f274 100644 --- a/spec/fixtures/imports/sales_logs/discounted_ownership_sales_log.xml +++ b/spec/fixtures/imports/sales_logs/discounted_ownership_sales_log.xml @@ -14,8 +14,7 @@ - - + Yes 2023-02-01 Discount ownership example diff --git a/spec/models/form/sales/pages/privacy_notice_spec.rb b/spec/models/form/sales/pages/privacy_notice_spec.rb index 86a214bfa..d8e25a3d8 100644 --- a/spec/models/form/sales/pages/privacy_notice_spec.rb +++ b/spec/models/form/sales/pages/privacy_notice_spec.rb @@ -28,6 +28,6 @@ RSpec.describe Form::Sales::Pages::PrivacyNotice, type: :model do end it "has correct depends_on" do - expect(page.depends_on).to eq([{ "noint" => 2 }]) + expect(page.depends_on).to be_nil end end From c0543418c490b37598ded9152043d04ded5b07f9 Mon Sep 17 00:00:00 2001 From: James Rose Date: Mon, 27 Feb 2023 11:13:51 +0000 Subject: [PATCH 6/8] Hide 'What are the monthly leasehold charges?' for 22/23 (#1350) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - We discovered earlier today (thanks Kat) that we have accidentally missed out ‘what are the monthly leasehold charges?’ in the ‘outright sales’ section of the 22/23 sales form on old CORE – the very last question on the form. - We've decided that we will not add the question back in for 22/23. However we will add it back in for the upcoming 23/24 forms. --- .../form/sales/subsections/outright_sale.rb | 10 ++- .../imports/sales_logs_import_service.rb | 2 - .../sales/subsections/outright_sale_spec.rb | 70 ++++++++++++++----- .../imports/sales_logs_import_service_spec.rb | 15 ---- 4 files changed, 59 insertions(+), 38 deletions(-) diff --git a/app/models/form/sales/subsections/outright_sale.rb b/app/models/form/sales/subsections/outright_sale.rb index 256c1c8dc..100c8fa64 100644 --- a/app/models/form/sales/subsections/outright_sale.rb +++ b/app/models/form/sales/subsections/outright_sale.rb @@ -20,12 +20,18 @@ class Form::Sales::Subsections::OutrightSale < ::Form::Subsection Form::Sales::Pages::ExtraBorrowing.new("extra_borrowing_outright_sale", nil, self), Form::Sales::Pages::AboutDepositWithoutDiscount.new("about_deposit_outright_sale", nil, self), Form::Sales::Pages::DepositValueCheck.new("outright_sale_deposit_value_check", nil, self), - Form::Sales::Pages::LeaseholdCharges.new("leasehold_charges_outright_sale", nil, self), + leasehold_charge_pages, Form::Sales::Pages::MonthlyChargesValueCheck.new("monthly_charges_outright_sale_value_check", nil, self), - ] + ].compact end def displayed_in_tasklist?(log) log.ownershipsch == 3 end + + def leasehold_charge_pages + if form.start_date.year >= 2023 + Form::Sales::Pages::LeaseholdCharges.new("leasehold_charges_outright_sale", nil, self) + end + end end diff --git a/app/services/imports/sales_logs_import_service.rb b/app/services/imports/sales_logs_import_service.rb index 25f8148b8..deea3fab6 100644 --- a/app/services/imports/sales_logs_import_service.rb +++ b/app/services/imports/sales_logs_import_service.rb @@ -397,8 +397,6 @@ module Imports end def set_default_values(attributes) - attributes["mscharge_known"] ||= 0 if attributes["ownershipsch"] == 3 - attributes["mscharge"] ||= 0 if attributes["ownershipsch"] == 3 attributes["armedforcesspouse"] ||= 7 attributes["hhregres"] ||= 8 attributes["disabled"] ||= 3 diff --git a/spec/models/form/sales/subsections/outright_sale_spec.rb b/spec/models/form/sales/subsections/outright_sale_spec.rb index b7b0dc846..ee659b83d 100644 --- a/spec/models/form/sales/subsections/outright_sale_spec.rb +++ b/spec/models/form/sales/subsections/outright_sale_spec.rb @@ -11,25 +11,57 @@ RSpec.describe Form::Sales::Subsections::OutrightSale, type: :model do expect(outright_sale.section).to eq(section) end - it "has correct pages" do - expect(outright_sale.pages.map(&:id)).to eq( - %w[ - purchase_price_outright_sale - about_price_outright_sale_value_check - mortgage_used_outright_sale - outright_sale_mortgage_used_mortgage_value_check - mortgage_amount_outright_sale - outright_sale_mortgage_amount_mortgage_value_check - mortgage_lender_outright_sale - mortgage_lender_other_outright_sale - mortgage_length_outright_sale - extra_borrowing_outright_sale - about_deposit_outright_sale - outright_sale_deposit_value_check - leasehold_charges_outright_sale - monthly_charges_outright_sale_value_check - ], - ) + describe "pages" do + let(:section) { instance_double(described_class, form: instance_double(Form, start_date:)) } + + context "when 2022" do + let(:start_date) { Time.utc(2022, 2, 8) } + + it "has correct pages" do + expect(outright_sale.pages.compact.map(&:id)).to eq( + %w[ + purchase_price_outright_sale + about_price_outright_sale_value_check + mortgage_used_outright_sale + outright_sale_mortgage_used_mortgage_value_check + mortgage_amount_outright_sale + outright_sale_mortgage_amount_mortgage_value_check + mortgage_lender_outright_sale + mortgage_lender_other_outright_sale + mortgage_length_outright_sale + extra_borrowing_outright_sale + about_deposit_outright_sale + outright_sale_deposit_value_check + monthly_charges_outright_sale_value_check + ], + ) + end + end + + context "when 2023" do + let(:start_date) { Time.utc(2023, 2, 8) } + + it "has correct pages" do + expect(outright_sale.pages.map(&:id)).to eq( + %w[ + purchase_price_outright_sale + about_price_outright_sale_value_check + mortgage_used_outright_sale + outright_sale_mortgage_used_mortgage_value_check + mortgage_amount_outright_sale + outright_sale_mortgage_amount_mortgage_value_check + mortgage_lender_outright_sale + mortgage_lender_other_outright_sale + mortgage_length_outright_sale + extra_borrowing_outright_sale + about_deposit_outright_sale + outright_sale_deposit_value_check + leasehold_charges_outright_sale + monthly_charges_outright_sale_value_check + ], + ) + end + end end it "has the correct id" do diff --git a/spec/services/imports/sales_logs_import_service_spec.rb b/spec/services/imports/sales_logs_import_service_spec.rb index b556d3e68..f57e2f3be 100644 --- a/spec/services/imports/sales_logs_import_service_spec.rb +++ b/spec/services/imports/sales_logs_import_service_spec.rb @@ -235,21 +235,6 @@ RSpec.describe Imports::SalesLogsImportService do end end - context "and it's an outright sale" do - let(:sales_log_id) { "outright_sale_sales_log" } - - before do - allow(logger).to receive(:warn).and_return(nil) - end - - it "infers mscharge_known as no" do - sales_log_service.send(:create_log, sales_log_xml) - - sales_log = SalesLog.find_by(old_id: sales_log_id) - expect(sales_log.mscharge_known).to eq(0) - end - end - context "when inferring age known" do let(:sales_log_id) { "discounted_ownership_sales_log" } From a12d26864bdbc27cf6508152768f872e5192040d Mon Sep 17 00:00:00 2001 From: Phil Lee Date: Mon, 27 Feb 2023 11:31:18 +0000 Subject: [PATCH 7/8] bulk upload handles invalid byte sequences (#1340) - sometimes csv is not exported as utf8 therefore invalid byte sequences are removed --- app/services/bulk_upload/lettings/csv_parser.rb | 1 + .../bulk_upload/lettings/csv_parser_spec.rb | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/app/services/bulk_upload/lettings/csv_parser.rb b/app/services/bulk_upload/lettings/csv_parser.rb index ae29f4a6f..6cead61ab 100644 --- a/app/services/bulk_upload/lettings/csv_parser.rb +++ b/app/services/bulk_upload/lettings/csv_parser.rb @@ -52,6 +52,7 @@ private @normalised_string = File.read(path, encoding: "bom|utf-8") @normalised_string.gsub!("\r\n", "\n") + @normalised_string.scrub!("") @normalised_string end diff --git a/spec/services/bulk_upload/lettings/csv_parser_spec.rb b/spec/services/bulk_upload/lettings/csv_parser_spec.rb index 9d381a5a0..782782e04 100644 --- a/spec/services/bulk_upload/lettings/csv_parser_spec.rb +++ b/spec/services/bulk_upload/lettings/csv_parser_spec.rb @@ -52,4 +52,21 @@ RSpec.describe BulkUpload::Lettings::CsvParser do expect(service.row_parsers[0].field_12).to eql(log.age1) end end + + context "when an invalid byte sequence" do + let(:file) { Tempfile.new } + let(:path) { file.path } + let(:log) { build(:lettings_log, :completed) } + let(:invalid_sequence) { "\x81" } + + before do + file.write(invalid_sequence) + file.write(BulkUpload::LogToCsv.new(log:, col_offset: 0).to_csv_row) + file.close + end + + it "parses csv correctly" do + expect(service.row_parsers[0].field_12).to eql(log.age1) + end + end end From c418312b6cf999e5aef2e6ac8e0ab20199d236f8 Mon Sep 17 00:00:00 2001 From: James Rose Date: Mon, 27 Feb 2023 12:24:38 +0000 Subject: [PATCH 8/8] Enable sales logs on production (#1352) --- config/initializers/feature_toggle.rb | 2 +- spec/features/sales_log_spec.rb | 4 ++-- spec/helpers/navigation_items_helper_spec.rb | 12 +++++++++--- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/config/initializers/feature_toggle.rb b/config/initializers/feature_toggle.rb index dda281e10..4fc402212 100644 --- a/config/initializers/feature_toggle.rb +++ b/config/initializers/feature_toggle.rb @@ -13,7 +13,7 @@ class FeatureToggle end def self.sales_log_enabled? - !Rails.env.production? + true end def self.managing_owning_enabled? diff --git a/spec/features/sales_log_spec.rb b/spec/features/sales_log_spec.rb index 270d9bf37..04331d824 100644 --- a/spec/features/sales_log_spec.rb +++ b/spec/features/sales_log_spec.rb @@ -53,9 +53,9 @@ RSpec.describe "Sales Log Features" do end end - context "when the sales log feature flag is toggled" do + context "when the sales log feature flag is disabled" do before do - allow(Rails.env).to receive(:production?).and_return(true) + allow(FeatureToggle).to receive(:sales_log_enabled?).and_return(false) end it "hides the create sales log button in production" do diff --git a/spec/helpers/navigation_items_helper_spec.rb b/spec/helpers/navigation_items_helper_spec.rb index 00e3cdccc..3656c49c5 100644 --- a/spec/helpers/navigation_items_helper_spec.rb +++ b/spec/helpers/navigation_items_helper_spec.rb @@ -7,9 +7,10 @@ RSpec.describe NavigationItemsHelper do let(:organisation_path) { "/organisations/#{current_user.organisation.id}" } describe "#primary items" do - context "when the sales log feature flag is enabled" do + context "when the sales log feature flag is disabled" do before do - allow(Rails.env).to receive(:production?).and_return(true) + allow(FeatureToggle).to receive(:sales_log_enabled?).and_return(false) + allow(FeatureToggle).to receive(:managing_owning_enabled?).and_return(false) end context "when the user is a data coordinator" do @@ -336,7 +337,12 @@ RSpec.describe NavigationItemsHelper do end end - context "when the sales log feature flag is disabled" do + context "when the sales log feature flag is enabled" do + before do + allow(FeatureToggle).to receive(:sales_log_enabled?).and_return(true) + allow(FeatureToggle).to receive(:managing_owning_enabled?).and_return(true) + end + context "when the user is a data coordinator" do context "when the user is on the lettings logs page" do let(:expected_navigation_items) do