diff --git a/.github/workflows/staging_pipeline.yml b/.github/workflows/staging_pipeline.yml index eae871b31..c4c1db5c1 100644 --- a/.github/workflows/staging_pipeline.yml +++ b/.github/workflows/staging_pipeline.yml @@ -229,7 +229,7 @@ jobs: cf set-env $APP_NAME S3_CONFIG $S3_CONFIG cf set-env $APP_NAME CSV_DOWNLOAD_PAAS_INSTANCE $CSV_DOWNLOAD_PAAS_INSTANCE cf set-env $APP_NAME SENTRY_DSN $SENTRY_DSN - cf push $APP_NAME --strategy rolling + cf push $APP_NAME --strategy rolling -t 180 aws_deploy: name: AWS Deploy diff --git a/app/models/form/lettings/pages/reason_for_leaving_last_settled_home_renewal.rb b/app/models/form/lettings/pages/reason_for_leaving_last_settled_home_renewal.rb index ba025915f..acc8e7045 100644 --- a/app/models/form/lettings/pages/reason_for_leaving_last_settled_home_renewal.rb +++ b/app/models/form/lettings/pages/reason_for_leaving_last_settled_home_renewal.rb @@ -6,6 +6,9 @@ class Form::Lettings::Pages::ReasonForLeavingLastSettledHomeRenewal < ::Form::Pa end def questions - @questions ||= [Form::Lettings::Questions::ReasonRenewal.new(nil, nil, self)] + @questions ||= [ + Form::Lettings::Questions::ReasonRenewal.new(nil, nil, self), + Form::Lettings::Questions::Reasonother.new(nil, nil, self), + ] end end diff --git a/app/models/form/lettings/questions/ppcodenk.rb b/app/models/form/lettings/questions/ppcodenk.rb index 25557a166..c8dbd24d0 100644 --- a/app/models/form/lettings/questions/ppcodenk.rb +++ b/app/models/form/lettings/questions/ppcodenk.rb @@ -8,11 +8,14 @@ class Form::Lettings::Questions::Ppcodenk < ::Form::Question @check_answers_card_number = 0 @hint_text = "This is also known as the household’s ‘last settled home’." @answer_options = ANSWER_OPTIONS - @conditional_for = { "ppostcode_full" => [1] } + @conditional_for = { "ppostcode_full" => [0] } @hidden_in_check_answers = { "depends_on" => [{ "ppcodenk" => 0 }, { "ppcodenk" => 1 }] } @question_number = 80 @disable_clearing_if_not_routed_or_dynamic_answer_options = true end - ANSWER_OPTIONS = { "1" => { "value" => "Yes" }, "0" => { "value" => "No" } }.freeze + ANSWER_OPTIONS = { + "0" => { "value" => "Yes" }, + "1" => { "value" => "No" }, + }.freeze end diff --git a/app/models/form/lettings/questions/ppostcode_full.rb b/app/models/form/lettings/questions/ppostcode_full.rb index 86b4c7b31..edcbaf30a 100644 --- a/app/models/form/lettings/questions/ppostcode_full.rb +++ b/app/models/form/lettings/questions/ppostcode_full.rb @@ -6,7 +6,12 @@ class Form::Lettings::Questions::PpostcodeFull < ::Form::Question @header = "Postcode for the previous accommodation" @type = "text" @width = 5 - @inferred_check_answers_value = [{ "condition" => { "ppcodenk" => 0 }, "value" => "Not known" }] + @inferred_check_answers_value = [{ + "condition" => { + "ppcodenk" => 1, + }, + "value" => "Not known", + }] @check_answers_card_number = 0 @hint_text = "" @inferred_answers = { "prevloc" => { "is_previous_la_inferred" => true } } diff --git a/app/models/form/lettings/questions/reason_renewal.rb b/app/models/form/lettings/questions/reason_renewal.rb index 86aae9c39..0158eba63 100644 --- a/app/models/form/lettings/questions/reason_renewal.rb +++ b/app/models/form/lettings/questions/reason_renewal.rb @@ -9,10 +9,27 @@ class Form::Lettings::Questions::ReasonRenewal < ::Form::Question @hint_text = "You told us this letting is a renewal. We have removed some options because of this." @answer_options = ANSWER_OPTIONS @question_number = 77 + @conditional_for = { + "reasonother" => [ + 20, + ], + } end ANSWER_OPTIONS = { "40" => { "value" => "End of assured shorthold tenancy (no fault)" }, "42" => { "value" => "End of fixed term tenancy (no fault)" }, + "20" => { + "value" => "Other", + }, + "47" => { + "value" => "Tenant prefers not to say", + }, + "divider" => { + "value" => true, + }, + "28" => { + "value" => "Don’t know", + }, }.freeze end diff --git a/app/models/form/question.rb b/app/models/form/question.rb index c27dca9c2..9f0a8aac9 100644 --- a/app/models/form/question.rb +++ b/app/models/form/question.rb @@ -116,20 +116,20 @@ class Form::Question if is_derived_or_has_inferred_check_answers_value?(log) "Change" elsif type == "checkbox" - answer_options.keys.any? { |key| value_is_yes?(log[key], log.lettings?) } ? "Change" : "Answer" + answer_options.keys.any? { |key| value_is_yes?(log[key]) } ? "Change" : "Answer" else log[id].blank? ? "Answer" : "Change" end end def unanswered?(log) - return answer_options.keys.none? { |key| value_is_yes?(log[key], log.lettings?) } if type == "checkbox" + return answer_options.keys.none? { |key| value_is_yes?(log[key]) } if type == "checkbox" log[id].blank? end def completed?(log) - return answer_options.keys.any? { |key| value_is_yes?(log[key], log.lettings?) } if type == "checkbox" + return answer_options.keys.any? { |key| value_is_yes?(log[key]) } if type == "checkbox" log[id].present? || !log.respond_to?(id.to_sym) || has_inferred_display_value?(log) end @@ -166,18 +166,18 @@ class Form::Question label || value.to_s end - def value_is_yes?(value, is_lettings) + def value_is_yes?(value) case type when "checkbox" value == 1 when "radio" - is_lettings ? RADIO_YES_VALUE_LETTINGS[id.to_sym]&.include?(value) : RADIO_YES_VALUE_SALES[id.to_sym]&.include?(value) + RADIO_YES_VALUE[id.to_sym]&.include?(value) else %w[yes].include?(value.downcase) end end - def value_is_no?(value, is_lettings) + def value_is_no?(value) case type when "checkbox" value && value.zero? @@ -272,9 +272,9 @@ private def checkbox_answer_label(log) answer = [] - return "Yes" if id == "declaration" && value_is_yes?(log["declaration"], log.lettings?) + return "Yes" if id == "declaration" && value_is_yes?(log["declaration"]) - answer_options.each { |key, options| value_is_yes?(log[key], log.lettings?) ? answer << options["value"] : nil } + answer_options.each { |key, options| value_is_yes?(log[key]) ? answer << options["value"] : nil } answer.join(", ") end @@ -338,6 +338,7 @@ private is_carehome: [1], hbrentshortfall: [1], net_income_value_check: [0], + ppcodenk: [0], }.freeze RADIO_NO_VALUE = { @@ -362,13 +363,9 @@ private is_carehome: [0], hbrentshortfall: [2], net_income_value_check: [1], + ppcodenk: [1], }.freeze - RADIO_YES_VALUE_LETTINGS = RADIO_YES_VALUE.merge({ ppcodenk: [1] }) - RADIO_YES_VALUE_SALES = RADIO_YES_VALUE.merge({ ppcodenk: [0] }) - RADIO_NO_VALUE_LETTINGS = RADIO_NO_VALUE.merge({ ppcodenk: [0] }) - RADIO_NO_VALUE_SALES = RADIO_NO_VALUE.merge({ ppcodenk: [1] }) - RADIO_DONT_KNOW_VALUE = { sheltered: [3], underoccupation_benefitcap: [3], diff --git a/app/models/lettings_log.rb b/app/models/lettings_log.rb index 9ab6329d2..6d15699cc 100644 --- a/app/models/lettings_log.rb +++ b/app/models/lettings_log.rb @@ -260,8 +260,8 @@ class LettingsLog < Log end def previous_postcode_known? - # 1: Yes - ppcodenk == 1 + # 0: Yes + ppcodenk&.zero? end def previous_la_known? @@ -628,16 +628,22 @@ private def process_postcode_changes! self.postcode_full = upcase_and_remove_whitespace(postcode_full) - process_postcode(postcode_full, "postcode_known", "is_la_inferred", "la") + return if postcode_full.blank? + + self.postcode_known = 1 + inferred_la = get_inferred_la(postcode_full) + self.is_la_inferred = inferred_la.present? + self.la = inferred_la if inferred_la.present? end - def process_postcode(postcode, postcode_known_key, la_inferred_key, la_key) - return if postcode.blank? + def process_previous_postcode_changes! + self.ppostcode_full = upcase_and_remove_whitespace(ppostcode_full) + return if ppostcode_full.blank? - self[postcode_known_key] = 1 - inferred_la = get_inferred_la(postcode) - self[la_inferred_key] = inferred_la.present? - self[la_key] = inferred_la if inferred_la.present? + self.ppcodenk = 0 + inferred_la = get_inferred_la(ppostcode_full) + self.is_previous_la_inferred = inferred_la.present? + self.prevloc = inferred_la if inferred_la.present? end def get_has_benefits diff --git a/app/models/log.rb b/app/models/log.rb index 024470b21..849927cf1 100644 --- a/app/models/log.rb +++ b/app/models/log.rb @@ -232,11 +232,6 @@ private PIO = PostcodeService.new - def process_previous_postcode_changes! - self.ppostcode_full = upcase_and_remove_whitespace(ppostcode_full) - process_postcode(ppostcode_full, "ppcodenk", "is_previous_la_inferred", "prevloc") - end - LA_CHANGES = { "E07000027" => "E06000064", # Barrow-in-Furness => Westmorland and Furness "E07000030" => "E06000064", # Eden => Westmorland and Furness diff --git a/app/models/sales_log.rb b/app/models/sales_log.rb index 283485d11..f144336bf 100644 --- a/app/models/sales_log.rb +++ b/app/models/sales_log.rb @@ -278,15 +278,6 @@ class SalesLog < Log mortgage + deposit end - def process_postcode(postcode, postcode_known_key, la_inferred_key, la_key) - return if postcode.blank? - - self[postcode_known_key] = 0 - inferred_la = get_inferred_la(postcode) - self[la_inferred_key] = inferred_la.present? - self[la_key] = inferred_la if inferred_la.present? - end - def outright_sale? ownershipsch == 3 end @@ -305,7 +296,22 @@ class SalesLog < Log def process_postcode_changes! self.postcode_full = upcase_and_remove_whitespace(postcode_full) - process_postcode(postcode_full, "pcodenk", "is_la_inferred", "la") + return if postcode_full.blank? + + self.pcodenk = 0 + inferred_la = get_inferred_la(postcode_full) + self.is_la_inferred = inferred_la.present? + self.la = inferred_la if inferred_la.present? + end + + def process_previous_postcode_changes! + self.ppostcode_full = upcase_and_remove_whitespace(ppostcode_full) + return if ppostcode_full.blank? + + self.ppcodenk = 0 + inferred_la = get_inferred_la(ppostcode_full) + self.is_previous_la_inferred = inferred_la.present? + self.prevloc = inferred_la if inferred_la.present? end def reset_created_by! diff --git a/app/models/validations/household_validations.rb b/app/models/validations/household_validations.rb index 07a3fa467..1f4e0998d 100644 --- a/app/models/validations/household_validations.rb +++ b/app/models/validations/household_validations.rb @@ -4,10 +4,6 @@ module Validations::HouseholdValidations # Validations methods need to be called 'validate_' to run on model save # or 'validate_' to run on submit as well def validate_reasonable_preference(record) - if record.is_not_homeless? && record.rp_homeless == 1 - record.errors.add :reasonable_preference_reason, I18n.t("validations.household.reasonpref.not_homeless") - record.errors.add :homeless, I18n.t("validations.household.homeless.reasonpref.not_homeless") - end if !record.given_reasonable_preference? && [record.rp_homeless, record.rp_insan_unsat, record.rp_medwel, record.rp_hardship, record.rp_dontknow].any? { |a| a == 1 } record.errors.add :reasonable_preference_reason, I18n.t("validations.household.reasonable_preference_reason.reason_not_required") end diff --git a/app/models/validations/sales/sale_information_validations.rb b/app/models/validations/sales/sale_information_validations.rb index b802fa08f..3f4d06a30 100644 --- a/app/models/validations/sales/sale_information_validations.rb +++ b/app/models/validations/sales/sale_information_validations.rb @@ -11,19 +11,6 @@ module Validations::Sales::SaleInformationValidations end end - def validate_years_living_in_property_before_purchase(record) - return unless record.proplen&.nonzero? - - case record.type - when 18 - record.errors.add :type, I18n.t("validations.sale_information.proplen.social_homebuy") - record.errors.add :proplen, I18n.t("validations.sale_information.proplen.social_homebuy") - when 28, 29 - record.errors.add :type, I18n.t("validations.sale_information.proplen.rent_to_buy") - record.errors.add :proplen, I18n.t("validations.sale_information.proplen.rent_to_buy") - end - end - def validate_exchange_date(record) return unless record.exdate && record.saledate diff --git a/app/services/bulk_upload/lettings/year2022/row_parser.rb b/app/services/bulk_upload/lettings/year2022/row_parser.rb index 9bef9312f..1f8839989 100644 --- a/app/services/bulk_upload/lettings/year2022/row_parser.rb +++ b/app/services/bulk_upload/lettings/year2022/row_parser.rb @@ -689,14 +689,10 @@ private end def validate_reasonable_preference_homeless - if field_69 == 1 && homeless == 1 && field_70 == 1 - errors.add(:field_70, I18n.t("validations.household.reasonpref.not_homeless")) - else - reason_fields = %i[field_70 field_71 field_72 field_73 field_74] - if field_69 == 1 && reason_fields.all? { |field| attributes[field.to_s].blank? } - reason_fields.each do |field| - errors.add(field, I18n.t("validations.not_answered", question: "reason for reasonable preference")) - end + reason_fields = %i[field_70 field_71 field_72 field_73 field_74] + if field_69 == 1 && reason_fields.all? { |field| attributes[field.to_s].blank? } + reason_fields.each do |field| + errors.add(field, I18n.t("validations.not_answered", question: "reason for reasonable preference")) end end end @@ -1285,9 +1281,9 @@ private def ppcodenk case field_65 when 1 - 1 - when 2 0 + when 2 + 1 end end diff --git a/app/services/bulk_upload/lettings/year2023/row_parser.rb b/app/services/bulk_upload/lettings/year2023/row_parser.rb index 8d023792c..9839846f8 100644 --- a/app/services/bulk_upload/lettings/year2023/row_parser.rb +++ b/app/services/bulk_upload/lettings/year2023/row_parser.rb @@ -622,14 +622,10 @@ private end def validate_reasonable_preference_homeless - if field_110 == 1 && field_105 == 1 && field_111 == 1 - errors.add(:field_111, I18n.t("validations.household.reasonpref.not_homeless")) - else - reason_fields = %i[field_111 field_112 field_113 field_114 field_115] - if field_110 == 1 && reason_fields.all? { |field| attributes[field.to_s].blank? } - reason_fields.each do |field| - errors.add(field, I18n.t("validations.not_answered", question: "reason for reasonable preference")) - end + reason_fields = %i[field_111 field_112 field_113 field_114 field_115] + if field_110 == 1 && reason_fields.all? { |field| attributes[field.to_s].blank? } + reason_fields.each do |field| + errors.add(field, I18n.t("validations.not_answered", question: "reason for reasonable preference")) end end end @@ -1424,9 +1420,9 @@ private def ppcodenk case field_106 when 1 - 1 - when 2 0 + when 2 + 1 end end diff --git a/app/services/exports/lettings_log_export_service.rb b/app/services/exports/lettings_log_export_service.rb index ea5ab66f9..9bcc8676c 100644 --- a/app/services/exports/lettings_log_export_service.rb +++ b/app/services/exports/lettings_log_export_service.rb @@ -8,13 +8,13 @@ module Exports @logger = logger end - def export_xml_lettings_logs(full_update: false) + def export_xml_lettings_logs(full_update: false, collection_year: nil) start_time = Time.zone.now daily_run_number = get_daily_run_number archives_for_manifest = {} base_number = LogsExport.where(empty_export: false).maximum(:base_number) || 1 recent_export = LogsExport.order("started_at").last - available_collection_years.each do |collection| + collection_years_to_export(collection_year).each do |collection| export = build_export_run(collection, start_time, base_number, full_update) archives = write_export_archive(export, collection, start_time, recent_export, full_update) @@ -119,7 +119,7 @@ module Exports def retrieve_lettings_logs(start_time, recent_export, full_update) if !full_update && recent_export params = { from: recent_export.started_at, to: start_time } - LettingsLog.exportable.where("updated_at >= :from and updated_at <= :to", params) + LettingsLog.exportable.where("(updated_at >= :from AND updated_at <= :to) OR (values_updated_at IS NOT NULL AND values_updated_at >= :from AND values_updated_at <= :to)", params) else params = { to: start_time } LettingsLog.exportable.where("updated_at <= :to", params) @@ -267,7 +267,9 @@ module Exports xml_doc_to_temp_file(doc) end - def available_collection_years + def collection_years_to_export(collection_year) + return [collection_year] if collection_year.present? + FormHandler.instance.lettings_forms.values.map { |f| f.start_date.year }.uniq end end diff --git a/app/services/imports/import_report_service.rb b/app/services/imports/import_report_service.rb index 61d43a9eb..4e387c4c4 100644 --- a/app/services/imports/import_report_service.rb +++ b/app/services/imports/import_report_service.rb @@ -11,6 +11,7 @@ module Imports def create_reports(report_suffix) generate_missing_data_coordinators_report(report_suffix) generate_logs_report(report_suffix) + generate_unassigned_logs_report(report_suffix) end def generate_missing_data_coordinators_report(report_suffix) @@ -53,5 +54,35 @@ module Imports @logger.info("Logs report available in s3 import bucket at #{report_name}") end + + def generate_unassigned_logs_report(report_suffix) + Rails.logger.info("Generating unassigned logs report") + + rep = CSV.generate do |report| + headers = ["Owning Organisation ID", "Old Owning Organisation ID", "Managing Organisation ID", "Old Managing Organisation ID", "Log ID", "Old Log ID", "Tenancy code", "Purchaser code"] + report << headers + + @institutions_csv.each do |row| + name = row[0] + organisation = Organisation.find_by(name:) + next unless organisation + + unassigned_user = organisation.users.find_by(name: "Unassigned") + next unless unassigned_user + + organisation.owned_lettings_logs.where(created_by: unassigned_user).each do |lettings_log| + report << [organisation.id, organisation.old_org_id, lettings_log.managing_organisation.id, lettings_log.managing_organisation.old_org_id, lettings_log.id, lettings_log.old_id, lettings_log.tenancycode, nil] + end + organisation.owned_sales_logs.where(created_by: unassigned_user).each do |sales_log| + report << [organisation.id, organisation.old_org_id, nil, nil, sales_log.id, sales_log.old_id, nil, sales_log.purchid] + end + end + end + + report_name = "UnassignedLogsReport_#{report_suffix}" + @storage_service.write_file(report_name, BYTE_ORDER_MARK + rep) + + @logger.info("Unassigned logs report available in s3 import bucket at #{report_name}") + end end end diff --git a/app/services/imports/import_service.rb b/app/services/imports/import_service.rb index 2a2e0f3a6..d631c7fe4 100644 --- a/app/services/imports/import_service.rb +++ b/app/services/imports/import_service.rb @@ -23,7 +23,11 @@ module Imports end def field_value(xml_document, namespace, field, *args) - xml_document.at_xpath("//#{namespace}:#{field}", *args)&.text + if namespace.present? + xml_document.at_xpath("//#{namespace}:#{field}", *args)&.text + else + xml_document.at_xpath("//dclg:#{field}", *args + [{ "dclg" => "dclg:institution" }])&.text + end end def meta_field_value(xml_document, field) diff --git a/app/services/imports/lettings_logs_import_service.rb b/app/services/imports/lettings_logs_import_service.rb index 339dc5ca9..138126cbb 100644 --- a/app/services/imports/lettings_logs_import_service.rb +++ b/app/services/imports/lettings_logs_import_service.rb @@ -174,7 +174,7 @@ module Imports 0 end - attributes["offered"] = safe_string_as_integer(xml_doc, "Q20") + attributes["offered"] = safe_string_as_decimal(xml_doc, "Q20") attributes["propcode"] = string_or_nil(xml_doc, "Q21a") attributes["beds"] = safe_string_as_integer(xml_doc, "Q22") attributes["unittype_gn"] = unsafe_string_as_integer(xml_doc, "Q23") @@ -251,9 +251,27 @@ module Imports owner_id = meta_field_value(xml_doc, "owner-user-id").strip if owner_id.present? user = LegacyUser.find_by(old_user_id: owner_id)&.user + if user.blank? + @logger.error("Lettings log '#{attributes['old_id']}' belongs to legacy user with owner-user-id: '#{owner_id}' which cannot be found. Assigning log to 'Unassigned' user.") + if User.find_by(name: "Unassigned", organisation_id: attributes["managing_organisation_id"]) + user = User.find_by(name: "Unassigned", organisation_id: attributes["managing_organisation_id"]) + else + user = User.new( + name: "Unassigned", + organisation_id: attributes["managing_organisation_id"], + is_dpo: false, + encrypted_password: SecureRandom.hex(10), + email: SecureRandom.uuid, + confirmed_at: Time.zone.now, + active: false, + ) + user.save!(validate: false) + end + end attributes["created_by"] = user end + attributes["values_updated_at"] = Time.zone.now apply_date_consistency!(attributes) apply_household_consistency!(attributes) @@ -465,11 +483,11 @@ module Imports def previous_postcode_known(xml_doc, previous_postcode, prevloc) previous_postcode_known = string_or_nil(xml_doc, "Q12bnot") if previous_postcode_known == "Temporary_or_Unknown" || (previous_postcode.nil? && prevloc.present?) - 0 # not known + 1 # not known elsif previous_postcode.nil? nil else - 1 # known + 0 # known end end diff --git a/app/services/imports/logs_import_service.rb b/app/services/imports/logs_import_service.rb index c6c158089..930956f16 100644 --- a/app/services/imports/logs_import_service.rb +++ b/app/services/imports/logs_import_service.rb @@ -106,7 +106,7 @@ module Imports "F" when "Other", "Non-binary" "X" - when "Refused" + when "Refused", "Person prefers not to say", "Buyer prefers not to say" "R" end end @@ -120,7 +120,7 @@ module Imports "P" when "Other", "Non-binary" "X" - when "Refused", "Buyer prefers not to say" + when "Refused", "Person prefers not to say", "Buyer prefers not to say" "R" end end diff --git a/app/services/imports/organisation_import_service.rb b/app/services/imports/organisation_import_service.rb index 1ec1575bc..d48038bfd 100644 --- a/app/services/imports/organisation_import_service.rb +++ b/app/services/imports/organisation_import_service.rb @@ -46,6 +46,8 @@ module Imports def organisation_field_value(xml_document, field) field_value(xml_document, "institution", field) + rescue Nokogiri::SyntaxError + field_value(xml_document, nil, field) end end end diff --git a/app/services/imports/sales_logs_import_service.rb b/app/services/imports/sales_logs_import_service.rb index 0d780c87d..ecee55c22 100644 --- a/app/services/imports/sales_logs_import_service.rb +++ b/app/services/imports/sales_logs_import_service.rb @@ -179,8 +179,26 @@ module Imports if owner_id.present? user = LegacyUser.find_by(old_user_id: owner_id)&.user + if user.blank? + @logger.error("Sales log '#{attributes['old_id']}' belongs to legacy user with owner-user-id: '#{owner_id}' which cannot be found. Assigning log to 'Unassigned' user.") + if User.find_by(name: "Unassigned", organisation_id: attributes["owning_organisation_id"]) + user = User.find_by(name: "Unassigned", organisation_id: attributes["owning_organisation_id"]) + else + user = User.new( + name: "Unassigned", + organisation_id: attributes["owning_organisation_id"], + is_dpo: false, + encrypted_password: SecureRandom.hex(10), + email: SecureRandom.uuid, + confirmed_at: Time.zone.now, + active: false, + ) + user.save!(validate: false) + end + end attributes["created_by"] = user end + attributes["values_updated_at"] = Time.zone.now set_default_values(attributes) if previous_status.include?("submitted") sales_log = save_sales_log(attributes, previous_status) diff --git a/app/services/imports/user_import_service.rb b/app/services/imports/user_import_service.rb index b0fe32bef..e280dcfe1 100644 --- a/app/services/imports/user_import_service.rb +++ b/app/services/imports/user_import_service.rb @@ -39,9 +39,15 @@ module Imports user.active = user_field_value(xml_document, "active") user.skip_confirmation_notification! - user.save! - user.legacy_users.create!(old_user_id:) - user + + begin + user.save! + user.legacy_users.create!(old_user_id:) + user + rescue ActiveRecord::RecordInvalid => e + @logger.error(e.message) + @logger.error("Could not save user with email: #{email}") + end end end diff --git a/config/forms/2021_2022.json b/config/forms/2021_2022.json index 75660731d..086f201c2 100644 --- a/config/forms/2021_2022.json +++ b/config/forms/2021_2022.json @@ -6488,15 +6488,15 @@ "disable_clearing_if_not_routed_or_dynamic_answer_options": true, "answer_options": { "1": { - "value": "Yes" + "value": "No" }, "0": { - "value": "No" + "value": "Yes" } }, "conditional_for": { "ppostcode_full": [ - 1 + 0 ] }, "hidden_in_check_answers": { @@ -6524,7 +6524,7 @@ }, "inferred_check_answers_value": [{ "condition": { - "ppcodenk": 0 + "ppcodenk": 1 }, "value": "Not known" }] diff --git a/config/forms/2022_2023.json b/config/forms/2022_2023.json index 8d5f674f3..db11b0950 100644 --- a/config/forms/2022_2023.json +++ b/config/forms/2022_2023.json @@ -6228,8 +6228,30 @@ }, "42": { "value": "End of fixed term tenancy (no fault)" + }, + "20": { + "value": "Other" + }, + "47": { + "value":"Tenant prefers not to say" + }, + "divider": { + "value": true + }, + "28": { + "value": "Don’t know" } + }, + "conditional_for": { + "reasonother": [ + 20 + ] } + }, + "reasonother": { + "header": "What is the reason?", + "hint_text": "", + "type": "text" } }, "depends_on": [ @@ -6388,16 +6410,16 @@ "type": "radio", "disable_clearing_if_not_routed_or_dynamic_answer_options": true, "answer_options": { - "1": { + "0": { "value": "Yes" }, - "0": { + "1": { "value": "No" } }, "conditional_for": { "ppostcode_full": [ - 1 + 0 ] }, "hidden_in_check_answers": { @@ -6425,7 +6447,7 @@ }, "inferred_check_answers_value": [{ "condition": { - "ppcodenk": 0 + "ppcodenk": 1 }, "value": "Not known" }] diff --git a/config/locales/en.yml b/config/locales/en.yml index 4307158e2..5d0f435f9 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -441,8 +441,6 @@ en: mortgage: "Mortgage value cannot be £0 if a mortgage was used for the purchase of this property" household: - reasonpref: - not_homeless: "Answer cannot be ‘homeless or about to lose their home’ as the tenant was not homeless immediately prior to this letting" reasonable_preference_reason: reason_required: "Enter a reason if you've answered 'yes' to reasonable preference" reason_not_required: "Do not enter a reason if you've answered 'no' to reasonable preference" diff --git a/db/migrate/20230828145454_add_migrated_on_fields.rb b/db/migrate/20230828145454_add_migrated_on_fields.rb new file mode 100644 index 000000000..afac3925f --- /dev/null +++ b/db/migrate/20230828145454_add_migrated_on_fields.rb @@ -0,0 +1,6 @@ +class AddMigratedOnFields < ActiveRecord::Migration[7.0] + def change + add_column :lettings_logs, :values_updated_at, :datetime + add_column :sales_logs, :values_updated_at, :datetime + end +end diff --git a/db/schema.rb b/db/schema.rb index fe5b6cde4..7218e75a5 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2023_08_16_083950) do +ActiveRecord::Schema[7.0].define(version: 2023_08_28_145454) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -297,6 +297,7 @@ ActiveRecord::Schema[7.0].define(version: 2023_08_16_083950) do t.integer "status_cache", default: 0, null: false t.datetime "discarded_at" t.integer "creation_method", default: 1 + t.datetime "values_updated_at" t.index ["bulk_upload_id"], name: "index_lettings_logs_on_bulk_upload_id" t.index ["created_by_id"], name: "index_lettings_logs_on_created_by_id" t.index ["location_id"], name: "index_lettings_logs_on_location_id" @@ -617,6 +618,7 @@ ActiveRecord::Schema[7.0].define(version: 2023_08_16_083950) do t.integer "stairowned_value_check" t.integer "creation_method", default: 1 t.integer "old_form_id" + t.datetime "values_updated_at" t.index ["bulk_upload_id"], name: "index_sales_logs_on_bulk_upload_id" t.index ["created_by_id"], name: "index_sales_logs_on_created_by_id" t.index ["old_id"], name: "index_sales_logs_on_old_id", unique: true diff --git a/lib/tasks/correct_ppcodenk_values.rake b/lib/tasks/correct_ppcodenk_values.rake new file mode 100644 index 000000000..86c678fc0 --- /dev/null +++ b/lib/tasks/correct_ppcodenk_values.rake @@ -0,0 +1,6 @@ +desc "Alter ppcodenk values for non imported lettings logs in the database" +task correct_ppcodenk_values: :environment do + LettingsLog.where.not(ppcodenk: nil).find_each do |log| + log.update_columns(ppcodenk: log.ppcodenk == 1 ? 0 : 1) + end +end diff --git a/lib/tasks/data_export.rake b/lib/tasks/data_export.rake index 0cb5b3a2e..e57c3779d 100644 --- a/lib/tasks/data_export.rake +++ b/lib/tasks/data_export.rake @@ -12,10 +12,11 @@ namespace :core do end desc "Export all data XMLs for import into Central Data System (CDS)" - task full_data_export_xml: :environment do |_task, _args| + task :full_data_export_xml, %i[year] => :environment do |_task, args| + collection_year = args[:year].present? ? args[:year].to_i : nil storage_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["EXPORT_PAAS_INSTANCE"]) export_service = Exports::LettingsLogExportService.new(storage_service) - export_service.export_xml_lettings_logs(full_update: true) + export_service.export_xml_lettings_logs(full_update: true, collection_year:) end end diff --git a/lib/tasks/import_address_from_csv.rake b/lib/tasks/import_address_from_csv.rake new file mode 100644 index 000000000..e910f3a77 --- /dev/null +++ b/lib/tasks/import_address_from_csv.rake @@ -0,0 +1,42 @@ +namespace :data_import do + desc "Import address data from a csv file" + task :import_address_from_csv, %i[file_name] => :environment do |_task, args| + file_name = args[:file_name] + + raise "Usage: rake data_import:import_address_from_csv['csv_file_name']" if file_name.blank? + + s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"]) + addresses_csv = CSV.parse(s3_service.get_file_io(file_name), headers: true) + + addresses_csv.each do |row| + lettings_log_id = row[0] + + if lettings_log_id.blank? + Rails.logger.info("Lettings log ID not provided for address: #{[row[1], row[2], row[3], row[4]].join(', ')}") + next + end + + lettings_log = LettingsLog.find_by(id: lettings_log_id) + if lettings_log.blank? + Rails.logger.info("Could not find a lettings log with id #{lettings_log_id}") + next + end + + lettings_log.uprn_known = 0 + lettings_log.uprn = nil + lettings_log.uprn_confirmed = nil + lettings_log.address_line1 = row[1] + lettings_log.address_line2 = row[2] + lettings_log.town_or_city = row[3] + lettings_log.postcode_full = row[4] + lettings_log.postcode_known = lettings_log.postcode_full.present? ? 1 : nil + lettings_log.county = nil + lettings_log.is_la_inferred = nil + lettings_log.la = nil + lettings_log.values_updated_at = Time.zone.now + + lettings_log.save! + Rails.logger.info("Updated lettings log #{lettings_log_id}, with address: #{[lettings_log.address_line1, lettings_log.address_line2, lettings_log.town_or_city, lettings_log.postcode_full].join(', ')}") + end + end +end diff --git a/manifest.yml b/manifest.yml index 5202e60a5..86793319f 100644 --- a/manifest.yml +++ b/manifest.yml @@ -34,8 +34,10 @@ applications: memory: 1G - type: worker command: bundle exec sidekiq -t 3 + disk_quota: 4G health-check-type: process instances: 2 + memory: 8G env: RAILS_ENV: production host: submit-social-housing-lettings-sales-data diff --git a/spec/factories/lettings_log.rb b/spec/factories/lettings_log.rb index 59eb48da2..6cce1dfea 100644 --- a/spec/factories/lettings_log.rb +++ b/spec/factories/lettings_log.rb @@ -166,7 +166,7 @@ FactoryBot.define do joint { 3 } address_line1 { "fake address" } town_or_city { "London" } - ppcodenk { 0 } + ppcodenk { 1 } tshortfall_known { 1 } end trait :export do diff --git a/spec/fixtures/exports/general_needs_log.xml b/spec/fixtures/exports/general_needs_log.xml index f996d5097..27a995304 100644 --- a/spec/fixtures/exports/general_needs_log.xml +++ b/spec/fixtures/exports/general_needs_log.xml @@ -109,7 +109,7 @@ 325.0 12.0 - 1 + 0 1 0 100.0 diff --git a/spec/fixtures/exports/general_needs_log_23_24.xml b/spec/fixtures/exports/general_needs_log_23_24.xml index 796c0c72e..aa3ca758a 100644 --- a/spec/fixtures/exports/general_needs_log_23_24.xml +++ b/spec/fixtures/exports/general_needs_log_23_24.xml @@ -109,7 +109,7 @@ 325.0 12.0 - 1 + 0 1 0 100.0 diff --git a/spec/fixtures/exports/supported_housing_logs.xml b/spec/fixtures/exports/supported_housing_logs.xml index 032590622..73595e76f 100644 --- a/spec/fixtures/exports/supported_housing_logs.xml +++ b/spec/fixtures/exports/supported_housing_logs.xml @@ -108,7 +108,7 @@ 325.0 12.0 - 1 + 0 1 0 100.0 diff --git a/spec/fixtures/files/addresses_reimport.csv b/spec/fixtures/files/addresses_reimport.csv new file mode 100644 index 000000000..998d2fbfa --- /dev/null +++ b/spec/fixtures/files/addresses_reimport.csv @@ -0,0 +1,6 @@ +lettings_log_id,address_line1,address_line2,town_or_city,postcode_full +{id},address 1,address 2,town,B1 1BB +{id2},address 3,address 4,city,B1 1BB +{id3},,,,C1 1CC +,address 5,address 6,city,D1 1DD +fake_id,address 7,address 8,city,D1 1DD diff --git a/spec/fixtures/files/lettings_log_csv_export_codes.csv b/spec/fixtures/files/lettings_log_csv_export_codes.csv index e4a1c53e3..2fc1624af 100644 --- a/spec/fixtures/files/lettings_log_csv_export_codes.csv +++ b/spec/fixtures/files/lettings_log_csv_export_codes.csv @@ -1,2 +1,2 @@ id,status,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,old_id,old_form_id,collection_start_year,owning_organisation_name,managing_organisation_name,needstype,lettype,renewal,startdate,renttype,rent_type_detail,irproduct,irproduct_other,lar,tenancycode,propcode,uprn_known,uprn,uprn_confirmed,address_line1,address_line2,town_or_city,county,postcode_full,is_la_inferred,la_label,la,first_time_property_let_as_social_housing,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,vacdays,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,hhmemb,pregnancy_value_check,age1_known,refused,hhtype,totchild,totelder,totadult,age1,retirement_value_check,sex1,ethnic_group,ethnic,national,ecstat1,details_known_2,relat2,age2_known,age2,sex2,ecstat2,details_known_3,relat3,age3_known,age3,sex3,ecstat3,details_known_4,relat4,age4_known,age4,sex4,ecstat4,details_known_5,relat5,age5_known,age5,sex5,ecstat5,details_known_6,relat6,age6_known,age6,sex6,ecstat6,details_known_7,relat7,age7_known,age7,sex7,ecstat7,details_known_8,relat8,age8_known,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,new_old,homeless,ppcodenk,ppostcode_full,previous_la_known,is_previous_la_inferred,prevloc_label,prevloc,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,letting_allocation_unknown,referral,net_income_known,incref,earnings,incfreq,net_income_value_check,hb,has_benefits,benefits,household_charge,nocharge,period,is_carehome,chcharge,wchchrg,carehome_charges_value_check,brent,wrent,rent_value_check,scharge,wscharge,pscharge,wpschrge,supcharg,wsupchrg,tcharge,wtcharge,hbrentshortfall,tshortfall_known,tshortfall,wtshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_admin_district,location_startdate -,completed,s.port@jeemayle.com,false,2023-06-26T00:00:00+01:00,,2023-06-26T00:00:00+01:00,1,,,2023,DLUHC,DLUHC,1,7,0,2023-06-26T00:00:00+01:00,2,1,,,,HIJKLMN,ABCDEFG,0,,,fake address,,London,,NW9 5LL,false,Barnet,E09000003,0,2,6,2,2,7,1,1,3,2023-06-24T00:00:00+01:00,,,1,2023-06-25T00:00:00+01:00,,3,1,4,,2,,1,2,,0,0,4,0,0,2,35,,F,0,2,13,0,0,P,0,32,M,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,4,1,2,1,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,2,7,4,,6,2,1,1,TN23 6LZ,1,false,Ashford,E07000105,1,0,1,0,0,0,0,0,1,,2,0,0,68,1,,6,1,1,,0,2,,,,,200.0,100.0,,50.0,25.0,40.0,20.0,35.0,17.5,325.0,162.5,1,0,12.0,6.0,,,,,,,,,,,,,,,,,,,, +,completed,s.port@jeemayle.com,false,2023-06-26T00:00:00+01:00,,2023-06-26T00:00:00+01:00,1,,,2023,DLUHC,DLUHC,1,7,0,2023-06-26T00:00:00+01:00,2,1,,,,HIJKLMN,ABCDEFG,0,,,fake address,,London,,NW9 5LL,false,Barnet,E09000003,0,2,6,2,2,7,1,1,3,2023-06-24T00:00:00+01:00,,,1,2023-06-25T00:00:00+01:00,,3,1,4,,2,,1,2,,0,0,4,0,0,2,35,,F,0,2,13,0,0,P,0,32,M,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,4,1,2,1,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,2,7,4,,6,2,1,0,TN23 6LZ,1,false,Ashford,E07000105,1,0,1,0,0,0,0,0,1,,2,0,0,68,1,,6,1,1,,0,2,,,,,200.0,100.0,,50.0,25.0,40.0,20.0,35.0,17.5,325.0,162.5,1,0,12.0,6.0,,,,,,,,,,,,,,,,,,,, diff --git a/spec/fixtures/files/lettings_log_csv_export_non_support_codes.csv b/spec/fixtures/files/lettings_log_csv_export_non_support_codes.csv index 4cfdc963c..8847034ee 100644 --- a/spec/fixtures/files/lettings_log_csv_export_non_support_codes.csv +++ b/spec/fixtures/files/lettings_log_csv_export_non_support_codes.csv @@ -1,2 +1,2 @@ id,status,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,collection_start_year,owning_organisation_name,managing_organisation_name,lettype,renewal,startdate,irproduct,irproduct_other,lar,tenancycode,propcode,uprn_known,uprn,address_line1,address_line2,town_or_city,county,postcode_full,la_label,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,refused,age1,sex1,ethnic_group,ethnic,national,ecstat1,relat2,age2,sex2,ecstat2,relat3,age3,sex3,ecstat3,relat4,age4,sex4,ecstat4,relat5,age5,sex5,ecstat5,relat6,age6,sex6,ecstat6,relat7,age7,sex7,ecstat7,relat8,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,homeless,ppcodenk,ppostcode_full,prevloc_label,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,referral,incref,earnings,incfreq,hb,has_benefits,benefits,household_charge,nocharge,period,chcharge,wchchrg,carehome_charges_value_check,brent,scharge,pscharge,supcharg,tcharge,hbrentshortfall,tshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_admin_district,location_startdate -,completed,choreographer@owtluk.com,false,2023-06-26T00:00:00+01:00,,2023-06-26T00:00:00+01:00,1,2023,DLUHC,DLUHC,7,0,2023-06-26T00:00:00+01:00,,,,HIJKLMN,ABCDEFG,0,,fake address,,London,,NW9 5LL,Barnet,2,6,2,2,7,1,1,3,2023-06-24T00:00:00+01:00,,1,2023-06-25T00:00:00+01:00,,3,1,4,,2,,1,0,35,F,0,2,13,0,P,32,M,6,,,,,,,,,,,,,,,,,,,,,,,,,1,4,1,2,1,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,2,7,4,,6,1,1,TN23 6LZ,Ashford,1,0,1,0,0,0,0,0,1,2,0,68,1,6,1,1,,0,2,,,,200.0,50.0,40.0,35.0,325.0,1,12.0,,,,,,,,,,,,,,,,,,,, +,completed,choreographer@owtluk.com,false,2023-06-26T00:00:00+01:00,,2023-06-26T00:00:00+01:00,1,2023,DLUHC,DLUHC,7,0,2023-06-26T00:00:00+01:00,,,,HIJKLMN,ABCDEFG,0,,fake address,,London,,NW9 5LL,Barnet,2,6,2,2,7,1,1,3,2023-06-24T00:00:00+01:00,,1,2023-06-25T00:00:00+01:00,,3,1,4,,2,,1,0,35,F,0,2,13,0,P,32,M,6,,,,,,,,,,,,,,,,,,,,,,,,,1,4,1,2,1,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,2,7,4,,6,1,0,TN23 6LZ,Ashford,1,0,1,0,0,0,0,0,1,2,0,68,1,6,1,1,,0,2,,,,200.0,50.0,40.0,35.0,325.0,1,12.0,,,,,,,,,,,,,,,,,,,, diff --git a/spec/fixtures/imports/institution/8c5bd5fb549c09a2c55d7cb90d7ba84927e64618.xml b/spec/fixtures/imports/institution/8c5bd5fb549c09a2c55d7cb90d7ba84927e64618.xml new file mode 100644 index 000000000..bd0bc58e2 --- /dev/null +++ b/spec/fixtures/imports/institution/8c5bd5fb549c09a2c55d7cb90d7ba84927e64618.xml @@ -0,0 +1,23 @@ + + 8c5bd5fb549c09z2c55d9cb90d7ba84927e64618 + HA Ltd + HOUSING-ASSOCIATION + true + false + CHHA + xxxxxxxx + false + + false + false + false + + true + true + LH9999 + 1034 + true + 1104 + 217 + 0 + diff --git a/spec/lib/tasks/correct_address_from_csv_spec.rb b/spec/lib/tasks/correct_address_from_csv_spec.rb new file mode 100644 index 000000000..0a3d1c563 --- /dev/null +++ b/spec/lib/tasks/correct_address_from_csv_spec.rb @@ -0,0 +1,154 @@ +require "rails_helper" +require "rake" + +RSpec.describe "data_import" do + def replace_entity_ids(lettings_log, second_lettings_log, third_lettings_log, export_template) + export_template.sub!(/\{id\}/, lettings_log.id.to_s) + export_template.sub!(/\{id2\}/, second_lettings_log.id.to_s) + export_template.sub!(/\{id3\}/, third_lettings_log.id.to_s) + end + + describe ":import_address_from_csv", type: :task do + subject(:task) { Rake::Task["data_import:import_address_from_csv"] } + + let(:instance_name) { "paas_import_instance" } + let(:storage_service) { instance_double(Storage::S3Service) } + let(:env_config_service) { instance_double(Configuration::EnvConfigurationService) } + let(:paas_config_service) { instance_double(Configuration::PaasConfigurationService) } + + before do + allow(Storage::S3Service).to receive(:new).and_return(storage_service) + allow(Configuration::EnvConfigurationService).to receive(:new).and_return(env_config_service) + allow(Configuration::PaasConfigurationService).to receive(:new).and_return(paas_config_service) + allow(ENV).to receive(:[]) + allow(ENV).to receive(:[]).with("IMPORT_PAAS_INSTANCE").and_return(instance_name) + allow(ENV).to receive(:[]).with("VCAP_SERVICES").and_return("dummy") + + Rake.application.rake_require("tasks/import_address_from_csv") + Rake::Task.define_task(:environment) + task.reenable + + WebMock.stub_request(:get, /api.postcodes.io/) + .to_return(status: 200, body: "{\"status\":404,\"error\":\"Postcode not found\"}", headers: {}) + WebMock.stub_request(:get, /api.postcodes.io\/postcodes\/B11BB/) + .to_return(status: 200, body: '{"status":200,"result":{"admin_district":"Westminster","codes":{"admin_district":"E08000035"}}}', headers: {}) + end + + context "when the rake task is run" do + let(:addresses_csv_path) { "addresses_reimport_123.csv" } + let(:wrong_file_path) { "/test/no_csv_here.csv" } + let!(:lettings_log) do + create(:lettings_log, + uprn_known: nil, + uprn: nil, + uprn_confirmed: nil, + address_line1: nil, + address_line2: nil, + town_or_city: nil, + county: nil, + postcode_known: 1, + postcode_full: "A1 1AA", + la: "E06000064", + is_la_inferred: true) + end + + let!(:second_lettings_log) do + create(:lettings_log, + uprn_known: 1, + uprn: "1", + uprn_confirmed: nil, + address_line1: "wrong address line1", + address_line2: "wrong address 2", + town_or_city: "wrong town", + county: "wrong city", + postcode_known: 1, + postcode_full: "A1 1AA", + la: "E06000064", + is_la_inferred: true) + end + + let!(:third_lettings_log) do + create(:lettings_log, + uprn_known: 1, + uprn: "1", + uprn_confirmed: nil, + address_line1: "wrong address line1", + address_line2: "wrong address 2", + town_or_city: "wrong town", + county: "wrong city", + postcode_known: 1, + postcode_full: "A1 1AA", + la: "E06000064", + is_la_inferred: true) + end + + before do + allow(storage_service).to receive(:get_file_io) + .with("addresses_reimport_123.csv") + .and_return(replace_entity_ids(lettings_log, second_lettings_log, third_lettings_log, File.open("./spec/fixtures/files/addresses_reimport.csv").read)) + end + + it "updates the log address when old address was not given" do + task.invoke(addresses_csv_path) + lettings_log.reload + expect(lettings_log.uprn_known).to eq(0) + expect(lettings_log.uprn).to eq(nil) + expect(lettings_log.uprn_confirmed).to eq(nil) + expect(lettings_log.address_line1).to eq("address 1") + expect(lettings_log.address_line2).to eq("address 2") + expect(lettings_log.town_or_city).to eq("town") + expect(lettings_log.county).to eq(nil) + expect(lettings_log.postcode_known).to eq(1) + expect(lettings_log.postcode_full).to eq("B1 1BB") + expect(lettings_log.la).to eq("E08000035") + expect(lettings_log.is_la_inferred).to eq(true) + end + + it "updates the log address when old address was given" do + task.invoke(addresses_csv_path) + second_lettings_log.reload + expect(second_lettings_log.uprn_known).to eq(0) + expect(second_lettings_log.uprn).to eq(nil) + expect(second_lettings_log.uprn_confirmed).to eq(nil) + expect(second_lettings_log.address_line1).to eq("address 3") + expect(second_lettings_log.address_line2).to eq("address 4") + expect(second_lettings_log.town_or_city).to eq("city") + expect(second_lettings_log.county).to eq(nil) + expect(second_lettings_log.postcode_known).to eq(1) + expect(second_lettings_log.postcode_full).to eq("B1 1BB") + expect(second_lettings_log.la).to eq("E08000035") + expect(second_lettings_log.is_la_inferred).to eq(true) + end + + it "updates the log address when new address is not given" do + task.invoke(addresses_csv_path) + third_lettings_log.reload + expect(third_lettings_log.uprn_known).to eq(0) + expect(third_lettings_log.uprn).to eq(nil) + expect(third_lettings_log.uprn_confirmed).to eq(nil) + expect(third_lettings_log.address_line1).to eq(nil) + expect(third_lettings_log.address_line2).to eq(nil) + expect(third_lettings_log.town_or_city).to eq(nil) + expect(third_lettings_log.county).to eq(nil) + expect(third_lettings_log.postcode_known).to eq(1) + expect(third_lettings_log.postcode_full).to eq("C11CC") + expect(third_lettings_log.la).to eq(nil) + expect(third_lettings_log.is_la_inferred).to eq(false) + end + + it "logs the progress of the update" do + expect(Rails.logger).to receive(:info).with("Updated lettings log #{lettings_log.id}, with address: address 1, address 2, town, B1 1BB") + expect(Rails.logger).to receive(:info).with("Updated lettings log #{second_lettings_log.id}, with address: address 3, address 4, city, B1 1BB") + expect(Rails.logger).to receive(:info).with("Updated lettings log #{third_lettings_log.id}, with address: , , , C11CC") + expect(Rails.logger).to receive(:info).with("Lettings log ID not provided for address: address 5, address 6, city, D1 1DD") + expect(Rails.logger).to receive(:info).with("Could not find a lettings log with id fake_id") + + task.invoke(addresses_csv_path) + end + + it "raises an error when no path is given" do + expect { task.invoke(nil) }.to raise_error(RuntimeError, "Usage: rake data_import:import_address_from_csv['csv_file_name']") + end + end + end +end diff --git a/spec/lib/tasks/correct_ppcodenk_values_spec.rb b/spec/lib/tasks/correct_ppcodenk_values_spec.rb new file mode 100644 index 000000000..fabfd05f3 --- /dev/null +++ b/spec/lib/tasks/correct_ppcodenk_values_spec.rb @@ -0,0 +1,67 @@ +require "rails_helper" +require "rake" + +RSpec.describe "correct_ppcodenk_values" do + describe ":correct_ppcodenk_values", type: :task do + subject(:task) { Rake::Task["correct_ppcodenk_values"] } + + before do + Rake.application.rake_require("tasks/correct_ppcodenk_values") + Rake::Task.define_task(:environment) + task.reenable + end + + context "when the rake task is run" do + let!(:lettings_log) { create(:lettings_log, :completed) } + + it "updates lettings logs with ppcodenk 0 to have ppcodenk 1" do + lettings_log.update!(ppcodenk: 0) + task.invoke + expect(lettings_log.reload.ppcodenk).to eq(1) + end + + it "updates lettings logs with ppcodenk 1 to have ppcodenk 0" do + lettings_log.update!(ppcodenk: 1) + task.invoke + expect(lettings_log.reload.ppcodenk).to eq(0) + end + + it "does not update lettings logs with ppcodenk nil" do + lettings_log.update!(ppcodenk: nil) + task.invoke + expect(lettings_log.reload.ppcodenk).to eq(nil) + end + + context "with multiple lettings logs" do + let(:lettings_log_2) { create(:lettings_log, :completed) } + let(:lettings_log_3) { create(:lettings_log, :completed) } + + it "only updates each log once" do + lettings_log.update!(ppcodenk: nil) + lettings_log_2.update!(ppcodenk: 0) + lettings_log_3.update!(ppcodenk: 1) + task.invoke + expect(lettings_log.reload.ppcodenk).to eq(nil) + expect(lettings_log_2.reload.ppcodenk).to eq(1) + expect(lettings_log_3.reload.ppcodenk).to eq(0) + end + end + + it "does not update updated_at value" do + lettings_log.updated_at = Time.zone.local(2021, 3, 3) + lettings_log.save!(validate: false) + expect(lettings_log.updated_at.to_date).to eq(Time.zone.local(2021, 3, 3)) + task.invoke + expect(lettings_log.updated_at.to_date).to eq(Time.zone.local(2021, 3, 3)) + end + + it "skips validations for previous years" do + lettings_log.update!(ppcodenk: 1) + lettings_log.startdate = Time.zone.local(2021, 3, 3) + lettings_log.save!(validate: false) + task.invoke + expect(lettings_log.reload.ppcodenk).to eq(0) + end + end + end +end diff --git a/spec/lib/tasks/data_export_spec.rb b/spec/lib/tasks/data_export_spec.rb index 6865f56ef..5a08ac40a 100644 --- a/spec/lib/tasks/data_export_spec.rb +++ b/spec/lib/tasks/data_export_spec.rb @@ -28,21 +28,22 @@ describe "rake core:data_export", type: task do end context "when running full export" do - let(:storage_service) { instance_double(Storage::S3Service) } - let(:paas_config_service) { instance_double(Configuration::PaasConfigurationService) } - let(:export_service) { instance_double(Exports::LettingsLogExportService) } let(:task) { Rake::Task["core:full_data_export_xml"] } - before do - allow(Storage::S3Service).to receive(:new).and_return(storage_service) - allow(Configuration::PaasConfigurationService).to receive(:new).and_return(paas_config_service) - allow(Exports::LettingsLogExportService).to receive(:new).and_return(export_service) + context "with all available years" do + it "calls the export service" do + expect(export_service).to receive(:export_xml_lettings_logs).with(full_update: true, collection_year: nil) + + task.invoke + end end - it "calls the export service" do - expect(export_service).to receive(:export_xml_lettings_logs).with(full_update: true) + context "with a specific year" do + it "calls the export service" do + expect(export_service).to receive(:export_xml_lettings_logs).with(full_update: true, collection_year: 2022) - task.invoke + task.invoke("2022") + end end end end diff --git a/spec/models/form/question_spec.rb b/spec/models/form/question_spec.rb index 08d08032f..4c3579793 100644 --- a/spec/models/form/question_spec.rb +++ b/spec/models/form/question_spec.rb @@ -60,37 +60,9 @@ RSpec.describe Form::Question, type: :model do end it "has a yes value helper" do - expect(question).to be_value_is_yes("Yes", true) - expect(question).to be_value_is_yes("YES", true) - expect(question).not_to be_value_is_yes("random", true) - end - - it "has a no value helper" do - expect(question).to be_value_is_no("No", true) - expect(question).to be_value_is_no("NO", true) - expect(question).not_to be_value_is_no("random", true) - end - - context "when there are different value helper values for lettings and sales" do - context "with a lettings log" do - let(:lettings_log) { FactoryBot.build(:lettings_log, :in_progress) } - let(:question) { Form::Lettings::Questions::Ppcodenk.new(nil, nil, Form::Lettings::Pages::PreviousPostcode.new("previous_postcode", nil, Form::Lettings::Subsections::HouseholdSituation.new(nil, nil, Form::Lettings::Sections::Household))) } - - it "has the correct values" do - expect(question.value_is_yes?(1, lettings_log.lettings?)).to be true - expect(question.value_is_no?(0, lettings_log.lettings?)).to be true - end - end - - context "with a sales log" do - let(:sales_log) { FactoryBot.build(:sales_log, :in_progress) } - let(:question) { Form::Sales::Questions::PreviousPostcodeKnown.new(nil, nil, Form::Sales::Pages::LastAccommodation.new("previous_postcode", nil, Form::Sales::Subsections::HouseholdSituation.new(nil, nil, Form::Sales::Sections::Household))) } - - it "has the correct values" do - expect(question.value_is_yes?(0, sales_log.lettings?)).to be true - expect(question.value_is_no?(1, sales_log.lettings?)).to be true - end - end + expect(question).to be_value_is_yes("Yes") + expect(question).to be_value_is_yes("YES") + expect(question).not_to be_value_is_yes("random") end context "when type is numeric" do @@ -130,10 +102,8 @@ RSpec.describe Form::Question, type: :model do let(:question_id) { "illness" } it "maps those options" do - expect(question).to be_value_is_yes(1, true) - expect(question).not_to be_value_is_no(1, true) + expect(question).to be_value_is_yes(1) expect(question).not_to be_value_is_refused(1) - expect(question).to be_value_is_no(2, true) expect(question).to be_value_is_dont_know(3) end end @@ -145,8 +115,7 @@ RSpec.describe Form::Question, type: :model do let(:question_id) { "layear" } it "maps those options" do - expect(question).not_to be_value_is_yes(7, true) - expect(question).not_to be_value_is_no(7, true) + expect(question).not_to be_value_is_yes(7) expect(question).not_to be_value_is_refused(7) expect(question).to be_value_is_dont_know(7) end @@ -231,13 +200,8 @@ RSpec.describe Form::Question, type: :model do end it "can map yes values" do - expect(question).to be_value_is_yes(1, true) - expect(question).not_to be_value_is_yes(0, true) - end - - it "can map no values" do - expect(question).to be_value_is_no(0, true) - expect(question).not_to be_value_is_no(1, true) + expect(question).to be_value_is_yes(1) + expect(question).not_to be_value_is_yes(0) end end diff --git a/spec/models/lettings_log_spec.rb b/spec/models/lettings_log_spec.rb index e89015233..74a04f9e6 100644 --- a/spec/models/lettings_log_spec.rb +++ b/spec/models/lettings_log_spec.rb @@ -1329,7 +1329,7 @@ RSpec.describe LettingsLog do managing_organisation: owning_organisation, owning_organisation:, created_by: created_by_user, - ppcodenk: 1, + ppcodenk: 0, ppostcode_full: "M1 1AE", }) end @@ -1369,7 +1369,7 @@ RSpec.describe LettingsLog do end it "correctly resets all fields if previous postcode not known" do - address_lettings_log.update!({ ppcodenk: 0 }) + address_lettings_log.update!({ ppcodenk: 1 }) record_from_db = described_class.find(address_lettings_log.id) expect(record_from_db["ppostcode_full"]).to eq(nil) @@ -1378,7 +1378,7 @@ RSpec.describe LettingsLog do end it "correctly resets la if la is not known" do - address_lettings_log.update!({ ppcodenk: 0 }) + address_lettings_log.update!({ ppcodenk: 1 }) address_lettings_log.update!({ previous_la_known: 1, prevloc: "S92000003" }) record_from_db = described_class.find(address_lettings_log.id) expect(record_from_db["prevloc"]).to eq("S92000003") @@ -1391,7 +1391,7 @@ RSpec.describe LettingsLog do end it "changes the prevloc if previous postcode changes from not known to known and provided" do - address_lettings_log.update!({ ppcodenk: 0 }) + address_lettings_log.update!({ ppcodenk: 1 }) address_lettings_log.update!({ previous_la_known: 1, prevloc: "E09000033" }) record_from_db = described_class.find(address_lettings_log.id) diff --git a/spec/models/validations/household_validations_spec.rb b/spec/models/validations/household_validations_spec.rb index f44c5d255..b3e20779d 100644 --- a/spec/models/validations/household_validations_spec.rb +++ b/spec/models/validations/household_validations_spec.rb @@ -12,36 +12,6 @@ RSpec.describe Validations::HouseholdValidations do end describe "reasonable preference validations" do - context "when reasonable preference is homeless" do - context "when the tenant was not previously homeless" do - it "adds an error" do - record.homeless = 1 - record.rp_homeless = 1 - household_validator.validate_reasonable_preference(record) - expect(record.errors["reasonable_preference_reason"]) - .to include(match I18n.t("validations.household.reasonpref.not_homeless")) - expect(record.errors["homeless"]) - .to include(match I18n.t("validations.household.homeless.reasonpref.not_homeless")) - end - end - - context "when reasonable preference is given" do - context "when the tenant was previously homeless" do - it "does not add an error" do - record.homeless = 1 - record.reasonpref = 1 - household_validator.validate_reasonable_preference(record) - expect(record.errors["reasonpref"]).to be_empty - expect(record.errors["homeless"]).to be_empty - record.homeless = 0 - household_validator.validate_reasonable_preference(record) - expect(record.errors["reasonpref"]).to be_empty - expect(record.errors["homeless"]).to be_empty - end - end - end - end - context "when reasonable preference is not given" do it "validates that no reason is needed" do record.reasonpref = 1 diff --git a/spec/models/validations/local_authority_validations_spec.rb b/spec/models/validations/local_authority_validations_spec.rb index 1c0388c34..3b903e4dc 100644 --- a/spec/models/validations/local_authority_validations_spec.rb +++ b/spec/models/validations/local_authority_validations_spec.rb @@ -4,35 +4,35 @@ RSpec.describe Validations::LocalAuthorityValidations do subject(:local_auth_validator) { validator_class.new } let(:validator_class) { Class.new { include Validations::LocalAuthorityValidations } } - let(:record) { FactoryBot.create(:lettings_log) } + let(:log) { create(:lettings_log) } describe "#validate_previous_accommodation_postcode" do - it "does not add an error if the record ppostcode_full is missing" do - record.ppostcode_full = nil - local_auth_validator.validate_previous_accommodation_postcode(record) - expect(record.errors).to be_empty + it "does not add an error if the log ppostcode_full is missing" do + log.ppostcode_full = nil + local_auth_validator.validate_previous_accommodation_postcode(log) + expect(log.errors).to be_empty end - it "does not add an error if the record ppostcode_full is valid (uppercase space)" do - record.ppcodenk = 1 - record.ppostcode_full = "M1 1AE" - local_auth_validator.validate_previous_accommodation_postcode(record) - expect(record.errors).to be_empty + it "does not add an error if the log ppostcode_full is valid (uppercase space)" do + log.ppcodenk = 0 + log.ppostcode_full = "M1 1AE" + local_auth_validator.validate_previous_accommodation_postcode(log) + expect(log.errors).to be_empty end - it "does not add an error if the record ppostcode_full is valid (lowercase no space)" do - record.ppcodenk = 1 - record.ppostcode_full = "m11ae" - local_auth_validator.validate_previous_accommodation_postcode(record) - expect(record.errors).to be_empty + it "does not add an error if the log ppostcode_full is valid (lowercase no space)" do + log.ppcodenk = 0 + log.ppostcode_full = "m11ae" + local_auth_validator.validate_previous_accommodation_postcode(log) + expect(log.errors).to be_empty end it "does add an error when the postcode is invalid" do - record.ppcodenk = 1 - record.ppostcode_full = "invalid" - local_auth_validator.validate_previous_accommodation_postcode(record) - expect(record.errors).not_to be_empty - expect(record.errors["ppostcode_full"]).to include(match I18n.t("validations.postcode")) + log.ppcodenk = 0 + log.ppostcode_full = "invalid" + local_auth_validator.validate_previous_accommodation_postcode(log) + expect(log.errors).not_to be_empty + expect(log.errors["ppostcode_full"]).to include(match I18n.t("validations.postcode")) end end end diff --git a/spec/models/validations/sales/sale_information_validations_spec.rb b/spec/models/validations/sales/sale_information_validations_spec.rb index cccb19d40..dbd4c1e49 100644 --- a/spec/models/validations/sales/sale_information_validations_spec.rb +++ b/spec/models/validations/sales/sale_information_validations_spec.rb @@ -171,60 +171,6 @@ RSpec.describe Validations::Sales::SaleInformationValidations do end end - describe "#validate_years_living_in_property_before_purchase" do - context "when proplen blank" do - let(:record) { build(:sales_log, proplen: nil) } - - it "does not add an error" do - sale_information_validator.validate_years_living_in_property_before_purchase(record) - - expect(record.errors).not_to be_present - end - end - - context "when type blank" do - let(:record) { build(:sales_log, type: nil) } - - it "does not add an error" do - sale_information_validator.validate_years_living_in_property_before_purchase(record) - - expect(record.errors).not_to be_present - end - end - - context "when proplen 0" do - let(:record) { build(:sales_log, proplen: 0) } - - it "does not add an error" do - sale_information_validator.validate_years_living_in_property_before_purchase(record) - - expect(record.errors).not_to be_present - end - end - - context "when type Rent to Buy and proplen > 0" do - let(:record) { build(:sales_log, proplen: 1, type: 28) } - - it "adds an error" do - sale_information_validator.validate_years_living_in_property_before_purchase(record) - - expect(record.errors[:type]).to include(I18n.t("validations.sale_information.proplen.rent_to_buy")) - expect(record.errors[:proplen]).to include(I18n.t("validations.sale_information.proplen.rent_to_buy")) - end - end - - context "when type Social HomeBuy and proplen > 0" do - let(:record) { build(:sales_log, proplen: 1, type: 18) } - - it "adds an error" do - sale_information_validator.validate_years_living_in_property_before_purchase(record) - - expect(record.errors[:type]).to include(I18n.t("validations.sale_information.proplen.social_homebuy")) - expect(record.errors[:proplen]).to include(I18n.t("validations.sale_information.proplen.social_homebuy")) - end - end - end - describe "#validate_discounted_ownership_value" do context "when sale is on or after 24/25 collection window" do context "when grant is routed to" do diff --git a/spec/services/bulk_upload/lettings/log_creator_spec.rb b/spec/services/bulk_upload/lettings/log_creator_spec.rb index 03724577b..6dfaff46c 100644 --- a/spec/services/bulk_upload/lettings/log_creator_spec.rb +++ b/spec/services/bulk_upload/lettings/log_creator_spec.rb @@ -67,7 +67,7 @@ RSpec.describe BulkUpload::Lettings::LogCreator do waityear: 9, joint: 2, tenancy: 9, - ppcodenk: 0, + ppcodenk: 1, ) end @@ -153,7 +153,7 @@ RSpec.describe BulkUpload::Lettings::LogCreator do waityear: 9, joint: 2, tenancy: 9, - ppcodenk: 0, + ppcodenk: 1, ) end diff --git a/spec/services/bulk_upload/lettings/year2022/row_parser_spec.rb b/spec/services/bulk_upload/lettings/year2022/row_parser_spec.rb index 6b5eaa2ae..0a9f65d32 100644 --- a/spec/services/bulk_upload/lettings/year2022/row_parser_spec.rb +++ b/spec/services/bulk_upload/lettings/year2022/row_parser_spec.rb @@ -711,14 +711,6 @@ RSpec.describe BulkUpload::Lettings::Year2022::RowParser do end describe "#field_68 - 74" do - context "when not homeless but reasonable preference for homelessness" do - let(:attributes) { { bulk_upload:, field_68: "1", field_69: "1", field_70: "1" } } - - it "is not permitted" do - expect(parser.errors[:field_70]).to be_present - end - end - context "when there is a reasonable preference but none is given" do let(:attributes) { { bulk_upload:, field_69: "1", field_70: nil, field_71: nil, field_72: nil, field_73: nil, field_74: nil } } @@ -1562,7 +1554,7 @@ RSpec.describe BulkUpload::Lettings::Year2022::RowParser do let(:attributes) { { bulk_upload:, field_65: "2" } } it "sets correct value from mapping" do - expect(parser.log.ppcodenk).to eq(0) + expect(parser.log.ppcodenk).to eq(1) end end diff --git a/spec/services/bulk_upload/lettings/year2023/row_parser_spec.rb b/spec/services/bulk_upload/lettings/year2023/row_parser_spec.rb index cac166f46..a3b9152c5 100644 --- a/spec/services/bulk_upload/lettings/year2023/row_parser_spec.rb +++ b/spec/services/bulk_upload/lettings/year2023/row_parser_spec.rb @@ -803,14 +803,6 @@ RSpec.describe BulkUpload::Lettings::Year2023::RowParser do end describe "#field_105, field_110 - 15" do - context "when not homeless but reasonable preference for homelessness" do - let(:attributes) { { bulk_upload:, field_105: "1", field_110: "1", field_111: "1" } } - - it "is not permitted" do - expect(parser.errors[:field_111]).to be_present - end - end - context "when there is a reasonable preference but none is given" do let(:attributes) { { bulk_upload:, field_110: "1", field_111: nil, field_112: nil, field_113: nil, field_114: nil, field_115: nil } } @@ -1642,7 +1634,7 @@ RSpec.describe BulkUpload::Lettings::Year2023::RowParser do let(:attributes) { { bulk_upload:, field_106: "2" } } it "sets correct value from mapping" do - expect(parser.log.ppcodenk).to eq(0) + expect(parser.log.ppcodenk).to eq(1) end end diff --git a/spec/services/bulk_upload/processor_spec.rb b/spec/services/bulk_upload/processor_spec.rb index f01f2daf7..9f287c592 100644 --- a/spec/services/bulk_upload/processor_spec.rb +++ b/spec/services/bulk_upload/processor_spec.rb @@ -176,7 +176,7 @@ RSpec.describe BulkUpload::Processor do waityear: 9, joint: 2, tenancy: 9, - ppcodenk: 0, + ppcodenk: 1, voiddate: nil, mrcdate: nil, startdate: Date.new(2022, 10, 1), @@ -385,7 +385,7 @@ RSpec.describe BulkUpload::Processor do waityear: 9, joint: 2, tenancy: 2, - ppcodenk: 0, + ppcodenk: 1, voiddate: Date.new(2022, 1, 1), reason: 40, leftreg: 3, diff --git a/spec/services/exports/lettings_log_export_service_spec.rb b/spec/services/exports/lettings_log_export_service_spec.rb index c13396c52..7b9f55b93 100644 --- a/spec/services/exports/lettings_log_export_service_spec.rb +++ b/spec/services/exports/lettings_log_export_service_spec.rb @@ -219,6 +219,16 @@ RSpec.describe Exports::LettingsLogExportService do export_service.export_xml_lettings_logs end + + it "generates zip export files only for specified year" do + expect(storage_service).to receive(:write_file).with(expected_zip_filename2, any_args) + expect(Rails.logger).to receive(:info).with("Building export run for 2022") + expect(Rails.logger).to receive(:info).with("Creating core_2022_2023_apr_mar_f0001_inc0001 - 1 logs") + expect(Rails.logger).to receive(:info).with("Added core_2022_2023_apr_mar_f0001_inc0001_pt001.xml") + expect(Rails.logger).to receive(:info).with("Writing core_2022_2023_apr_mar_f0001_inc0001.zip") + + export_service.export_xml_lettings_logs(collection_year: 2022) + end end end @@ -318,6 +328,26 @@ RSpec.describe Exports::LettingsLogExportService do expect(LogsExport.last.increment_number).to eq(1) end end + + context "and a log has been migrated since the previous partial export" do + before do + FactoryBot.create(:lettings_log, startdate: Time.zone.local(2022, 2, 1), updated_at: Time.zone.local(2022, 4, 27), values_updated_at: Time.zone.local(2022, 4, 29)) + FactoryBot.create(:lettings_log, startdate: Time.zone.local(2022, 2, 1), updated_at: Time.zone.local(2022, 4, 27), values_updated_at: Time.zone.local(2022, 4, 29)) + LogsExport.create!(started_at: Time.zone.local(2022, 4, 28), base_number: 1, increment_number: 1) + end + + it "generates an XML manifest file with the expected content within the ZIP file" do + expected_content = replace_record_number(local_manifest_file.read, 2) + expect(storage_service).to receive(:write_file).with(expected_master_manifest_rerun, any_args) + expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) do |_, content| + entry = Zip::File.open_buffer(content).find_entry(expected_manifest_filename) + expect(entry).not_to be_nil + expect(entry.get_input_stream.read).to eq(expected_content) + end + + export_service.export_xml_lettings_logs + end + end end context "when exporting a supported housing lettings logs in XML" do diff --git a/spec/services/imports/import_report_service_spec.rb b/spec/services/imports/import_report_service_spec.rb index 1d241754d..abd318a8d 100644 --- a/spec/services/imports/import_report_service_spec.rb +++ b/spec/services/imports/import_report_service_spec.rb @@ -67,4 +67,35 @@ RSpec.describe Imports::ImportReportService do report_service.generate_logs_report("report_suffix.csv") end end + + describe "#generate_unassigned_logs_report" do + context "when there is no unassigned user (all the logs have ben assigned)" do + let(:institutions_csv) { CSV.parse("Institution name,Id,Old Completed lettings logs,Old In progress lettings logs,Old Completed sales logs,Old In progress sales logs\norg1,1,2,1,4,3", headers: true) } + + it "writes an empty unassigned logs report" do + expect(storage_service).to receive(:write_file).with("UnassignedLogsReport_report_suffix.csv", "\uFEFFOwning Organisation ID,Old Owning Organisation ID,Managing Organisation ID,Old Managing Organisation ID,Log ID,Old Log ID,Tenancy code,Purchaser code\n") + + report_service.generate_unassigned_logs_report("report_suffix.csv") + end + end + + context "when some logs have been added to Unassigned user" do + let(:organisation) { create(:organisation, old_org_id: "1", name: "org1") } + let(:organisation2) { create(:organisation, old_org_id: "2", name: "org2") } + let(:unassigned_user) { create(:user, name: "Unassigned", organisation:) } + let(:institutions_csv) { CSV.parse("Institution name,Id,Old Completed lettings logs,Old In progress lettings logs,Old Completed sales logs,Old In progress sales logs\norg1,1,2,1,4,3", headers: true) } + let!(:lettings_log) { create(:lettings_log, owning_organisation: organisation, managing_organisation: organisation2, created_by: unassigned_user, tenancycode: "tenancycode", old_id: "12") } + let!(:sales_log) { create(:sales_log, owning_organisation: organisation, created_by: unassigned_user, purchid: "purchid", old_id: "23") } + + before do + create(:organisation_relationship, parent_organisation: organisation, child_organisation: organisation2) + end + + it "writes a report with all unassigned logs" do + expect(storage_service).to receive(:write_file).with("UnassignedLogsReport_report_suffix.csv", "\uFEFFOwning Organisation ID,Old Owning Organisation ID,Managing Organisation ID,Old Managing Organisation ID,Log ID,Old Log ID,Tenancy code,Purchaser code\n#{organisation.id},1,#{organisation2.id},2,#{lettings_log.id},12,tenancycode,\n#{organisation.id},1,,,#{sales_log.id},23,,purchid\n") + + report_service.generate_unassigned_logs_report("report_suffix.csv") + end + end + end end diff --git a/spec/services/imports/lettings_logs_import_service_spec.rb b/spec/services/imports/lettings_logs_import_service_spec.rb index 07fd6996e..1025fcd3b 100644 --- a/spec/services/imports/lettings_logs_import_service_spec.rb +++ b/spec/services/imports/lettings_logs_import_service_spec.rb @@ -135,6 +135,50 @@ RSpec.describe Imports::LettingsLogsImportService do let(:lettings_log_file) { open_file(fixture_directory, lettings_log_id) } let(:lettings_log_xml) { Nokogiri::XML(lettings_log_file) } + context "and the user does not exist" do + before { lettings_log_xml.at_xpath("//meta:owner-user-id").content = "fake_id" } + + it "creates a new unassigned user" do + expect(logger).to receive(:error).with("Lettings log '0ead17cb-1668-442d-898c-0d52879ff592' belongs to legacy user with owner-user-id: 'fake_id' which cannot be found. Assigning log to 'Unassigned' user.") + lettings_log_service.send(:create_log, lettings_log_xml) + + lettings_log = LettingsLog.where(old_id: lettings_log_id).first + expect(lettings_log&.created_by&.name).to eq("Unassigned") + end + + it "only creates one unassigned user" do + expect(logger).to receive(:error).with("Lettings log '0ead17cb-1668-442d-898c-0d52879ff592' belongs to legacy user with owner-user-id: 'fake_id' which cannot be found. Assigning log to 'Unassigned' user.") + expect(logger).to receive(:error).with("Lettings log 'fake_id' belongs to legacy user with owner-user-id: 'fake_id' which cannot be found. Assigning log to 'Unassigned' user.") + lettings_log_service.send(:create_log, lettings_log_xml) + lettings_log_xml.at_xpath("//meta:document-id").content = "fake_id" + lettings_log_service.send(:create_log, lettings_log_xml) + + lettings_log = LettingsLog.where(old_id: lettings_log_id).first + second_lettings_log = LettingsLog.where(old_id: "fake_id").first + expect(lettings_log&.created_by).to eq(second_lettings_log&.created_by) + end + + context "when unassigned user exist for a different organisation" do + let!(:other_unassigned_user) { create(:user, name: "Unassigned") } + + it "creates a new unassigned user for current organisation" do + expect(logger).to receive(:error).with("Lettings log '0ead17cb-1668-442d-898c-0d52879ff592' belongs to legacy user with owner-user-id: 'fake_id' which cannot be found. Assigning log to 'Unassigned' user.") + lettings_log_service.send(:create_log, lettings_log_xml) + + lettings_log = LettingsLog.where(old_id: lettings_log_id).first + expect(lettings_log&.created_by&.name).to eq("Unassigned") + expect(lettings_log&.created_by).not_to eq(other_unassigned_user) + end + end + end + + it "correctly sets imported at date" do + lettings_log_service.send(:create_log, lettings_log_xml) + + lettings_log = LettingsLog.where(old_id: lettings_log_id).first + expect(lettings_log&.values_updated_at).to eq(Time.zone.local(2022, 1, 1)) + end + context "and the void date is after the start date" do before { lettings_log_xml.at_xpath("//xmlns:VYEAR").content = 2023 } @@ -442,6 +486,63 @@ RSpec.describe Imports::LettingsLogsImportService do end end + context "and the number of times the property was relet is 0.00" do + before do + lettings_log_xml.at_xpath("//xmlns:Q20").content = "0.00" + end + + it "does not raise an error" do + expect { lettings_log_service.send(:create_log, lettings_log_xml) } + .not_to raise_error + end + + it "does not clear offered answer" do + lettings_log_service.send(:create_log, lettings_log_xml) + lettings_log = LettingsLog.find_by(old_id: lettings_log_id) + + expect(lettings_log).not_to be_nil + expect(lettings_log.offered).to equal(0) + end + end + + context "and the gender identity is refused" do + before do + lettings_log_xml.at_xpath("//xmlns:P1Sex").content = "Person prefers not to say" + end + + it "does not raise an error" do + expect { lettings_log_service.send(:create_log, lettings_log_xml) } + .not_to raise_error + end + + it "saves the correct answer" do + lettings_log_service.send(:create_log, lettings_log_xml) + lettings_log = LettingsLog.find_by(old_id: lettings_log_id) + + expect(lettings_log).not_to be_nil + expect(lettings_log.sex1).to eq("R") + end + end + + context "and the relationship is refused" do + before do + lettings_log_xml.at_xpath("//xmlns:P2Rel").content = "Person prefers not to say" + end + + it "does not raise an error" do + expect { lettings_log_service.send(:create_log, lettings_log_xml) } + .not_to raise_error + end + + it "saves the correct answer" do + lettings_log_service.send(:create_log, lettings_log_xml) + lettings_log = LettingsLog.find_by(old_id: lettings_log_id) + + expect(lettings_log).not_to be_nil + expect(lettings_log.relat2).to eq("R") + end + end + context "when the log being imported was manually entered" do it "sets the creation method correctly" do lettings_log_service.send(:create_log, lettings_log_xml) diff --git a/spec/services/imports/organisation_import_service_spec.rb b/spec/services/imports/organisation_import_service_spec.rb index 41e843e3c..fd02d8eda 100644 --- a/spec/services/imports/organisation_import_service_spec.rb +++ b/spec/services/imports/organisation_import_service_spec.rb @@ -8,11 +8,17 @@ RSpec.describe Imports::OrganisationImportService do let(:filenames) { %w[my_folder/my_file1.xml my_folder/my_file2.xml] } let(:fixture_directory) { "spec/fixtures/imports/institution" } - def create_organisation_file(fixture_directory, visible_id, name = nil) - file = File.open("#{fixture_directory}/7c5bd5fb549c09a2c55d7cb90d7ba84927e64618.xml") + def create_organisation_file(fixture_directory, visible_id, filename, namespace_given, name = nil) + file = File.open("#{fixture_directory}/#{filename}.xml") doc = Nokogiri::XML(file) - doc.at_xpath("//institution:visible-id").content = visible_id if visible_id - doc.at_xpath("//institution:name").content = name if name + if namespace_given + doc.at_xpath("//institution:visible-id").content = visible_id if visible_id + doc.at_xpath("//institution:name").content = name if name + else + doc.at_xpath("//dclg:visible-id", { "dclg" => "dclg:institution" }).content = visible_id if visible_id + doc.at_xpath("//dclg:name", { "dclg" => "dclg:institution" }).content = name if name + end + StringIO.new(doc.to_xml) end @@ -24,10 +30,10 @@ RSpec.describe Imports::OrganisationImportService do .and_return(filenames) allow(storage_service).to receive(:get_file_io) .with(filenames[0]) - .and_return(create_organisation_file(fixture_directory, 1)) + .and_return(create_organisation_file(fixture_directory, 1, "7c5bd5fb549c09a2c55d7cb90d7ba84927e64618", true)) allow(storage_service).to receive(:get_file_io) .with(filenames[1]) - .and_return(create_organisation_file(fixture_directory, 2)) + .and_return(create_organisation_file(fixture_directory, 2, "7c5bd5fb549c09a2c55d7cb90d7ba84927e64618", true)) end it "successfully create an organisation with the expected data" do @@ -74,8 +80,8 @@ RSpec.describe Imports::OrganisationImportService do before do allow(storage_service).to receive(:list_files).and_return([filenames[0]]) allow(storage_service).to receive(:get_file_io).and_return( - create_organisation_file(fixture_directory, 1), - create_organisation_file(fixture_directory, 1, "my_new_organisation"), + create_organisation_file(fixture_directory, 1, "7c5bd5fb549c09a2c55d7cb90d7ba84927e64618", true), + create_organisation_file(fixture_directory, 1, "7c5bd5fb549c09a2c55d7cb90d7ba84927e64618", true, "my_new_organisation"), ) end @@ -92,4 +98,56 @@ RSpec.describe Imports::OrganisationImportService do expect(Organisation).to exist(old_visible_id: "1", name: "HA Ltd") end end + + context "when importing organisations with no namespace" do + subject(:import_service) { described_class.new(storage_service) } + + before do + allow(storage_service).to receive(:list_files) + .and_return(filenames) + allow(storage_service).to receive(:get_file_io) + .with(filenames[0]) + .and_return(create_organisation_file(fixture_directory, 1, "8c5bd5fb549c09a2c55d7cb90d7ba84927e64618", false)) + allow(storage_service).to receive(:get_file_io) + .with(filenames[1]) + .and_return(create_organisation_file(fixture_directory, 2, "8c5bd5fb549c09a2c55d7cb90d7ba84927e64618", false)) + end + + it "successfully create an organisation with the expected data" do + import_service.create_organisations(folder_name) + + organisation = Organisation.find_by(old_visible_id: "1") + expect(organisation.name).to eq("HA Ltd") + expect(organisation.provider_type).to eq("PRP") + expect(organisation.phone).to eq("xxxxxxxx") + expect(organisation.holds_own_stock).to be_truthy + expect(organisation.active).to be_truthy + # expect(organisation.old_association_type).to eq() string VS integer + # expect(organisation.software_supplier_id).to eq() boolean VS string + expect(organisation.housing_management_system).to eq("") # Need examples + expect(organisation.choice_based_lettings).to be_falsey + expect(organisation.common_housing_register).to be_falsey + expect(organisation.choice_allocation_policy).to be_falsey + expect(organisation.cbl_proportion_percentage).to be_nil # Need example + expect(organisation.enter_affordable_logs).to be_truthy + expect(organisation.owns_affordable_logs).to be_truthy # owns_affordable_rent + expect(organisation.housing_registration_no).to eq("LH9999") + expect(organisation.general_needs_units).to eq(1104) + expect(organisation.supported_housing_units).to eq(217) + expect(organisation.unspecified_units).to eq(0) + expect(organisation.unspecified_units).to eq(0) + expect(organisation.old_org_id).to eq("8c5bd5fb549c09z2c55d9cb90d7ba84927e64618") + expect(organisation.old_visible_id).to eq("1") + end + + it "successfully create multiple organisations" do + expect(storage_service).to receive(:list_files).with(folder_name) + expect(storage_service).to receive(:get_file_io).with(filenames[0]).ordered + expect(storage_service).to receive(:get_file_io).with(filenames[1]).ordered + + expect { import_service.create_organisations(folder_name) }.to change(Organisation, :count).by(2) + expect(Organisation).to exist(old_visible_id: "1") + expect(Organisation).to exist(old_visible_id: "2") + end + end end diff --git a/spec/services/imports/sales_logs_import_service_spec.rb b/spec/services/imports/sales_logs_import_service_spec.rb index 740614775..bbabc5c1a 100644 --- a/spec/services/imports/sales_logs_import_service_spec.rb +++ b/spec/services/imports/sales_logs_import_service_spec.rb @@ -102,9 +102,17 @@ RSpec.describe Imports::SalesLogsImportService do end context "when importing a specific log" do + let(:sales_log_id) { "shared_ownership_sales_log" } let(:sales_log_file) { open_file(fixture_directory, sales_log_id) } let(:sales_log_xml) { Nokogiri::XML(sales_log_file) } + it "correctly sets values updated at date" do + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.where(old_id: sales_log_id).first + expect(sales_log&.values_updated_at).to eq(Time.zone.local(2023, 2, 1)) + end + context "and the organisation legacy ID does not exist" do let(:sales_log_id) { "shared_ownership_sales_log" } @@ -116,6 +124,45 @@ RSpec.describe Imports::SalesLogsImportService do end end + context "and the user does not exist" do + let(:sales_log_id) { "shared_ownership_sales_log" } + + before { sales_log_xml.at_xpath("//meta:owner-user-id").content = "fake_id" } + + it "creates a new unassigned user" do + expect(logger).to receive(:error).with("Sales log 'shared_ownership_sales_log' belongs to legacy user with owner-user-id: 'fake_id' which cannot be found. Assigning log to 'Unassigned' user.") + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.where(old_id: sales_log_id).first + expect(sales_log&.created_by&.name).to eq("Unassigned") + end + + it "only creates one unassigned user" do + expect(logger).to receive(:error).with("Sales log 'shared_ownership_sales_log' belongs to legacy user with owner-user-id: 'fake_id' which cannot be found. Assigning log to 'Unassigned' user.") + expect(logger).to receive(:error).with("Sales log 'fake_id' belongs to legacy user with owner-user-id: 'fake_id' which cannot be found. Assigning log to 'Unassigned' user.") + sales_log_service.send(:create_log, sales_log_xml) + sales_log_xml.at_xpath("//meta:document-id").content = "fake_id" + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.where(old_id: sales_log_id).first + second_sales_log = SalesLog.where(old_id: "fake_id").first + expect(sales_log&.created_by).to eq(second_sales_log&.created_by) + end + + context "when unassigned user exist for a different organisation" do + let!(:other_unassigned_user) { create(:user, name: "Unassigned") } + + it "creates a new unassigned user for current organisation" do + expect(logger).to receive(:error).with("Sales log 'shared_ownership_sales_log' belongs to legacy user with owner-user-id: 'fake_id' which cannot be found. Assigning log to 'Unassigned' user.") + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.where(old_id: sales_log_id).first + expect(sales_log&.created_by&.name).to eq("Unassigned") + expect(sales_log&.created_by).not_to eq(other_unassigned_user) + end + end + end + context "and the log startdate is before 22/23 collection period" do let(:sales_log_id) { "shared_ownership_sales_log" } diff --git a/spec/services/imports/user_import_service_spec.rb b/spec/services/imports/user_import_service_spec.rb index ea3388169..5e56b2782 100644 --- a/spec/services/imports/user_import_service_spec.rb +++ b/spec/services/imports/user_import_service_spec.rb @@ -44,10 +44,24 @@ RSpec.describe Imports::UserImportService do end it "refuses to create a user belonging to a non existing organisation" do - expect(logger).to receive(:error).with(/ActiveRecord::RecordInvalid/) + expect(logger).to receive(:error).with(/Could not save user with email: john.doe@gov.uk/) + expect(logger).to receive(:error).with(/Validation failed: Organisation Select the user’s organisation/) import_service.create_users("user_directory") end + context "when the user with the same email already exists" do + before do + create(:organisation, old_org_id:) + create(:user, email: "john.doe@gov.uk") + end + + it "logs an error and user email" do + expect(logger).to receive(:error).with(/Could not save user with email: john.doe@gov.uk/) + expect(logger).to receive(:error).with(/Validation failed: email Enter an email address that hasn’t already been used to sign up/) + import_service.create_users("user_directory") + end + end + context "when the user is a data coordinator" do let(:old_user_id) { "d4729b1a5dfb68bb1e01c08445830c0add40907c" }