From e0385abad10db165e74825584b3d52eb489b50d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Meny?= Date: Thu, 26 May 2022 11:29:37 +0100 Subject: [PATCH] Updates exported fields based on May 25th feedback (#613) --- .../exports/case_log_export_constants.rb | 138 ++++++++++++++++++ .../exports/case_log_export_service.rb | 59 +++++--- spec/factories/organisation.rb | 1 + spec/fixtures/exports/case_logs.csv | 4 +- spec/fixtures/exports/case_logs.xml | 46 ++---- .../exports/case_log_export_service_spec.rb | 5 +- 6 files changed, 194 insertions(+), 59 deletions(-) create mode 100644 app/services/exports/case_log_export_constants.rb diff --git a/app/services/exports/case_log_export_constants.rb b/app/services/exports/case_log_export_constants.rb new file mode 100644 index 000000000..dfc1e66e8 --- /dev/null +++ b/app/services/exports/case_log_export_constants.rb @@ -0,0 +1,138 @@ +module Exports::CaseLogExportConstants + MAX_XML_RECORDS = 10_000 + LOG_ID_OFFSET = 300_000_000_000 + + QUARTERS = { + 0 => "jan_mar", + 1 => "apr_jun", + 2 => "jul_sep", + 3 => "oct_dec", + }.freeze + + EXPORT_FIELDS = Set[ + "armedforces", + "beds", + "benefits", + "brent", + "builtype", + "cap", + "cbl", + "chcharge", + "chr", + "cligrp1", + "cligrp2", + "createddate", # New metadata coming from our system + "confidential", + "earnings", + "ethnic", + "form", + "has_benefits", + "hb", + "hbrentshortfall", + "hcnum", + "hhmemb", + "homeless", + "housingneeds", + "illness", + "incfreq", + "income", + "incref", + "intstay", + "irproduct", + "irproduct_other", + "joint", + "la", + "lar", + "layear", + "leftreg", + "lettype", + "manhcnum", + "maningorgid", + "maningorgname", + "mobstand", + "mrcdate", + "national", + "needstype", + "new_old", + "newprop", + "nocharge", + "offered", + "owningorgid", + "owningorgname", + "period", + "postcode_full", + "ppcodenk", + "ppostcode_full", + "preg_occ", + "prevloc", + "prevten", + "propcode", + "providertype", + "pscharge", + "reason", + "reasonother", + "reasonpref", + "referral", + "refused", + "reghome", + "renttype", + "renewal", + "reservist", + "rp_dontknow", + "rp_hardship", + "rp_homeless", + "rp_insan_unsat", + "rp_medwel", + "rsnvac", + "scharge", + "scheme", + "schtype", + "shelteredaccom", + "startdate", + "startertenancy", + "supcharg", + "support", + "status", # New metadata coming from our system + "tcharge", + "tshortfall", + "tenancy", + "tenancycode", + "tenancylength", + "tenancyother", + "totadult", + "totchild", + "totelder", + "underoccupation_benefitcap", + "unitletas", + "units", + "unittype_gn", + "unittype_sh", + "uploaddate", + "username", + "voiddate", + "waityear", + "wchair", + "wchchrg", + "wpschrge", + "wrent", + "wscharge", + "wsupchrg", + "wtcharge", + "wtshortfall", + ] + + (1..8).each do |index| + EXPORT_FIELDS << "age#{index}" + EXPORT_FIELDS << "ecstat#{index}" + EXPORT_FIELDS << "sex#{index}" + end + (2..8).each do |index| + EXPORT_FIELDS << "relat#{index}" + end + (1..10).each do |index| + EXPORT_FIELDS << "illness_type_#{index}" + end + %w[a b c d e f g h].each do |letter| + EXPORT_FIELDS << "housingneeds_#{letter}" + end +end diff --git a/app/services/exports/case_log_export_service.rb b/app/services/exports/case_log_export_service.rb index 54f0b96ed..33429e397 100644 --- a/app/services/exports/case_log_export_service.rb +++ b/app/services/exports/case_log_export_service.rb @@ -1,14 +1,6 @@ module Exports class CaseLogExportService - QUARTERS = { - 0 => "jan_mar", - 1 => "apr_jun", - 2 => "jul_sep", - 3 => "oct_dec", - }.freeze - - LOG_ID_OFFSET = 300_000_000_000 - MAX_XML_RECORDS = 10_000 + include Exports::CaseLogExportConstants def initialize(storage_service, logger = Rails.logger) @storage_service = storage_service @@ -158,14 +150,43 @@ module Exports xml_doc_to_temp_file(doc) end - def apply_cds_transformation!(attribute_hash) - # OLD_CORE FORM ID support - attribute_hash["form"] = attribute_hash["old_form_id"] - attribute_hash["newform"] = attribute_hash["id"] + LOG_ID_OFFSET + def apply_cds_transformation(case_log) + attribute_hash = case_log.attributes_before_type_cast + + attribute_hash["form"] = attribute_hash["old_form_id"] || (attribute_hash["id"] + LOG_ID_OFFSET) + + # Changes specific to collection start year + if case_log.collection_start_year == 2021 + attribute_hash.delete("joint") + end + if case_log.collection_start_year == 2022 + attribute_hash.delete("underoccupation_benefitcap") + end + + # Organisation fields + if case_log.owning_organisation.present? + attribute_hash["owningorgid"] = case_log.owning_organisation.old_visible_id || (case_log.owning_organisation.id + LOG_ID_OFFSET) + attribute_hash["owningorgname"] = case_log.owning_organisation.name + attribute_hash["hcnum"] = case_log.owning_organisation.housing_registration_no + end + if case_log.managing_organisation.present? + attribute_hash["maningorgid"] = case_log.managing_organisation.old_visible_id || (case_log.managing_organisation.id + LOG_ID_OFFSET) + attribute_hash["maningorgname"] = case_log.managing_organisation.name + attribute_hash["manhcnum"] = case_log.managing_organisation.housing_registration_no + end + + # Mapping which would require a change in our data model + attribute_hash["createddate"] = attribute_hash["created_at"] + attribute_hash["uploaddate"] = attribute_hash["updated_at"] + attribute_hash["tenancycode"] = attribute_hash["tenancy_code"] + attribute_hash["ppcodenk"] = attribute_hash["previous_postcode_known"] + # Age refused (1..8).each do |index| attribute_hash["age#{index}"] = -9 if attribute_hash["age#{index}_known"] == 1 end + + attribute_hash end def filter_keys!(attributes) @@ -173,17 +194,18 @@ module Exports end def is_omitted_field?(field_name) - omitted_attrs = %w[id old_form_id old_id ethnic_group] pattern_age = /age\d_known/ - field_name.starts_with?("details_known_") || pattern_age.match(field_name) || omitted_attrs.include?(field_name) ? true : false + details_known_prefix = "details_known_" + field_name.starts_with?(details_known_prefix) || + pattern_age.match(field_name) || + !EXPORT_FIELDS.include?(field_name) end def build_export_csv(case_logs) csv_io = CSV.generate do |csv| attribute_keys = nil case_logs.each do |case_log| - attribute_hash = case_log.attributes_before_type_cast - apply_cds_transformation!(attribute_hash) + attribute_hash = apply_cds_transformation(case_log) if attribute_keys.nil? attribute_keys = attribute_hash.keys filter_keys!(attribute_keys) @@ -200,8 +222,7 @@ module Exports doc = Nokogiri::XML("") case_logs.each do |case_log| - attribute_hash = case_log.attributes_before_type_cast - apply_cds_transformation!(attribute_hash) + attribute_hash = apply_cds_transformation(case_log) form = doc.create_element("form") doc.at("forms") << form attribute_hash.each do |key, value| diff --git a/spec/factories/organisation.rb b/spec/factories/organisation.rb index 981217736..e88abf0c5 100644 --- a/spec/factories/organisation.rb +++ b/spec/factories/organisation.rb @@ -4,6 +4,7 @@ FactoryBot.define do address_line1 { "2 Marsham Street" } address_line2 { "London" } provider_type { "LA" } + housing_registration_no { "1234" } postcode { "SW1P 4DF" } created_at { Time.zone.now } updated_at { Time.zone.now } diff --git a/spec/fixtures/exports/case_logs.csv b/spec/fixtures/exports/case_logs.csv index 5817cf75f..a3e2cea0d 100644 --- a/spec/fixtures/exports/case_logs.csv +++ b/spec/fixtures/exports/case_logs.csv @@ -1,2 +1,2 @@ -status,created_at,updated_at,tenant_code,age1,sex1,ethnic,national,prevten,ecstat1,hhmemb,age2,sex2,ecstat2,age3,sex3,ecstat3,age4,sex4,ecstat4,age5,sex5,ecstat5,age6,sex6,ecstat6,age7,sex7,ecstat7,age8,sex8,ecstat8,homeless,underoccupation_benefitcap,leftreg,reservist,illness,preg_occ,tenancy_code,startertenancy,tenancylength,tenancy,ppostcode_full,rsnvac,unittype_gn,beds,offered,wchair,earnings,incfreq,benefits,period,layear,waityear,postcode_full,reasonpref,cbl,chr,cap,reasonother,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,illness_type_1,illness_type_2,illness_type_3,illness_type_4,illness_type_8,illness_type_5,illness_type_6,illness_type_7,illness_type_9,illness_type_10,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,tenancyother,net_income_value_check,property_owner_organisation,property_manager_organisation,sale_or_letting,irproduct_other,purchaser_code,reason,propcode,majorrepairs,la,prevloc,hb,hbrentshortfall,property_relet,mrcdate,incref,sale_completion_date,startdate,armedforces,first_time_property_let_as_social_housing,unitletas,builtype,voiddate,owning_organisation_id,managing_organisation_id,renttype,needstype,lettype,postcode_known,is_la_inferred,totchild,totelder,totadult,net_income_known,nocharge,is_carehome,household_charge,referral,brent,scharge,pscharge,supcharg,tcharge,tshortfall,chcharge,declaration,previous_postcode_known,previous_la_known,is_previous_la_inferred,ethnic_other,letting_allocation_unknown,rent_type,has_benefits,renewal,wrent,wscharge,wpschrge,wsupchrg,wtcharge,wtshortfall,refused,housingneeds,wchchrg,newprop,relat2,relat3,relat4,relat5,relat6,relat7,relat8,rent_value_check,lar,irproduct,joint,created_by_id,illness_type_0,tshortfall_known,shelteredaccom,retirement_value_check,form,newform -2,2022-02-08 16:52:15 UTC,2022-02-08 16:52:15 UTC,BZ737,35,F,2,4,6,0,2,32,M,6,,,,,,,,,,,,,,,,,,,1,0,1,0,1,2,BZ757,0,5,1,SE26RT,6,7,3,2,1,68,1,1,2,2,1,NW15TY,1,1,1,0,,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,,,Test,Test,,,798794,4,123,1,E09000003,E07000105,6,1,0,2020-05-05 10:36:49 UTC,0,,2022-02-02 10:36:49 UTC,1,,2,1,2019-11-03 00:00:00 UTC,{owning_org_id},{managing_org_id},2,1,7,1,false,0,0,2,1,0,0,,,200.0,50.0,40.0,35.0,325.0,12.0,,1,1,1,false,,,1,1,0,100.0,25.0,20.0,17.5,162.5,6.0,0,1,,2,P,,,,,,,,,,,{created_by_id},,0,0,,,{id} +status,age1,sex1,ethnic,national,prevten,ecstat1,hhmemb,age2,sex2,ecstat2,age3,sex3,ecstat3,age4,sex4,ecstat4,age5,sex5,ecstat5,age6,sex6,ecstat6,age7,sex7,ecstat7,age8,sex8,ecstat8,homeless,underoccupation_benefitcap,leftreg,reservist,illness,preg_occ,startertenancy,tenancylength,tenancy,ppostcode_full,rsnvac,unittype_gn,beds,offered,wchair,earnings,incfreq,benefits,period,layear,waityear,postcode_full,reasonpref,cbl,chr,cap,reasonother,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,illness_type_1,illness_type_2,illness_type_3,illness_type_4,illness_type_8,illness_type_5,illness_type_6,illness_type_7,illness_type_9,illness_type_10,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,tenancyother,irproduct_other,reason,propcode,la,prevloc,hb,hbrentshortfall,mrcdate,incref,startdate,armedforces,unitletas,builtype,voiddate,renttype,needstype,lettype,totchild,totelder,totadult,nocharge,referral,brent,scharge,pscharge,supcharg,tcharge,tshortfall,chcharge,has_benefits,renewal,wrent,wscharge,wpschrge,wsupchrg,wtcharge,wtshortfall,refused,housingneeds,wchchrg,newprop,relat2,relat3,relat4,relat5,relat6,relat7,relat8,lar,irproduct,shelteredaccom,form,owningorgid,owningorgname,hcnum,maningorgid,maningorgname,manhcnum,createddate,uploaddate,tenancycode,ppcodenk +2,35,F,2,4,6,0,2,32,M,6,,,,,,,,,,,,,,,,,,,1,0,1,0,1,2,0,5,1,SE26RT,6,7,3,2,1,68,1,1,2,2,1,NW15TY,1,1,1,0,,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,,,4,123,E09000003,E07000105,6,1,2020-05-05 10:36:49 UTC,0,2022-02-02 10:36:49 UTC,1,2,1,2019-11-03 00:00:00 UTC,2,1,7,0,0,2,0,,200.0,50.0,40.0,35.0,325.0,12.0,,1,0,100.0,25.0,20.0,17.5,162.5,6.0,0,1,,2,P,,,,,,,,,0,{id},{owning_org_id},DLUHC,1234,{managing_org_id},DLUHC,1234,2022-02-08 16:52:15 UTC,2022-02-08 16:52:15 UTC,BZ757,1 diff --git a/spec/fixtures/exports/case_logs.xml b/spec/fixtures/exports/case_logs.xml index e2110fefd..1b899d444 100644 --- a/spec/fixtures/exports/case_logs.xml +++ b/spec/fixtures/exports/case_logs.xml @@ -2,9 +2,6 @@
2 - 2022-02-08 16:52:15 UTC - 2022-02-08 16:52:15 UTC - BZ737 35 F 2 @@ -39,7 +36,6 @@ 0 1 2 - BZ757 0 5 1 @@ -83,43 +79,27 @@ 0 0 - - Test - Test - - 798794 4 123 - 1 E09000003 E07000105 6 1 - 0 2020-05-05 10:36:49 UTC 0 - 2022-02-02 10:36:49 UTC 1 - 2 1 2019-11-03 00:00:00 UTC - {owning_org_id} - {managing_org_id} 2 1 7 - 1 - false 0 0 2 - 1 0 - 0 - 200.0 50.0 @@ -128,13 +108,6 @@ 325.0 12.0 - 1 - 1 - 1 - false - - - 1 1 0 100.0 @@ -154,17 +127,20 @@ - - - {created_by_id} - - 0 0 - - - {id} + {id} + {owning_org_id} + DLUHC + 1234 + {managing_org_id} + DLUHC + 1234 + 2022-02-08 16:52:15 UTC + 2022-02-08 16:52:15 UTC + BZ757 + 1 1
diff --git a/spec/services/exports/case_log_export_service_spec.rb b/spec/services/exports/case_log_export_service_spec.rb index d3cc2c2f8..0e5f7095e 100644 --- a/spec/services/exports/case_log_export_service_spec.rb +++ b/spec/services/exports/case_log_export_service_spec.rb @@ -16,9 +16,8 @@ RSpec.describe Exports::CaseLogExportService do def replace_entity_ids(case_log, export_template) export_template.sub!(/\{id\}/, (case_log["id"] + Exports::CaseLogExportService::LOG_ID_OFFSET).to_s) - export_template.sub!(/\{owning_org_id\}/, case_log["owning_organisation_id"].to_s) - export_template.sub!(/\{managing_org_id\}/, case_log["managing_organisation_id"].to_s) - export_template.sub!(/\{created_by_id\}/, case_log["created_by_id"].to_s) + export_template.sub!(/\{owning_org_id\}/, (case_log["owning_organisation_id"] + Exports::CaseLogExportService::LOG_ID_OFFSET).to_s) + export_template.sub!(/\{managing_org_id\}/, (case_log["managing_organisation_id"] + Exports::CaseLogExportService::LOG_ID_OFFSET).to_s) end def replace_record_number(export_template, record_number)