diff --git a/app/services/exports/case_log_export_constants.rb b/app/services/exports/case_log_export_constants.rb
new file mode 100644
index 000000000..dfc1e66e8
--- /dev/null
+++ b/app/services/exports/case_log_export_constants.rb
@@ -0,0 +1,138 @@
+module Exports::CaseLogExportConstants
+ MAX_XML_RECORDS = 10_000
+ LOG_ID_OFFSET = 300_000_000_000
+
+ QUARTERS = {
+ 0 => "jan_mar",
+ 1 => "apr_jun",
+ 2 => "jul_sep",
+ 3 => "oct_dec",
+ }.freeze
+
+ EXPORT_FIELDS = Set[
+ "armedforces",
+ "beds",
+ "benefits",
+ "brent",
+ "builtype",
+ "cap",
+ "cbl",
+ "chcharge",
+ "chr",
+ "cligrp1",
+ "cligrp2",
+ "createddate", # New metadata coming from our system
+ "confidential",
+ "earnings",
+ "ethnic",
+ "form",
+ "has_benefits",
+ "hb",
+ "hbrentshortfall",
+ "hcnum",
+ "hhmemb",
+ "homeless",
+ "housingneeds",
+ "illness",
+ "incfreq",
+ "income",
+ "incref",
+ "intstay",
+ "irproduct",
+ "irproduct_other",
+ "joint",
+ "la",
+ "lar",
+ "layear",
+ "leftreg",
+ "lettype",
+ "manhcnum",
+ "maningorgid",
+ "maningorgname",
+ "mobstand",
+ "mrcdate",
+ "national",
+ "needstype",
+ "new_old",
+ "newprop",
+ "nocharge",
+ "offered",
+ "owningorgid",
+ "owningorgname",
+ "period",
+ "postcode_full",
+ "ppcodenk",
+ "ppostcode_full",
+ "preg_occ",
+ "prevloc",
+ "prevten",
+ "propcode",
+ "providertype",
+ "pscharge",
+ "reason",
+ "reasonother",
+ "reasonpref",
+ "referral",
+ "refused",
+ "reghome",
+ "renttype",
+ "renewal",
+ "reservist",
+ "rp_dontknow",
+ "rp_hardship",
+ "rp_homeless",
+ "rp_insan_unsat",
+ "rp_medwel",
+ "rsnvac",
+ "scharge",
+ "scheme",
+ "schtype",
+ "shelteredaccom",
+ "startdate",
+ "startertenancy",
+ "supcharg",
+ "support",
+ "status", # New metadata coming from our system
+ "tcharge",
+ "tshortfall",
+ "tenancy",
+ "tenancycode",
+ "tenancylength",
+ "tenancyother",
+ "totadult",
+ "totchild",
+ "totelder",
+ "underoccupation_benefitcap",
+ "unitletas",
+ "units",
+ "unittype_gn",
+ "unittype_sh",
+ "uploaddate",
+ "username",
+ "voiddate",
+ "waityear",
+ "wchair",
+ "wchchrg",
+ "wpschrge",
+ "wrent",
+ "wscharge",
+ "wsupchrg",
+ "wtcharge",
+ "wtshortfall",
+ ]
+
+ (1..8).each do |index|
+ EXPORT_FIELDS << "age#{index}"
+ EXPORT_FIELDS << "ecstat#{index}"
+ EXPORT_FIELDS << "sex#{index}"
+ end
+ (2..8).each do |index|
+ EXPORT_FIELDS << "relat#{index}"
+ end
+ (1..10).each do |index|
+ EXPORT_FIELDS << "illness_type_#{index}"
+ end
+ %w[a b c d e f g h].each do |letter|
+ EXPORT_FIELDS << "housingneeds_#{letter}"
+ end
+end
diff --git a/app/services/exports/case_log_export_service.rb b/app/services/exports/case_log_export_service.rb
index 54f0b96ed..33429e397 100644
--- a/app/services/exports/case_log_export_service.rb
+++ b/app/services/exports/case_log_export_service.rb
@@ -1,14 +1,6 @@
module Exports
class CaseLogExportService
- QUARTERS = {
- 0 => "jan_mar",
- 1 => "apr_jun",
- 2 => "jul_sep",
- 3 => "oct_dec",
- }.freeze
-
- LOG_ID_OFFSET = 300_000_000_000
- MAX_XML_RECORDS = 10_000
+ include Exports::CaseLogExportConstants
def initialize(storage_service, logger = Rails.logger)
@storage_service = storage_service
@@ -158,14 +150,43 @@ module Exports
xml_doc_to_temp_file(doc)
end
- def apply_cds_transformation!(attribute_hash)
- # OLD_CORE FORM ID support
- attribute_hash["form"] = attribute_hash["old_form_id"]
- attribute_hash["newform"] = attribute_hash["id"] + LOG_ID_OFFSET
+ def apply_cds_transformation(case_log)
+ attribute_hash = case_log.attributes_before_type_cast
+
+ attribute_hash["form"] = attribute_hash["old_form_id"] || (attribute_hash["id"] + LOG_ID_OFFSET)
+
+ # Changes specific to collection start year
+ if case_log.collection_start_year == 2021
+ attribute_hash.delete("joint")
+ end
+ if case_log.collection_start_year == 2022
+ attribute_hash.delete("underoccupation_benefitcap")
+ end
+
+ # Organisation fields
+ if case_log.owning_organisation.present?
+ attribute_hash["owningorgid"] = case_log.owning_organisation.old_visible_id || (case_log.owning_organisation.id + LOG_ID_OFFSET)
+ attribute_hash["owningorgname"] = case_log.owning_organisation.name
+ attribute_hash["hcnum"] = case_log.owning_organisation.housing_registration_no
+ end
+ if case_log.managing_organisation.present?
+ attribute_hash["maningorgid"] = case_log.managing_organisation.old_visible_id || (case_log.managing_organisation.id + LOG_ID_OFFSET)
+ attribute_hash["maningorgname"] = case_log.managing_organisation.name
+ attribute_hash["manhcnum"] = case_log.managing_organisation.housing_registration_no
+ end
+
+ # Mapping which would require a change in our data model
+ attribute_hash["createddate"] = attribute_hash["created_at"]
+ attribute_hash["uploaddate"] = attribute_hash["updated_at"]
+ attribute_hash["tenancycode"] = attribute_hash["tenancy_code"]
+ attribute_hash["ppcodenk"] = attribute_hash["previous_postcode_known"]
+
# Age refused
(1..8).each do |index|
attribute_hash["age#{index}"] = -9 if attribute_hash["age#{index}_known"] == 1
end
+
+ attribute_hash
end
def filter_keys!(attributes)
@@ -173,17 +194,18 @@ module Exports
end
def is_omitted_field?(field_name)
- omitted_attrs = %w[id old_form_id old_id ethnic_group]
pattern_age = /age\d_known/
- field_name.starts_with?("details_known_") || pattern_age.match(field_name) || omitted_attrs.include?(field_name) ? true : false
+ details_known_prefix = "details_known_"
+ field_name.starts_with?(details_known_prefix) ||
+ pattern_age.match(field_name) ||
+ !EXPORT_FIELDS.include?(field_name)
end
def build_export_csv(case_logs)
csv_io = CSV.generate do |csv|
attribute_keys = nil
case_logs.each do |case_log|
- attribute_hash = case_log.attributes_before_type_cast
- apply_cds_transformation!(attribute_hash)
+ attribute_hash = apply_cds_transformation(case_log)
if attribute_keys.nil?
attribute_keys = attribute_hash.keys
filter_keys!(attribute_keys)
@@ -200,8 +222,7 @@ module Exports
doc = Nokogiri::XML("")
case_logs.each do |case_log|
- attribute_hash = case_log.attributes_before_type_cast
- apply_cds_transformation!(attribute_hash)
+ attribute_hash = apply_cds_transformation(case_log)
form = doc.create_element("form")
doc.at("forms") << form
attribute_hash.each do |key, value|
diff --git a/spec/factories/organisation.rb b/spec/factories/organisation.rb
index 981217736..e88abf0c5 100644
--- a/spec/factories/organisation.rb
+++ b/spec/factories/organisation.rb
@@ -4,6 +4,7 @@ FactoryBot.define do
address_line1 { "2 Marsham Street" }
address_line2 { "London" }
provider_type { "LA" }
+ housing_registration_no { "1234" }
postcode { "SW1P 4DF" }
created_at { Time.zone.now }
updated_at { Time.zone.now }
diff --git a/spec/fixtures/exports/case_logs.csv b/spec/fixtures/exports/case_logs.csv
index 5817cf75f..a3e2cea0d 100644
--- a/spec/fixtures/exports/case_logs.csv
+++ b/spec/fixtures/exports/case_logs.csv
@@ -1,2 +1,2 @@
-status,created_at,updated_at,tenant_code,age1,sex1,ethnic,national,prevten,ecstat1,hhmemb,age2,sex2,ecstat2,age3,sex3,ecstat3,age4,sex4,ecstat4,age5,sex5,ecstat5,age6,sex6,ecstat6,age7,sex7,ecstat7,age8,sex8,ecstat8,homeless,underoccupation_benefitcap,leftreg,reservist,illness,preg_occ,tenancy_code,startertenancy,tenancylength,tenancy,ppostcode_full,rsnvac,unittype_gn,beds,offered,wchair,earnings,incfreq,benefits,period,layear,waityear,postcode_full,reasonpref,cbl,chr,cap,reasonother,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,illness_type_1,illness_type_2,illness_type_3,illness_type_4,illness_type_8,illness_type_5,illness_type_6,illness_type_7,illness_type_9,illness_type_10,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,tenancyother,net_income_value_check,property_owner_organisation,property_manager_organisation,sale_or_letting,irproduct_other,purchaser_code,reason,propcode,majorrepairs,la,prevloc,hb,hbrentshortfall,property_relet,mrcdate,incref,sale_completion_date,startdate,armedforces,first_time_property_let_as_social_housing,unitletas,builtype,voiddate,owning_organisation_id,managing_organisation_id,renttype,needstype,lettype,postcode_known,is_la_inferred,totchild,totelder,totadult,net_income_known,nocharge,is_carehome,household_charge,referral,brent,scharge,pscharge,supcharg,tcharge,tshortfall,chcharge,declaration,previous_postcode_known,previous_la_known,is_previous_la_inferred,ethnic_other,letting_allocation_unknown,rent_type,has_benefits,renewal,wrent,wscharge,wpschrge,wsupchrg,wtcharge,wtshortfall,refused,housingneeds,wchchrg,newprop,relat2,relat3,relat4,relat5,relat6,relat7,relat8,rent_value_check,lar,irproduct,joint,created_by_id,illness_type_0,tshortfall_known,shelteredaccom,retirement_value_check,form,newform
-2,2022-02-08 16:52:15 UTC,2022-02-08 16:52:15 UTC,BZ737,35,F,2,4,6,0,2,32,M,6,,,,,,,,,,,,,,,,,,,1,0,1,0,1,2,BZ757,0,5,1,SE26RT,6,7,3,2,1,68,1,1,2,2,1,NW15TY,1,1,1,0,,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,,,Test,Test,,,798794,4,123,1,E09000003,E07000105,6,1,0,2020-05-05 10:36:49 UTC,0,,2022-02-02 10:36:49 UTC,1,,2,1,2019-11-03 00:00:00 UTC,{owning_org_id},{managing_org_id},2,1,7,1,false,0,0,2,1,0,0,,,200.0,50.0,40.0,35.0,325.0,12.0,,1,1,1,false,,,1,1,0,100.0,25.0,20.0,17.5,162.5,6.0,0,1,,2,P,,,,,,,,,,,{created_by_id},,0,0,,,{id}
+status,age1,sex1,ethnic,national,prevten,ecstat1,hhmemb,age2,sex2,ecstat2,age3,sex3,ecstat3,age4,sex4,ecstat4,age5,sex5,ecstat5,age6,sex6,ecstat6,age7,sex7,ecstat7,age8,sex8,ecstat8,homeless,underoccupation_benefitcap,leftreg,reservist,illness,preg_occ,startertenancy,tenancylength,tenancy,ppostcode_full,rsnvac,unittype_gn,beds,offered,wchair,earnings,incfreq,benefits,period,layear,waityear,postcode_full,reasonpref,cbl,chr,cap,reasonother,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,illness_type_1,illness_type_2,illness_type_3,illness_type_4,illness_type_8,illness_type_5,illness_type_6,illness_type_7,illness_type_9,illness_type_10,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,tenancyother,irproduct_other,reason,propcode,la,prevloc,hb,hbrentshortfall,mrcdate,incref,startdate,armedforces,unitletas,builtype,voiddate,renttype,needstype,lettype,totchild,totelder,totadult,nocharge,referral,brent,scharge,pscharge,supcharg,tcharge,tshortfall,chcharge,has_benefits,renewal,wrent,wscharge,wpschrge,wsupchrg,wtcharge,wtshortfall,refused,housingneeds,wchchrg,newprop,relat2,relat3,relat4,relat5,relat6,relat7,relat8,lar,irproduct,shelteredaccom,form,owningorgid,owningorgname,hcnum,maningorgid,maningorgname,manhcnum,createddate,uploaddate,tenancycode,ppcodenk
+2,35,F,2,4,6,0,2,32,M,6,,,,,,,,,,,,,,,,,,,1,0,1,0,1,2,0,5,1,SE26RT,6,7,3,2,1,68,1,1,2,2,1,NW15TY,1,1,1,0,,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,,,4,123,E09000003,E07000105,6,1,2020-05-05 10:36:49 UTC,0,2022-02-02 10:36:49 UTC,1,2,1,2019-11-03 00:00:00 UTC,2,1,7,0,0,2,0,,200.0,50.0,40.0,35.0,325.0,12.0,,1,0,100.0,25.0,20.0,17.5,162.5,6.0,0,1,,2,P,,,,,,,,,0,{id},{owning_org_id},DLUHC,1234,{managing_org_id},DLUHC,1234,2022-02-08 16:52:15 UTC,2022-02-08 16:52:15 UTC,BZ757,1
diff --git a/spec/fixtures/exports/case_logs.xml b/spec/fixtures/exports/case_logs.xml
index e2110fefd..1b899d444 100644
--- a/spec/fixtures/exports/case_logs.xml
+++ b/spec/fixtures/exports/case_logs.xml
@@ -2,9 +2,6 @@
- {id}
+
+ {owning_org_id}
+ DLUHC
+ 1234
+ {managing_org_id}
+ DLUHC
+ 1234
+ 2022-02-08 16:52:15 UTC
+ 2022-02-08 16:52:15 UTC
+ BZ757
+ 1
1
diff --git a/spec/services/exports/case_log_export_service_spec.rb b/spec/services/exports/case_log_export_service_spec.rb
index d3cc2c2f8..0e5f7095e 100644
--- a/spec/services/exports/case_log_export_service_spec.rb
+++ b/spec/services/exports/case_log_export_service_spec.rb
@@ -16,9 +16,8 @@ RSpec.describe Exports::CaseLogExportService do
def replace_entity_ids(case_log, export_template)
export_template.sub!(/\{id\}/, (case_log["id"] + Exports::CaseLogExportService::LOG_ID_OFFSET).to_s)
- export_template.sub!(/\{owning_org_id\}/, case_log["owning_organisation_id"].to_s)
- export_template.sub!(/\{managing_org_id\}/, case_log["managing_organisation_id"].to_s)
- export_template.sub!(/\{created_by_id\}/, case_log["created_by_id"].to_s)
+ export_template.sub!(/\{owning_org_id\}/, (case_log["owning_organisation_id"] + Exports::CaseLogExportService::LOG_ID_OFFSET).to_s)
+ export_template.sub!(/\{managing_org_id\}/, (case_log["managing_organisation_id"] + Exports::CaseLogExportService::LOG_ID_OFFSET).to_s)
end
def replace_record_number(export_template, record_number)