Updates exported fields based on May 25th feedback (#613)

app/services/exports/case_log_export_constants.rb

@@ -0,0 +1,138 @@
+module Exports::CaseLogExportConstants
+  MAX_XML_RECORDS = 10_000
+  LOG_ID_OFFSET = 300_000_000_000
+
+  QUARTERS = {
+    0 => "jan_mar",
+    1 => "apr_jun",
+    2 => "jul_sep",
+    3 => "oct_dec",
+  }.freeze
+
+  EXPORT_FIELDS = Set[
+    "armedforces",
+    "beds",
+    "benefits",
+    "brent",
+    "builtype",
+    "cap",
+    "cbl",
+    "chcharge",
+    "chr",
+    "cligrp1",
+    "cligrp2",
+    "createddate", # New metadata coming from our system
+    "confidential",
+    "earnings",
+    "ethnic",
+    "form",
+    "has_benefits",
+    "hb",
+    "hbrentshortfall",
+    "hcnum",
+    "hhmemb",
+    "homeless",
+    "housingneeds",
+    "illness",
+    "incfreq",
+    "income",
+    "incref",
+    "intstay",
+    "irproduct",
+    "irproduct_other",
+    "joint",
+    "la",
+    "lar",
+    "layear",
+    "leftreg",
+    "lettype",
+    "manhcnum",
+    "maningorgid",
+    "maningorgname",
+    "mobstand",
+    "mrcdate",
+    "national",
+    "needstype",
+    "new_old",
+    "newprop",
+    "nocharge",
+    "offered",
+    "owningorgid",
+    "owningorgname",
+    "period",
+    "postcode_full",
+    "ppcodenk",
+    "ppostcode_full",
+    "preg_occ",
+    "prevloc",
+    "prevten",
+    "propcode",
+    "providertype",
+    "pscharge",
+    "reason",
+    "reasonother",
+    "reasonpref",
+    "referral",
+    "refused",
+    "reghome",
+    "renttype",
+    "renewal",
+    "reservist",
+    "rp_dontknow",
+    "rp_hardship",
+    "rp_homeless",
+    "rp_insan_unsat",
+    "rp_medwel",
+    "rsnvac",
+    "scharge",
+    "scheme",
+    "schtype",
+    "shelteredaccom",
+    "startdate",
+    "startertenancy",
+    "supcharg",
+    "support",
+    "status", # New metadata coming from our system
+    "tcharge",
+    "tshortfall",
+    "tenancy",
+    "tenancycode",
+    "tenancylength",
+    "tenancyother",
+    "totadult",
+    "totchild",
+    "totelder",
+    "underoccupation_benefitcap",
+    "unitletas",
+    "units",
+    "unittype_gn",
+    "unittype_sh",
+    "uploaddate",
+    "username",
+    "voiddate",
+    "waityear",
+    "wchair",
+    "wchchrg",
+    "wpschrge",
+    "wrent",
+    "wscharge",
+    "wsupchrg",
+    "wtcharge",
+    "wtshortfall",
+  ]
+
+  (1..8).each do |index|
+    EXPORT_FIELDS << "age#{index}"
+    EXPORT_FIELDS << "ecstat#{index}"
+    EXPORT_FIELDS << "sex#{index}"
+  end
+  (2..8).each do |index|
+    EXPORT_FIELDS << "relat#{index}"
+  end
+  (1..10).each do |index|
+    EXPORT_FIELDS << "illness_type_#{index}"
+  end
+  %w[a b c d e f g h].each do |letter|
+    EXPORT_FIELDS << "housingneeds_#{letter}"
+  end
+end

app/services/exports/case_log_export_service.rb

@@ -1,14 +1,6 @@
 module Exports
   class CaseLogExportService
-    QUARTERS = {
+    include Exports::CaseLogExportConstants
-      0 => "jan_mar",
-      1 => "apr_jun",
-      2 => "jul_sep",
-      3 => "oct_dec",
-    }.freeze
-
-    LOG_ID_OFFSET = 300_000_000_000
-    MAX_XML_RECORDS = 10_000
 
     def initialize(storage_service, logger = Rails.logger)
       @storage_service = storage_service
@@ -158,14 +150,43 @@ module Exports
       xml_doc_to_temp_file(doc)
     end
 
-    def apply_cds_transformation!(attribute_hash)
+    def apply_cds_transformation(case_log)
-      # OLD_CORE FORM ID support
+      attribute_hash = case_log.attributes_before_type_cast
-      attribute_hash["form"] = attribute_hash["old_form_id"]
+
-      attribute_hash["newform"] = attribute_hash["id"] + LOG_ID_OFFSET
+      attribute_hash["form"] = attribute_hash["old_form_id"] || (attribute_hash["id"] + LOG_ID_OFFSET)
+
+      # Changes specific to collection start year
+      if case_log.collection_start_year == 2021
+        attribute_hash.delete("joint")
+      end
+      if case_log.collection_start_year == 2022
+        attribute_hash.delete("underoccupation_benefitcap")
+      end
+
+      # Organisation fields
+      if case_log.owning_organisation.present?
+        attribute_hash["owningorgid"] = case_log.owning_organisation.old_visible_id || (case_log.owning_organisation.id + LOG_ID_OFFSET)
+        attribute_hash["owningorgname"] = case_log.owning_organisation.name
+        attribute_hash["hcnum"] = case_log.owning_organisation.housing_registration_no
+      end
+      if case_log.managing_organisation.present?
+        attribute_hash["maningorgid"] = case_log.managing_organisation.old_visible_id || (case_log.managing_organisation.id + LOG_ID_OFFSET)
+        attribute_hash["maningorgname"] = case_log.managing_organisation.name
+        attribute_hash["manhcnum"] = case_log.managing_organisation.housing_registration_no
+      end
+
+      # Mapping which would require a change in our data model
+      attribute_hash["createddate"] = attribute_hash["created_at"]
+      attribute_hash["uploaddate"] = attribute_hash["updated_at"]
+      attribute_hash["tenancycode"] = attribute_hash["tenancy_code"]
+      attribute_hash["ppcodenk"] = attribute_hash["previous_postcode_known"]
+
       # Age refused
       (1..8).each do |index|
         attribute_hash["age#{index}"] = -9 if attribute_hash["age#{index}_known"] == 1
       end
+
+      attribute_hash
     end
 
     def filter_keys!(attributes)
@@ -173,17 +194,18 @@ module Exports
     end
 
     def is_omitted_field?(field_name)
-      omitted_attrs = %w[id old_form_id old_id ethnic_group]
       pattern_age = /age\d_known/
-      field_name.starts_with?("details_known_") || pattern_age.match(field_name) || omitted_attrs.include?(field_name) ? true : false
+      details_known_prefix = "details_known_"
+      field_name.starts_with?(details_known_prefix) ||
+        pattern_age.match(field_name) ||
+        !EXPORT_FIELDS.include?(field_name)
     end
 
     def build_export_csv(case_logs)
       csv_io = CSV.generate do |csv|
         attribute_keys = nil
         case_logs.each do |case_log|
-          attribute_hash = case_log.attributes_before_type_cast
+          attribute_hash = apply_cds_transformation(case_log)
-          apply_cds_transformation!(attribute_hash)
           if attribute_keys.nil?
             attribute_keys = attribute_hash.keys
             filter_keys!(attribute_keys)
@@ -200,8 +222,7 @@ module Exports
       doc = Nokogiri::XML("<forms/>")
 
       case_logs.each do |case_log|
-        attribute_hash = case_log.attributes_before_type_cast
+        attribute_hash = apply_cds_transformation(case_log)
-        apply_cds_transformation!(attribute_hash)
         form = doc.create_element("form")
         doc.at("forms") << form
         attribute_hash.each do |key, value|

spec/factories/organisation.rb

@@ -4,6 +4,7 @@ FactoryBot.define do
     address_line1 { "2 Marsham Street" }
     address_line2 { "London" }
     provider_type { "LA" }
+    housing_registration_no { "1234" }
     postcode { "SW1P 4DF" }
     created_at { Time.zone.now }
     updated_at { Time.zone.now }

spec/fixtures/exports/case_logs.csv

@@ -1,2 +1,2 @@
-status,created_at,updated_at,tenant_code,age1,sex1,ethnic,national,prevten,ecstat1,hhmemb,age2,sex2,ecstat2,age3,sex3,ecstat3,age4,sex4,ecstat4,age5,sex5,ecstat5,age6,sex6,ecstat6,age7,sex7,ecstat7,age8,sex8,ecstat8,homeless,underoccupation_benefitcap,leftreg,reservist,illness,preg_occ,tenancy_code,startertenancy,tenancylength,tenancy,ppostcode_full,rsnvac,unittype_gn,beds,offered,wchair,earnings,incfreq,benefits,period,layear,waityear,postcode_full,reasonpref,cbl,chr,cap,reasonother,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,illness_type_1,illness_type_2,illness_type_3,illness_type_4,illness_type_8,illness_type_5,illness_type_6,illness_type_7,illness_type_9,illness_type_10,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,tenancyother,net_income_value_check,property_owner_organisation,property_manager_organisation,sale_or_letting,irproduct_other,purchaser_code,reason,propcode,majorrepairs,la,prevloc,hb,hbrentshortfall,property_relet,mrcdate,incref,sale_completion_date,startdate,armedforces,first_time_property_let_as_social_housing,unitletas,builtype,voiddate,owning_organisation_id,managing_organisation_id,renttype,needstype,lettype,postcode_known,is_la_inferred,totchild,totelder,totadult,net_income_known,nocharge,is_carehome,household_charge,referral,brent,scharge,pscharge,supcharg,tcharge,tshortfall,chcharge,declaration,previous_postcode_known,previous_la_known,is_previous_la_inferred,ethnic_other,letting_allocation_unknown,rent_type,has_benefits,renewal,wrent,wscharge,wpschrge,wsupchrg,wtcharge,wtshortfall,refused,housingneeds,wchchrg,newprop,relat2,relat3,relat4,relat5,relat6,relat7,relat8,rent_value_check,lar,irproduct,joint,created_by_id,illness_type_0,tshortfall_known,shelteredaccom,retirement_value_check,form,newform
+status,age1,sex1,ethnic,national,prevten,ecstat1,hhmemb,age2,sex2,ecstat2,age3,sex3,ecstat3,age4,sex4,ecstat4,age5,sex5,ecstat5,age6,sex6,ecstat6,age7,sex7,ecstat7,age8,sex8,ecstat8,homeless,underoccupation_benefitcap,leftreg,reservist,illness,preg_occ,startertenancy,tenancylength,tenancy,ppostcode_full,rsnvac,unittype_gn,beds,offered,wchair,earnings,incfreq,benefits,period,layear,waityear,postcode_full,reasonpref,cbl,chr,cap,reasonother,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,illness_type_1,illness_type_2,illness_type_3,illness_type_4,illness_type_8,illness_type_5,illness_type_6,illness_type_7,illness_type_9,illness_type_10,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,tenancyother,irproduct_other,reason,propcode,la,prevloc,hb,hbrentshortfall,mrcdate,incref,startdate,armedforces,unitletas,builtype,voiddate,renttype,needstype,lettype,totchild,totelder,totadult,nocharge,referral,brent,scharge,pscharge,supcharg,tcharge,tshortfall,chcharge,has_benefits,renewal,wrent,wscharge,wpschrge,wsupchrg,wtcharge,wtshortfall,refused,housingneeds,wchchrg,newprop,relat2,relat3,relat4,relat5,relat6,relat7,relat8,lar,irproduct,shelteredaccom,form,owningorgid,owningorgname,hcnum,maningorgid,maningorgname,manhcnum,createddate,uploaddate,tenancycode,ppcodenk
-2,2022-02-08 16:52:15 UTC,2022-02-08 16:52:15 UTC,BZ737,35,F,2,4,6,0,2,32,M,6,,,,,,,,,,,,,,,,,,,1,0,1,0,1,2,BZ757,0,5,1,SE26RT,6,7,3,2,1,68,1,1,2,2,1,NW15TY,1,1,1,0,,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,,,Test,Test,,,798794,4,123,1,E09000003,E07000105,6,1,0,2020-05-05 10:36:49 UTC,0,,2022-02-02 10:36:49 UTC,1,,2,1,2019-11-03 00:00:00 UTC,{owning_org_id},{managing_org_id},2,1,7,1,false,0,0,2,1,0,0,,,200.0,50.0,40.0,35.0,325.0,12.0,,1,1,1,false,,,1,1,0,100.0,25.0,20.0,17.5,162.5,6.0,0,1,,2,P,,,,,,,,,,,{created_by_id},,0,0,,,{id}
+2,35,F,2,4,6,0,2,32,M,6,,,,,,,,,,,,,,,,,,,1,0,1,0,1,2,0,5,1,SE26RT,6,7,3,2,1,68,1,1,2,2,1,NW15TY,1,1,1,0,,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,,,4,123,E09000003,E07000105,6,1,2020-05-05 10:36:49 UTC,0,2022-02-02 10:36:49 UTC,1,2,1,2019-11-03 00:00:00 UTC,2,1,7,0,0,2,0,,200.0,50.0,40.0,35.0,325.0,12.0,,1,0,100.0,25.0,20.0,17.5,162.5,6.0,0,1,,2,P,,,,,,,,,0,{id},{owning_org_id},DLUHC,1234,{managing_org_id},DLUHC,1234,2022-02-08 16:52:15 UTC,2022-02-08 16:52:15 UTC,BZ757,1

spec/fixtures/exports/case_logs.xml

@@ -2,9 +2,6 @@
 <forms>
   <form>
     <status>2</status>
-    <created_at>2022-02-08 16:52:15 UTC</created_at>
-    <updated_at>2022-02-08 16:52:15 UTC</updated_at>
-    <tenant_code>BZ737</tenant_code>
     <age1>35</age1>
     <sex1>F</sex1>
     <ethnic>2</ethnic>
@@ -39,7 +36,6 @@
     <reservist>0</reservist>
     <illness>1</illness>
     <preg_occ>2</preg_occ>
-    <tenancy_code>BZ757</tenancy_code>
     <startertenancy>0</startertenancy>
     <tenancylength>5</tenancylength>
     <tenancy>1</tenancy>
@@ -83,43 +79,27 @@
     <rp_hardship>0</rp_hardship>
     <rp_dontknow>0</rp_dontknow>
     <tenancyother/>
-    <net_income_value_check/>
-    <property_owner_organisation>Test</property_owner_organisation>
-    <property_manager_organisation>Test</property_manager_organisation>
-    <sale_or_letting/>
     <irproduct_other/>
-    <purchaser_code>798794</purchaser_code>
     <reason>4</reason>
     <propcode>123</propcode>
-    <majorrepairs>1</majorrepairs>
     <la>E09000003</la>
     <prevloc>E07000105</prevloc>
     <hb>6</hb>
     <hbrentshortfall>1</hbrentshortfall>
-    <property_relet>0</property_relet>
     <mrcdate>2020-05-05 10:36:49 UTC</mrcdate>
     <incref>0</incref>
-    <sale_completion_date/>
     <startdate>2022-02-02 10:36:49 UTC</startdate>
     <armedforces>1</armedforces>
-    <first_time_property_let_as_social_housing/>
     <unitletas>2</unitletas>
     <builtype>1</builtype>
     <voiddate>2019-11-03 00:00:00 UTC</voiddate>
-    <owning_organisation_id>{owning_org_id}</owning_organisation_id>
-    <managing_organisation_id>{managing_org_id}</managing_organisation_id>
     <renttype>2</renttype>
     <needstype>1</needstype>
     <lettype>7</lettype>
-    <postcode_known>1</postcode_known>
-    <is_la_inferred>false</is_la_inferred>
     <totchild>0</totchild>
     <totelder>0</totelder>
     <totadult>2</totadult>
-    <net_income_known>1</net_income_known>
     <nocharge>0</nocharge>
-    <is_carehome>0</is_carehome>
-    <household_charge/>
     <referral/>
     <brent>200.0</brent>
     <scharge>50.0</scharge>
@@ -128,13 +108,6 @@
     <tcharge>325.0</tcharge>
     <tshortfall>12.0</tshortfall>
     <chcharge/>
-    <declaration>1</declaration>
-    <previous_postcode_known>1</previous_postcode_known>
-    <previous_la_known>1</previous_la_known>
-    <is_previous_la_inferred>false</is_previous_la_inferred>
-    <ethnic_other/>
-    <letting_allocation_unknown/>
-    <rent_type>1</rent_type>
     <has_benefits>1</has_benefits>
     <renewal>0</renewal>
     <wrent>100.0</wrent>
@@ -154,17 +127,20 @@
     <relat6/>
     <relat7/>
     <relat8/>
-    <rent_value_check/>
     <lar/>
     <irproduct/>
-    <joint/>
-    <created_by_id>{created_by_id}</created_by_id>
-    <illness_type_0/>
-    <tshortfall_known>0</tshortfall_known>
     <shelteredaccom>0</shelteredaccom>
-    <retirement_value_check/>
+    <form>{id}</form>
-    <form/>
+    <owningorgid>{owning_org_id}</owningorgid>
-    <newform>{id}</newform>
+    <owningorgname>DLUHC</owningorgname>
+    <hcnum>1234</hcnum>
+    <maningorgid>{managing_org_id}</maningorgid>
+    <maningorgname>DLUHC</maningorgname>
+    <manhcnum>1234</manhcnum>
+    <createddate>2022-02-08 16:52:15 UTC</createddate>
+    <uploaddate>2022-02-08 16:52:15 UTC</uploaddate>
+    <tenancycode>BZ757</tenancycode>
+    <ppcodenk>1</ppcodenk>
     <providertype>1</providertype>
   </form>
 </forms>

spec/services/exports/case_log_export_service_spec.rb

@@ -16,9 +16,8 @@ RSpec.describe Exports::CaseLogExportService do
 
   def replace_entity_ids(case_log, export_template)
     export_template.sub!(/\{id\}/, (case_log["id"] + Exports::CaseLogExportService::LOG_ID_OFFSET).to_s)
-    export_template.sub!(/\{owning_org_id\}/, case_log["owning_organisation_id"].to_s)
+    export_template.sub!(/\{owning_org_id\}/, (case_log["owning_organisation_id"] + Exports::CaseLogExportService::LOG_ID_OFFSET).to_s)
-    export_template.sub!(/\{managing_org_id\}/, case_log["managing_organisation_id"].to_s)
+    export_template.sub!(/\{managing_org_id\}/, (case_log["managing_organisation_id"] + Exports::CaseLogExportService::LOG_ID_OFFSET).to_s)
-    export_template.sub!(/\{created_by_id\}/, case_log["created_by_id"].to_s)
   end
 
   def replace_record_number(export_template, record_number)