From b834e0eeb1a093a278bf75c96580151f79eb80cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Meny?= Date: Thu, 26 May 2022 14:28:39 +0100 Subject: [PATCH] Avoid variable number of columns during CSV export (#614) --- .../exports/case_log_export_constants.rb | 5 +++++ .../exports/case_log_export_service.rb | 21 ++++++++++--------- spec/fixtures/exports/case_logs.csv | 4 ++-- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/app/services/exports/case_log_export_constants.rb b/app/services/exports/case_log_export_constants.rb index dfc1e66e8..4899c76e4 100644 --- a/app/services/exports/case_log_export_constants.rb +++ b/app/services/exports/case_log_export_constants.rb @@ -2,6 +2,11 @@ module Exports::CaseLogExportConstants MAX_XML_RECORDS = 10_000 LOG_ID_OFFSET = 300_000_000_000 + EXPORT_MODE = { + xml: 1, + csv: 2, + }.freeze + QUARTERS = { 0 => "jan_mar", 1 => "apr_jun", diff --git a/app/services/exports/case_log_export_service.rb b/app/services/exports/case_log_export_service.rb index 33429e397..0f0d7dd84 100644 --- a/app/services/exports/case_log_export_service.rb +++ b/app/services/exports/case_log_export_service.rb @@ -150,17 +150,18 @@ module Exports xml_doc_to_temp_file(doc) end - def apply_cds_transformation(case_log) + def apply_cds_transformation(case_log, export_mode) attribute_hash = case_log.attributes_before_type_cast - attribute_hash["form"] = attribute_hash["old_form_id"] || (attribute_hash["id"] + LOG_ID_OFFSET) - # Changes specific to collection start year - if case_log.collection_start_year == 2021 - attribute_hash.delete("joint") - end - if case_log.collection_start_year == 2022 - attribute_hash.delete("underoccupation_benefitcap") + # We can't have a variable number of columns in CSV + unless export_mode == EXPORT_MODE[:csv] + case case_log.collection_start_year + when 2021 + attribute_hash.delete("joint") + when 2022 + attribute_hash.delete("underoccupation_benefitcap") + end end # Organisation fields @@ -205,7 +206,7 @@ module Exports csv_io = CSV.generate do |csv| attribute_keys = nil case_logs.each do |case_log| - attribute_hash = apply_cds_transformation(case_log) + attribute_hash = apply_cds_transformation(case_log, EXPORT_MODE[:csv]) if attribute_keys.nil? attribute_keys = attribute_hash.keys filter_keys!(attribute_keys) @@ -222,7 +223,7 @@ module Exports doc = Nokogiri::XML("") case_logs.each do |case_log| - attribute_hash = apply_cds_transformation(case_log) + attribute_hash = apply_cds_transformation(case_log, EXPORT_MODE[:xml]) form = doc.create_element("form") doc.at("forms") << form attribute_hash.each do |key, value| diff --git a/spec/fixtures/exports/case_logs.csv b/spec/fixtures/exports/case_logs.csv index a3e2cea0d..af3c88a07 100644 --- a/spec/fixtures/exports/case_logs.csv +++ b/spec/fixtures/exports/case_logs.csv @@ -1,2 +1,2 @@ -status,age1,sex1,ethnic,national,prevten,ecstat1,hhmemb,age2,sex2,ecstat2,age3,sex3,ecstat3,age4,sex4,ecstat4,age5,sex5,ecstat5,age6,sex6,ecstat6,age7,sex7,ecstat7,age8,sex8,ecstat8,homeless,underoccupation_benefitcap,leftreg,reservist,illness,preg_occ,startertenancy,tenancylength,tenancy,ppostcode_full,rsnvac,unittype_gn,beds,offered,wchair,earnings,incfreq,benefits,period,layear,waityear,postcode_full,reasonpref,cbl,chr,cap,reasonother,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,illness_type_1,illness_type_2,illness_type_3,illness_type_4,illness_type_8,illness_type_5,illness_type_6,illness_type_7,illness_type_9,illness_type_10,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,tenancyother,irproduct_other,reason,propcode,la,prevloc,hb,hbrentshortfall,mrcdate,incref,startdate,armedforces,unitletas,builtype,voiddate,renttype,needstype,lettype,totchild,totelder,totadult,nocharge,referral,brent,scharge,pscharge,supcharg,tcharge,tshortfall,chcharge,has_benefits,renewal,wrent,wscharge,wpschrge,wsupchrg,wtcharge,wtshortfall,refused,housingneeds,wchchrg,newprop,relat2,relat3,relat4,relat5,relat6,relat7,relat8,lar,irproduct,shelteredaccom,form,owningorgid,owningorgname,hcnum,maningorgid,maningorgname,manhcnum,createddate,uploaddate,tenancycode,ppcodenk -2,35,F,2,4,6,0,2,32,M,6,,,,,,,,,,,,,,,,,,,1,0,1,0,1,2,0,5,1,SE26RT,6,7,3,2,1,68,1,1,2,2,1,NW15TY,1,1,1,0,,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,,,4,123,E09000003,E07000105,6,1,2020-05-05 10:36:49 UTC,0,2022-02-02 10:36:49 UTC,1,2,1,2019-11-03 00:00:00 UTC,2,1,7,0,0,2,0,,200.0,50.0,40.0,35.0,325.0,12.0,,1,0,100.0,25.0,20.0,17.5,162.5,6.0,0,1,,2,P,,,,,,,,,0,{id},{owning_org_id},DLUHC,1234,{managing_org_id},DLUHC,1234,2022-02-08 16:52:15 UTC,2022-02-08 16:52:15 UTC,BZ757,1 +status,age1,sex1,ethnic,national,prevten,ecstat1,hhmemb,age2,sex2,ecstat2,age3,sex3,ecstat3,age4,sex4,ecstat4,age5,sex5,ecstat5,age6,sex6,ecstat6,age7,sex7,ecstat7,age8,sex8,ecstat8,homeless,underoccupation_benefitcap,leftreg,reservist,illness,preg_occ,startertenancy,tenancylength,tenancy,ppostcode_full,rsnvac,unittype_gn,beds,offered,wchair,earnings,incfreq,benefits,period,layear,waityear,postcode_full,reasonpref,cbl,chr,cap,reasonother,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,illness_type_1,illness_type_2,illness_type_3,illness_type_4,illness_type_8,illness_type_5,illness_type_6,illness_type_7,illness_type_9,illness_type_10,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,tenancyother,irproduct_other,reason,propcode,la,prevloc,hb,hbrentshortfall,mrcdate,incref,startdate,armedforces,unitletas,builtype,voiddate,renttype,needstype,lettype,totchild,totelder,totadult,nocharge,referral,brent,scharge,pscharge,supcharg,tcharge,tshortfall,chcharge,has_benefits,renewal,wrent,wscharge,wpschrge,wsupchrg,wtcharge,wtshortfall,refused,housingneeds,wchchrg,newprop,relat2,relat3,relat4,relat5,relat6,relat7,relat8,lar,irproduct,joint,shelteredaccom,form,owningorgid,owningorgname,hcnum,maningorgid,maningorgname,manhcnum,createddate,uploaddate,tenancycode,ppcodenk +2,35,F,2,4,6,0,2,32,M,6,,,,,,,,,,,,,,,,,,,1,0,1,0,1,2,0,5,1,SE26RT,6,7,3,2,1,68,1,1,2,2,1,NW15TY,1,1,1,0,,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,,,4,123,E09000003,E07000105,6,1,2020-05-05 10:36:49 UTC,0,2022-02-02 10:36:49 UTC,1,2,1,2019-11-03 00:00:00 UTC,2,1,7,0,0,2,0,,200.0,50.0,40.0,35.0,325.0,12.0,,1,0,100.0,25.0,20.0,17.5,162.5,6.0,0,1,,2,P,,,,,,,,,,0,{id},{owning_org_id},DLUHC,1234,{managing_org_id},DLUHC,1234,2022-02-08 16:52:15 UTC,2022-02-08 16:52:15 UTC,BZ757,1