Cldc 1434 csv column ordering (#842)

* order csv attributes based on the form

* Move is inferred csv attributes next to the inferred fields

* Add is dpo, move scheme fields to back

* move checkbox answer options

* extract move_csv_attributes method

* choose year based on which case logs are downloaded

* typo

* Extract case log csv class

* Move csv class to services

* update schema

app/models/case_log.rb

@@ -74,7 +74,6 @@ class CaseLog < ApplicationRecord
   NUM_OF_WEEKS_FROM_PERIOD = { 2 => 26, 3 => 13, 4 => 12, 5 => 50, 6 => 49, 7 => 48, 8 => 47, 9 => 46, 1 => 52 }.freeze
   SUFFIX_FROM_PERIOD = { 2 => "every 2 weeks", 3 => "every 4 weeks", 4 => "every month" }.freeze
   RETIREMENT_AGES = { "M" => 67, "F" => 60, "X" => 67 }.freeze
-  CSV_FIELDS_TO_OMIT = %w[hhmemb net_income_value_check sale_or_letting first_time_property_let_as_social_housing renttype needstype postcode_known is_la_inferred totchild totelder totadult net_income_known is_carehome previous_la_known is_previous_la_inferred age1_known age2_known age3_known age4_known age5_known age6_known age7_known age8_known letting_allocation_unknown details_known_2 details_known_3 details_known_4 details_known_5 details_known_6 details_known_7 details_known_8 rent_type wrent wscharge wpschrge wsupchrg wtcharge wtshortfall rent_value_check old_form_id old_id retirement_value_check tshortfall_known pregnancy_value_check hhtype new_old vacdays].freeze
   enum status: STATUS
 
   def form
@@ -439,29 +438,12 @@ class CaseLog < ApplicationRecord
     created_by&.name
   end
 
-  def self.to_csv(user = nil)
+  def is_dpo
-    CSV.generate(headers: true) do |csv|
+    created_by&.is_dpo
-      attributes = csv_attributes(user)
-      csv << attributes
-
-      all.find_each do |record|
-        csv << attributes.map do |att|
-          record.form.get_question(att, record)&.label_from_value(record.send(att)) || label_from_value(record.send(att))
-        end
-      end
-    end
   end
 
-  def self.label_from_value(value)
+  def self.to_csv(user = nil)
-    return "Yes" if value == true
+    Csv::CaseLogCsvService.new(user).to_csv
-    return "No" if value == false
-
-    value
-  end
-
-  def self.csv_attributes(user)
-    attributes = attribute_names - %w[owning_organisation_id managing_organisation_id created_by_id] + %w[unittype_sh owning_organisation_name managing_organisation_name created_by_name]
-    user.present? && !user.support? ? attributes - CSV_FIELDS_TO_OMIT : attributes
   end
 
   def soft_min_for_period

app/services/csv/case_log_csv_service.rb

@@ -0,0 +1,90 @@
+module Csv
+  class CaseLogCsvService
+    CSV_FIELDS_TO_OMIT = %w[hhmemb net_income_value_check sale_or_letting first_time_property_let_as_social_housing renttype needstype postcode_known is_la_inferred totchild totelder totadult net_income_known is_carehome previous_la_known is_previous_la_inferred age1_known age2_known age3_known age4_known age5_known age6_known age7_known age8_known letting_allocation_unknown details_known_2 details_known_3 details_known_4 details_known_5 details_known_6 details_known_7 details_known_8 rent_type wrent wscharge wpschrge wsupchrg wtcharge wtshortfall rent_value_check old_form_id old_id retirement_value_check tshortfall_known pregnancy_value_check hhtype new_old vacdays].freeze
+
+    def initialize(user)
+      @user = user
+      set_csv_attributes
+    end
+
+    def to_csv
+      CSV.generate(headers: true) do |csv|
+        csv << @attributes
+
+        CaseLog.all.find_each do |record|
+          csv << @attributes.map do |att|
+            record.form.get_question(att, record)&.label_from_value(record.send(att)) || label_from_value(record.send(att))
+          end
+        end
+      end
+    end
+
+  private
+
+    def label_from_value(value)
+      return "Yes" if value == true
+      return "No" if value == false
+
+      value
+    end
+
+    def set_csv_attributes
+      @attributes = CaseLog.attribute_names + %w[unittype_sh]
+      replace_csv_id_fields_with_names
+      order_csv_attributes
+
+      @attributes -= CSV_FIELDS_TO_OMIT if @user.present? && !@user.support?
+    end
+
+    def replace_csv_id_fields_with_names
+      { "managing_organisation_id": "managing_organisation_name", "owning_organisation_id": "owning_organisation_name", "created_by_id": "created_by_name" }.each { |current, new_attribute| @attributes[@attributes.index(current.to_s)] = new_attribute }
+    end
+
+    def order_csv_attributes
+      downloaded_form_years = CaseLog.all.map(&:collection_start_year).uniq.compact
+
+      downloaded_form_fields = downloaded_form_years.count == 1 && downloaded_form_years[0].present? ? FormHandler.instance.get_form("#{downloaded_form_years[0]}_#{downloaded_form_years[0] + 1}").questions : FormHandler.instance.forms.first.second.questions
+      ordered_default_form_questions = move_checkbox_answer_options(downloaded_form_fields)
+
+      @attributes = (ordered_default_form_questions & @attributes) + (@attributes - ordered_default_form_questions)
+      move_metadata_fields_to_front
+      move_is_inferred_fields
+      move_scheme_fields_to_back
+    end
+
+    def move_checkbox_answer_options(form_questions)
+      checkboxes = form_questions.filter { |question| question.type == "checkbox" }.map { |question| { "#{question.id}": question.answer_options.keys } }
+      attributes = form_questions.map(&:id).uniq
+
+      checkboxes.each do |checkbox_question|
+        checkbox_question.values[0].each do |answer_option|
+          attributes.insert(attributes.find_index(checkbox_question.keys[0].to_s), answer_option)
+        end
+      end
+      attributes
+    end
+
+    def move_metadata_fields_to_front
+      metadata_fields = %w[managing_organisation_name owning_organisation_name is_dpo created_by_name updated_at created_at status id]
+      move_csv_attributes(metadata_fields, 0)
+    end
+
+    def move_is_inferred_fields
+      { is_la_inferred: "la", is_previous_la_inferred: "prevloc" }.each do |field, inferred_field|
+        @attributes.delete(field.to_s)
+        @attributes.insert(@attributes.find_index(inferred_field), field.to_s)
+      end
+    end
+
+    def move_scheme_fields_to_back
+      move_csv_attributes(%w[scheme_id location_id], -1)
+    end
+
+    def move_csv_attributes(fields_to_move, index)
+      fields_to_move.each do |field|
+        @attributes.delete(field)
+        @attributes.insert(index, field)
+      end
+    end
+  end
+end

db/schema.rb

@@ -202,6 +202,8 @@ ActiveRecord::Schema[7.0].define(version: 2022_08_10_152340) do
     t.bigint "location_id"
     t.integer "major_repairs_date_value_check"
     t.integer "void_date_value_check"
+    t.integer "housingneeds_type"
+    t.integer "housingneeds_other"
     t.index ["created_by_id"], name: "index_case_logs_on_created_by_id"
     t.index ["location_id"], name: "index_case_logs_on_location_id"
     t.index ["managing_organisation_id"], name: "index_case_logs_on_managing_organisation_id"

spec/fixtures/files/case_logs_download.csv

@@ -1,2 +1,2 @@
-id,status,created_at,updated_at,tenancycode,age1,sex1,ethnic,national,prevten,ecstat1,hhmemb,age2,sex2,ecstat2,age3,sex3,ecstat3,age4,sex4,ecstat4,age5,sex5,ecstat5,age6,sex6,ecstat6,age7,sex7,ecstat7,age8,sex8,ecstat8,homeless,underoccupation_benefitcap,leftreg,reservist,illness,preg_occ,startertenancy,tenancylength,tenancy,ppostcode_full,rsnvac,unittype_gn,beds,offered,wchair,earnings,incfreq,benefits,period,layear,waityear,postcode_full,reasonpref,cbl,chr,cap,reasonother,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,illness_type_1,illness_type_2,illness_type_3,illness_type_4,illness_type_8,illness_type_5,illness_type_6,illness_type_7,illness_type_9,illness_type_10,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,tenancyother,net_income_value_check,property_owner_organisation,property_manager_organisation,sale_or_letting,irproduct_other,purchaser_code,reason,propcode,majorrepairs,la,prevloc,hb,hbrentshortfall,property_relet,mrcdate,incref,sale_completion_date,startdate,armedforces,first_time_property_let_as_social_housing,unitletas,builtype,voiddate,renttype,needstype,lettype,postcode_known,is_la_inferred,totchild,totelder,totadult,net_income_known,nocharge,is_carehome,household_charge,referral,brent,scharge,pscharge,supcharg,tcharge,tshortfall,chcharge,declaration,ppcodenk,previous_la_known,is_previous_la_inferred,age1_known,age2_known,age3_known,age4_known,age5_known,age6_known,age7_known,age8_known,ethnic_group,ethnic_other,letting_allocation_unknown,details_known_2,details_known_3,details_known_4,details_known_5,details_known_6,details_known_7,details_known_8,rent_type,has_benefits,renewal,wrent,wscharge,wpschrge,wsupchrg,wtcharge,wtshortfall,refused,housingneeds,wchchrg,newprop,relat2,relat3,relat4,relat5,relat6,relat7,relat8,rent_value_check,old_form_id,lar,irproduct,old_id,joint,illness_type_0,retirement_value_check,tshortfall_known,sheltered,pregnancy_value_check,hhtype,new_old,vacdays,scheme_id,location_id,major_repairs_date_value_check,void_date_value_check,unittype_sh,owning_organisation_name,managing_organisation_name,created_by_name
+id,status,created_at,updated_at,created_by_name,is_dpo,owning_organisation_name,managing_organisation_name,needstype,renewal,startdate,rent_type,irproduct_other,tenancycode,propcode,age1,sex1,ecstat1,hhmemb,relat2,age2,sex2,retirement_value_check,ecstat2,armedforces,leftreg,illness,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_h,is_previous_la_inferred,prevloc,illness_type_1,illness_type_2,is_la_inferred,la,postcode_known,postcode_full,previous_la_known,wchair,preg_occ,cbl,earnings,incfreq,net_income_value_check,benefits,hb,period,brent,scharge,pscharge,supcharg,tcharge,offered,layear,ppostcode_full,mrcdate,declaration,ethnic,national,prevten,age3,sex3,ecstat3,age4,sex4,ecstat4,age5,sex5,ecstat5,age6,sex6,ecstat6,age7,sex7,ecstat7,age8,sex8,ecstat8,homeless,underoccupation_benefitcap,reservist,startertenancy,tenancylength,tenancy,rsnvac,unittype_gn,beds,waityear,reasonpref,chr,cap,reasonother,housingneeds_f,housingneeds_g,illness_type_3,illness_type_4,illness_type_8,illness_type_5,illness_type_6,illness_type_7,illness_type_9,illness_type_10,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,tenancyother,property_owner_organisation,property_manager_organisation,sale_or_letting,purchaser_code,reason,majorrepairs,hbrentshortfall,property_relet,incref,sale_completion_date,first_time_property_let_as_social_housing,unitletas,builtype,voiddate,renttype,lettype,totchild,totelder,totadult,net_income_known,nocharge,is_carehome,household_charge,referral,tshortfall,chcharge,ppcodenk,age1_known,age2_known,age3_known,age4_known,age5_known,age6_known,age7_known,age8_known,ethnic_group,ethnic_other,letting_allocation_unknown,details_known_2,details_known_3,details_known_4,details_known_5,details_known_6,details_known_7,details_known_8,has_benefits,wrent,wscharge,wpschrge,wsupchrg,wtcharge,wtshortfall,refused,housingneeds,wchchrg,newprop,relat3,relat4,relat5,relat6,relat7,relat8,rent_value_check,old_form_id,lar,irproduct,old_id,joint,illness_type_0,tshortfall_known,sheltered,pregnancy_value_check,hhtype,new_old,vacdays,major_repairs_date_value_check,void_date_value_check,housingneeds_type,housingneeds_other,unittype_sh,scheme_id,location_id
-{id},in_progress,2022-02-08 16:52:15 +0000,2022-02-08 16:52:15 +0000,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,SE1 1TE,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,Westminster,,,,,,,,,,,,,,,Supported housing,,,No,0,0,0,,0,,,,,,,,,,,,,,No,,,,,,,,,,,,,,,,,,,,0,,,,,,,,0,,,,,,,,,,,,,,,,,,,,,,9,,,{scheme_id},SE1 1TE,,,6,DLUHC,DLUHC,Danny Rojas
+{id},in_progress,2022-02-08 16:52:15 +0000,2022-02-08 16:52:15 +0000,Danny Rojas,No,DLUHC,DLUHC,Supported housing,,,,,,,,,,,,,,,,,,,,,,,No,,,,No,Westminster,,SE1 1TE,,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,0,0,,0,,,,,,,,,,,,,,,,,,,,,,,,,0,,,,,,,0,,,,,,,,,,,,,,,,,,,,9,,,,,,,6,{scheme_id},SE1 1TE

spec/fixtures/files/case_logs_download_non_support.csv

@@ -1,2 +1,2 @@
-id,status,created_at,updated_at,tenancycode,age1,sex1,ethnic,national,prevten,ecstat1,age2,sex2,ecstat2,age3,sex3,ecstat3,age4,sex4,ecstat4,age5,sex5,ecstat5,age6,sex6,ecstat6,age7,sex7,ecstat7,age8,sex8,ecstat8,homeless,underoccupation_benefitcap,leftreg,reservist,illness,preg_occ,startertenancy,tenancylength,tenancy,ppostcode_full,rsnvac,unittype_gn,beds,offered,wchair,earnings,incfreq,benefits,period,layear,waityear,postcode_full,reasonpref,cbl,chr,cap,reasonother,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,illness_type_1,illness_type_2,illness_type_3,illness_type_4,illness_type_8,illness_type_5,illness_type_6,illness_type_7,illness_type_9,illness_type_10,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,tenancyother,property_owner_organisation,property_manager_organisation,irproduct_other,purchaser_code,reason,propcode,majorrepairs,la,prevloc,hb,hbrentshortfall,property_relet,mrcdate,incref,sale_completion_date,startdate,armedforces,unitletas,builtype,voiddate,lettype,nocharge,household_charge,referral,brent,scharge,pscharge,supcharg,tcharge,tshortfall,chcharge,declaration,ppcodenk,ethnic_group,ethnic_other,has_benefits,renewal,refused,housingneeds,wchchrg,newprop,relat2,relat3,relat4,relat5,relat6,relat7,relat8,lar,irproduct,joint,illness_type_0,sheltered,scheme_id,location_id,major_repairs_date_value_check,void_date_value_check,unittype_sh,owning_organisation_name,managing_organisation_name,created_by_name
+id,status,created_at,updated_at,created_by_name,is_dpo,owning_organisation_name,managing_organisation_name,renewal,startdate,irproduct_other,tenancycode,propcode,age1,sex1,ecstat1,relat2,age2,sex2,ecstat2,armedforces,leftreg,illness,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_h,prevloc,illness_type_1,illness_type_2,la,postcode_full,wchair,preg_occ,cbl,earnings,incfreq,benefits,hb,period,brent,scharge,pscharge,supcharg,tcharge,offered,layear,ppostcode_full,mrcdate,declaration,ethnic,national,prevten,age3,sex3,ecstat3,age4,sex4,ecstat4,age5,sex5,ecstat5,age6,sex6,ecstat6,age7,sex7,ecstat7,age8,sex8,ecstat8,homeless,underoccupation_benefitcap,reservist,startertenancy,tenancylength,tenancy,rsnvac,unittype_gn,beds,waityear,reasonpref,chr,cap,reasonother,housingneeds_f,housingneeds_g,illness_type_3,illness_type_4,illness_type_8,illness_type_5,illness_type_6,illness_type_7,illness_type_9,illness_type_10,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,tenancyother,property_owner_organisation,property_manager_organisation,purchaser_code,reason,majorrepairs,hbrentshortfall,property_relet,incref,sale_completion_date,unitletas,builtype,voiddate,lettype,nocharge,household_charge,referral,tshortfall,chcharge,ppcodenk,ethnic_group,ethnic_other,has_benefits,refused,housingneeds,wchchrg,newprop,relat3,relat4,relat5,relat6,relat7,relat8,lar,irproduct,joint,illness_type_0,sheltered,major_repairs_date_value_check,void_date_value_check,housingneeds_type,housingneeds_other,unittype_sh,scheme_id,location_id
-{id},in_progress,2022-02-08 16:52:15 +0000,2022-02-08 16:52:15 +0000,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,SE1 1TE,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,Westminster,,,,,,,,,,,,,,0,,,,,,,,,,,,,,0,,0,,,,,,,,,,,,,,,,{scheme_id},SE1 1TE,,,6,DLUHC,DLUHC,Danny Rojas
+{id},in_progress,2022-02-08 16:52:15 +0000,2022-02-08 16:52:15 +0000,Danny Rojas,No,DLUHC,DLUHC,,,,,,,,,,,,,,,,,,,,,,,Westminster,SE1 1TE,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,,,,,,,,0,0,,,,,,,,,,,,,,,,,,,6,{scheme_id},SE1 1TE

spec/requests/case_logs_controller_spec.rb

@@ -766,7 +766,7 @@ RSpec.describe CaseLogsController, type: :request do
 
       it "downloads answer labels rather than values" do
         csv = CSV.parse(response.body)
-        expect(csv.second[10]).to eq("Full-time – 30 hours or more")
+        expect(csv.second[15]).to eq("Full-time – 30 hours or more")
       end
 
       it "downloads filtered logs" do