CLDC-3965: Sales CSV column changes (#3021)

* Update expected content * Update expected content * Update sales csv with 2025 mappings * Remove comments * Remove comments * Remove comments * Change hhype to hholdcount, exisiting bug * Revert "Change hhype to hholdcount, exisiting bug" This reverts commit ca8aab58ee. * Update test file * Add service charge fields to sales log CSV service * Add created_by_id to SUPPORT_ONLY_ATTRIBUTES in sales log CSV service * Add tests and CSV files for 2025 sales log export functionality * Update sales log CSV service to prepare for new service charge fields * Lint * Update sales log CSV export labels for 2025 * Lint * Add service charge fields and conditionally show service charge or mscharge in 2025 CSV * Remove comments
3 weeks ago · 98e1c0ea25
6 changed files with 213 additions and 25 deletions
--- a/app/models/sales_log.rb
+++ b/app/models/sales_log.rb
@ -542,4 +542,28 @@ class SalesLog < Log
  def log_type
    "sales_log"
  end
+
+  def has_servicecharge
+    has_mscharge if shared_ownership_scheme?
+  end
+
+  def has_servicecharge_label
+    form.get_question(:has_mscharge, self)&.label_from_value(has_mscharge) if shared_ownership_scheme?
+  end
+
+  def servicecharge
+    mscharge if shared_ownership_scheme?
+  end
+
+  def has_mscharge_value
+    has_mscharge if discounted_ownership_sale? || !form.start_year_2025_or_later?
+  end
+
+  def has_mscharge_label
+    form.get_question(:has_mscharge, self)&.label_from_value(has_mscharge) if discounted_ownership_sale? || !form.start_year_2025_or_later?
+  end
+
+  def mscharge_value
+    mscharge if discounted_ownership_sale? || !form.start_year_2025_or_later?
+  end
 end
--- a/app/services/csv/sales_log_csv_service.rb
+++ b/app/services/csv/sales_log_csv_service.rb
@ -68,6 +68,46 @@ module Csv
        labels: %i[monthly_charges_value_check],
        codes: %i[monthly_charges_value_check],
      },
+      stairlastday: {
+        labels: %i[lasttransaction day],
+        codes: %i[lasttransaction day],
+      },
+      stairlastmonth: {
+        labels: %i[lasttransaction month],
+        codes: %i[lasttransaction month],
+      },
+      stairlastyear: {
+        labels: %i[lasttransaction year],
+        codes: %i[lasttransaction year],
+      },
+      stairinitialday: {
+        labels: %i[initialpurchase day],
+        codes: %i[initialpurchase day],
+      },
+      stairinitialmonth: {
+        labels: %i[initialpurchase month],
+        codes: %i[initialpurchase month],
+      },
+      stairinitialyear: {
+        labels: %i[initialpurchase year],
+        codes: %i[initialpurchase year],
+      },
+      has_servicecharges: {
+        labels: %i[has_servicecharge_label],
+        codes: %i[has_servicecharge],
+      },
+      servicecharges: {
+        labels: %i[servicecharge],
+        codes: %i[servicecharge],
+      },
+      has_mscharge: {
+        labels: %i[has_mscharge_label],
+        codes: %i[has_mscharge_value],
+      },
+      mscharge: {
+        labels: %i[mscharge_value],
+        codes: %i[mscharge_value],
+      },
    }.freeze

    PERSON_DETAILS = {}.tap { |hash|
@ -97,7 +137,7 @@ module Csv

    ORDERED_ADDRESS_FIELDS = %w[uprn address_line1 address_line2 town_or_city county postcode_full is_la_inferred la_label la uprn_selection address_search_value_check address_line1_input postcode_full_input address_line1_as_entered address_line2_as_entered town_or_city_as_entered county_as_entered postcode_full_as_entered la_as_entered].freeze

-    SUPPORT_ONLY_ATTRIBUTES = %w[address_line1_as_entered address_line2_as_entered town_or_city_as_entered county_as_entered postcode_full_as_entered la_as_entered created_by value_value_check mscharge_value_check].freeze
+    SUPPORT_ONLY_ATTRIBUTES = %w[address_line1_as_entered address_line2_as_entered town_or_city_as_entered county_as_entered postcode_full_as_entered la_as_entered created_by created_by_id value_value_check mscharge_value_check].freeze

    SUPPORT_ATTRIBUTE_NAME_MAPPINGS = {
      "duplicate_set_id" => "DUPLICATESET",
@ -152,6 +192,19 @@ module Csv
      "uprn_confirmed" => "UPRNCONFIRMED",
    }.freeze

+    SUPPORT_ATTRIBUTE_NAME_MAPPINGS_2025 = {
+      "hholdcount" => "HHOLDCOUNT",
+      "created_by_id" => "CREATEDBYID",
+      "owning_organisation_id" => "OWNINGORGID",
+      "managing_organisation_id" => "MANINGORGID",
+      "assigned_to_id" => "USERNAMEID",
+      "updated_by" => "AMENDEDBY",
+      "updated_by_id" => "AMENDEDBYID",
+      "has_management_fee" => "HASESTATEFEE",
+      "management_fee" => "ESTATEFEE",
+      "has_servicecharges" => "HASSERVICECHARGES",
+    }.freeze
+
    UPRN_CONFIRMED_LABELS = {
      0 => "No",
      1 => "Yes",
@ -161,20 +214,54 @@ module Csv
      "uprn_confirmed" => UPRN_CONFIRMED_LABELS,
    }.freeze

+    ATTRIBUTE_MAPPINGS = {
+      "saledate" => %w[day month year],
+      "exdate" => %w[exday exmonth exyear],
+      "hodate" => %w[hoday homonth hoyear],
+      "ppostcode_full" => %w[ppostc1 ppostc2],
+      "la" => %w[la la_label],
+      "prevloc" => %w[prevloc prevloc_label],
+      "assigned_to_id" => %w[created_by assigned_to],
+      "owning_organisation_id" => %w[owning_organisation_name],
+      "managing_organisation_id" => %w[managing_organisation_name],
+      "value" => %w[value value_value_check],
+      "mscharge" => %w[mscharge mscharge_value_check],
+    }.freeze
+
+    ATTRIBUTE_MAPPINGS_2024 = {
+      "uprn" => %w[uprn uprn_confirmed address_line1_input postcode_full_input uprn_selection],
+    }.freeze
+
+    ATTRIBUTE_MAPPINGS_2025 = {
+      "duplicate_set_id" => %w[duplicate_set_id owning_organisation_name owning_organisation_id managing_organisation_name managing_organisation_id],
+      "created_by_id" => %w[created_by created_by_id assigned_to assigned_to_id],
+      "updated_by_id" => %w[updated_by updated_by_id],
+      "bulk_upload_id" => %w[bulk_upload_id collection_start_year],
+      "prevten" => %w[hhtype prevten],
+      "mrent" => %w[mrent has_servicecharges servicecharges has_management_fee management_fee],
+      "lasttransaction" => %w[stairlastday stairlastmonth stairlastyear],
+      "initialpurchase" => %w[stairinitialday stairinitialmonth stairinitialyear],
+      "mrentprestaircasing" => %w[mrentprestaircasing grant discount extrabor has_mscharge mscharge mscharge_value_check],
+    }.freeze
+
    def formatted_attribute_headers
      return @attributes unless @user.support?

+      mappings = SUPPORT_ATTRIBUTE_NAME_MAPPINGS
+      mappings = mappings.merge(SUPPORT_ATTRIBUTE_NAME_MAPPINGS_2025) if @year == 2025
+
      @attributes.map do |attribute|
-        SUPPORT_ATTRIBUTE_NAME_MAPPINGS[attribute] || attribute.upcase
+        mappings[attribute] || attribute.upcase
      end
    end

    def sales_log_attributes
      ordered_questions = FormHandler.instance.ordered_questions_for_year(@year, "sales")
      ordered_questions.reject! { |q| q.id.match?(/((?<!la)_known)|(_check)|(_asked)|nationality_all_group|nationality_all_buyer2_group/) }
-      attributes = insert_derived_and_related_attributes(ordered_questions)
-      order_address_fields_for_support(attributes)
-      final_attributes = non_question_fields + attributes
+      ordered_questions.reject! { |q| q.id.match?(/organisation_id|created_by|assigned_to|soctenant|has_management_fee|management_fee|grant|discount|has_mscharge|mscharge|extrabor/) } if @year >= 2025
+      attributes = insert_checkbox_options(ordered_questions)
+      final_attributes = insert_derived_and_related_attributes(non_question_fields + attributes)
+      order_address_fields_for_support(final_attributes)
      @user.support? ? final_attributes : final_attributes - SUPPORT_ONLY_ATTRIBUTES
    end

@ -187,38 +274,40 @@ module Csv
      }.compact
    end

-    def insert_derived_and_related_attributes(ordered_questions)
+    def insert_checkbox_options(ordered_questions)
      ordered_questions.flat_map do |question|
        if question.type == "checkbox"
          question.answer_options.keys
-        elsif attribute_mappings.key? question.id
-          attribute_mappings[question.id]
        else
          question.id
        end
      end
    end

+    def insert_derived_and_related_attributes(attributes)
+      attributes.flat_map do |attribute|
+        if attribute_mappings.key? attribute
+          attribute_mappings[attribute]
+        else
+          attribute
+        end
+      end
+    end
+
    def attribute_mappings
-      mappings = {
-        "saledate" => %w[day month year],
-        "exdate" => %w[exday exmonth exyear],
-        "hodate" => %w[hoday homonth hoyear],
-        "ppostcode_full" => %w[ppostc1 ppostc2],
-        "la" => %w[la la_label],
-        "prevloc" => %w[prevloc prevloc_label],
-        "assigned_to_id" => %w[created_by assigned_to],
-        "owning_organisation_id" => %w[owning_organisation_name],
-        "managing_organisation_id" => %w[managing_organisation_name],
-        "value" => %w[value value_value_check],
-        "mscharge" => %w[mscharge mscharge_value_check],
-      }
+      mappings = case @year
+                 when 2024
+                   ATTRIBUTE_MAPPINGS.merge(ATTRIBUTE_MAPPINGS_2024)
+                 when 2025
+                   ATTRIBUTE_MAPPINGS.merge(ATTRIBUTE_MAPPINGS_2024).merge(ATTRIBUTE_MAPPINGS_2025)
+                 else
+                   ATTRIBUTE_MAPPINGS
+                 end
+
      unless @user.support? && @year >= 2024
-        mappings["postcode_full"] = %w[pcode1 pcode2]
-      end
-      if @year >= 2024
-        mappings["uprn"] = %w[uprn uprn_confirmed address_line1_input postcode_full_input uprn_selection]
+        mappings = mappings.merge({ "postcode_full" => %w[pcode1 pcode2] })
      end
+
      mappings
    end

@ -240,6 +329,8 @@ module Csv
        %w[id status duplicate_set_id created_at updated_at old_form_id collection_start_year creation_method is_dpo]
      when 2024
        %w[id status duplicate_set_id created_at updated_at collection_start_year creation_method bulk_upload_id is_dpo]
+      when 2025
+        %w[id status duplicate_set_id created_at created_by_id updated_at updated_by_id creation_method bulk_upload_id]
      else
        %w[id status duplicate_set_id created_at updated_at collection_start_year creation_method bulk_upload_id is_dpo]
      end
--- a/spec/fixtures/files/sales_logs_csv_export_codes_25.csv
+++ b/spec/fixtures/files/sales_logs_csv_export_codes_25.csv
--- a/spec/fixtures/files/sales_logs_csv_export_labels_25.csv
+++ b/spec/fixtures/files/sales_logs_csv_export_labels_25.csv
--- a/spec/fixtures/files/sales_logs_csv_export_non_support_labels_25.csv
+++ b/spec/fixtures/files/sales_logs_csv_export_non_support_labels_25.csv
--- a/spec/services/csv/sales_log_csv_service_spec.rb
+++ b/spec/services/csv/sales_log_csv_service_spec.rb
@ -213,6 +213,26 @@ RSpec.describe Csv::SalesLogCsvService do
      end
    end

+    context "when the requested form is 2025" do
+      let(:now) { Time.zone.local(2025, 5, 1) }
+      let(:year) { 2025 }
+      let(:fixed_time) { Time.zone.local(2025, 5, 1) }
+
+      before do
+        log.update!(nationality_all: 36, manual_address_entry_selected: false, uprn: "1", uprn_known: 1)
+      end
+
+      it "exports the CSV with the 2025 ordering and all values correct" do
+        expected_content = CSV.read("spec/fixtures/files/sales_logs_csv_export_labels_25.csv")
+        values_to_delete = %w[ID OWNINGORGID MANINGORGID CREATEDBYID USERNAMEID AMENDEDBYID]
+        values_to_delete.each do |attribute|
+          index = attribute_line.index(attribute)
+          content_line[index] = nil
+        end
+        expect(csv[1..]).to eq expected_content[1..] # Skip the first line as it contains the definitions
+      end
+    end
+
    context "when the requested form is 2023" do
      let(:now) { Time.zone.local(2024, 1, 1) }
      let(:year) { 2023 }
@ -281,6 +301,26 @@ RSpec.describe Csv::SalesLogCsvService do
      expect(la_label_value).to eq "Westminster"
    end

+    context "when the requested form is 2025" do
+      let(:now) { Time.zone.local(2025, 5, 1) }
+      let(:fixed_time) { Time.zone.local(2025, 5, 1) }
+      let(:year) { 2025 }
+
+      before do
+        log.update!(manual_address_entry_selected: false, uprn: "1", uprn_known: 1)
+      end
+
+      it "exports the CSV with all values correct" do
+        expected_content = CSV.read("spec/fixtures/files/sales_logs_csv_export_codes_25.csv")
+        values_to_delete = %w[ID OWNINGORGID MANINGORGID CREATEDBYID USERNAMEID AMENDEDBYID]
+        values_to_delete.each do |attribute|
+          index = attribute_line.index(attribute)
+          content_line[index] = nil
+        end
+        expect(csv[1..]).to eq expected_content[1..] # Skip the first line as it contains the definitions
+      end
+    end
+
    context "when the requested form is 2024" do
      let(:now) { Time.zone.local(2024, 5, 1) }
      let(:fixed_time) { Time.zone.local(2024, 5, 1) }
@ -359,5 +399,29 @@ RSpec.describe Csv::SalesLogCsvService do
        end
      end
    end
+
+    context "and the requested form is 2025" do
+      let(:year) { 2025 }
+      let(:now) { Time.zone.local(2025, 5, 1) }
+      let(:fixed_time) { Time.zone.local(2025, 5, 1) }
+
+      before do
+        log.update!(nationality_all: 36, manual_address_entry_selected: false, uprn: "1", uprn_known: 1)
+      end
+
+      context "and exporting with labels" do
+        let(:export_type) { "labels" }
+
+        it "exports the CSV with all values correct" do
+          expected_content = CSV.read("spec/fixtures/files/sales_logs_csv_export_non_support_labels_25.csv")
+          values_to_delete = %w[id owning_organisation_id managing_organisation_id assigned_to_id updated_by_id]
+          values_to_delete.each do |attribute|
+            index = attribute_line.index(attribute)
+            content_line[index] = nil
+          end
+          expect(csv[1..]).to eq expected_content[1..] # Skip the first line as it contains the definitions
+        end
+      end
+    end
  end
 end