CLDC-3190 Add lettings BU for 24/25 (#2158)

* Copy 23 BU files to 24

* Update field mapping

* Add duplicate log error to charges

* Check the sum of charges for duplicates

* update stub

* Only add charges to duplciate hash if they exist

* Rebase change

* Update columns

* Add tcharge back to error mapping

* Add prepare your file

* Add csv parser 24/25 cases

* Fix date check

* Rebase fix

app/models/forms/bulk_upload_lettings/prepare_your_file.rb

@@ -12,6 +12,8 @@ module Forms
         case year
         when 2023
           "bulk_upload_lettings_logs/forms/prepare_your_file_2023"
+        when 2024
+          "bulk_upload_lettings_logs/forms/prepare_your_file_2024"
         end
       end
 

app/services/bulk_upload/lettings/log_creator.rb

@@ -33,6 +33,8 @@ private
     @csv_parser ||= case bulk_upload.year
                     when 2023
                       BulkUpload::Lettings::Year2023::CsvParser.new(path:)
+                    when 2024
+                      BulkUpload::Lettings::Year2024::CsvParser.new(path:)
                     else
                       raise "csv parser not found"
                     end

app/services/bulk_upload/lettings/validator.rb

@@ -100,6 +100,8 @@ private
     @csv_parser ||= case bulk_upload.year
                     when 2023
                       BulkUpload::Lettings::Year2023::CsvParser.new(path:)
+                    when 2024
+                      BulkUpload::Lettings::Year2024::CsvParser.new(path:)
                     else
                       raise "csv parser not found"
                     end

app/services/bulk_upload/lettings/year2023/csv_parser.rb

@@ -105,9 +105,9 @@ private
 
   def first_record_start_date
     if with_headers?
-      Date.new(row_parsers.first.field_98.to_i + 2000, row_parsers.first.field_97.to_i, row_parsers.first.field_96.to_i)
+      Date.new(row_parsers.first.field_9.to_i + 2000, row_parsers.first.field_8.to_i, row_parsers.first.field_7.to_i)
     else
-      Date.new(rows.first[97].to_i + 2000, rows.first[96].to_i, rows.first[95].to_i)
+      Date.new(rows.first[8].to_i + 2000, rows.first[7].to_i, rows.first[6].to_i)
     end
   end
 end

app/services/bulk_upload/lettings/year2024/csv_parser.rb

@@ -0,0 +1,113 @@
+require "csv"
+
+class BulkUpload::Lettings::Year2024::CsvParser
+  include CollectionTimeHelper
+
+  FIELDS = 130
+  MAX_COLUMNS = 131
+  FORM_YEAR = 2024
+
+  attr_reader :path
+
+  def initialize(path:)
+    @path = path
+  end
+
+  def row_offset
+    if with_headers?
+      rows.find_index { |row| row[0].match(/field number/i) } + 1
+    else
+      0
+    end
+  end
+
+  def col_offset
+    with_headers? ? 1 : 0
+  end
+
+  def cols
+    @cols ||= ("A".."EA").to_a
+  end
+
+  def row_parsers
+    @row_parsers ||= body_rows.map do |row|
+      stripped_row = row[col_offset..]
+      hash = Hash[field_numbers.zip(stripped_row)]
+
+      BulkUpload::Lettings::Year2024::RowParser.new(hash)
+    end
+  end
+
+  def body_rows
+    rows[row_offset..]
+  end
+
+  def rows
+    @rows ||= CSV.parse(normalised_string, row_sep:)
+  end
+
+  def column_for_field(field)
+    cols[field_numbers.find_index(field) + col_offset]
+  end
+
+  def correct_field_count?
+    valid_field_numbers_count = field_numbers.count { |f| f != "field_blank" }
+
+    valid_field_numbers_count == FIELDS
+  end
+
+  def too_many_columns?
+    return if with_headers?
+
+    max_columns_count = body_rows.map(&:size).max - col_offset
+
+    max_columns_count > MAX_COLUMNS
+  end
+
+  def wrong_template_for_year?
+    collection_start_year_for_date(first_record_start_date) != FORM_YEAR
+  rescue Date::Error
+    false
+  end
+
+private
+
+  def default_field_numbers
+    (1..FIELDS).map { |h| h.present? && h.to_s.match?(/^[0-9]+$/) ? "field_#{h}" : "field_blank" }
+  end
+
+  def field_numbers
+    @field_numbers ||= if with_headers?
+                         rows[row_offset - 1][col_offset..].map { |h| h.present? && h.match?(/^[0-9]+$/) ? "field_#{h}" : "field_blank" }
+                       else
+                         default_field_numbers
+                       end
+  end
+
+  def with_headers?
+    rows.map { |r| r[0] }.any? { |cell| cell&.match?(/field number/i) }
+  end
+
+  def row_sep
+    "\n"
+  end
+
+  def normalised_string
+    return @normalised_string if @normalised_string
+
+    @normalised_string = File.read(path, encoding: "bom|utf-8")
+    @normalised_string.gsub!("\r\n", "\n")
+    @normalised_string.scrub!("")
+    @normalised_string.tr!("\r", "\n")
+
+    @normalised_string
+  end
+
+  def first_record_start_date
+    if with_headers?
+      Date.new(row_parsers.first.field_10.to_i + 2000, row_parsers.first.field_9.to_i, row_parsers.first.field_8.to_i)
+    else
+      Date.new(rows.first[9].to_i + 2000, rows.first[8].to_i, rows.first[7].to_i)
+    end
+  end
+end

app/views/bulk_upload_lettings_logs/forms/prepare_your_file_2024.html.erb

@@ -0,0 +1,50 @@
+<% content_for :before_content do %>
+  <%= govuk_back_link href: @form.back_path %>
+<% end %>
+
+<div class="govuk-grid-row">
+  <div class="govuk-grid-column-two-thirds">
+    <%= form_with model: @form, scope: :form, url: bulk_upload_lettings_log_path(id: "prepare-your-file"), method: :patch do |f| %>
+      <%= f.hidden_field :year %>
+
+      <span class="govuk-caption-l">Upload lettings logs in bulk (<%= @form.year_combo %>)</span>
+      <h1 class="govuk-heading-l">Prepare your file</h1>
+      <p class="govuk-body govuk-!-margin-bottom-2"><%= govuk_link_to "Read the full guidance", bulk_upload_lettings_log_path(id: "guidance", form: { year: @form.year }) %> before you start if you have not used bulk upload before.</p>
+
+      <h2 class="govuk-heading-s">Download template</h2>
+
+      <p class="govuk-body govuk-!-margin-bottom-2">Use one of these templates to upload logs for 2024/25:</p>
+      <ul class="govuk-list govuk-list--bullet">
+        <li>
+          <%= govuk_link_to "Download the new template", @form.template_path %>: In this template, the questions are in the same order as the 2024/25 paper form and web form.
+        </li>
+
+        <li>
+          <%= govuk_link_to "Download the legacy template", @form.legacy_template_path %>: In this template, the questions are in the same order as the 2022/23 template, with new questions added on to the end.
+        </li>
+      </ul>
+      <p class="govuk-body govuk-!-margin-bottom-2">There are 7 or 8 rows of content in the templates. These rows are called the ‘headers’. They contain the CORE form questions and guidance about which questions are required and how to format your answers.</p>
+
+      <h2 class="govuk-heading-s">Create your file</h2>
+
+      <ul class="govuk-list govuk-list--bullet">
+        <li>Fill in the template with data from your housing management system. Your data should go below the headers, with one row per log. Leave column A blank - the bulk upload fields start in column B.</li>
+        <li>Make sure each column of your data aligns with the matching headers above. You may need to reorder your data.</li>
+        <li>Use the   <%= govuk_link_to "Lettings #{@form.year_combo} Bulk Upload Specification", @form.specification_path %> to check your data is in the correct format.</li>
+        <li><strong>Username field:</strong> To assign a log to someone else, enter the email address they use to log into CORE.</li>
+        <li>If you are using the new template, keep the headers. If you are using the legacy template, you can either keep or remove the headers. If you remove the headers, you should also remove the blank column A.</li>
+      </ul>
+
+      <%= govuk_inset_text(text: "You can upload both general needs and supported housing logs in the same file for 2023/24 data.") %>
+
+      <h2 class="govuk-heading-s">Save your file</h2>
+
+      <ul class="govuk-list govuk-list--bullet">
+        <li>Save your file as a CSV.</li>
+        <li>Your file should now be ready to upload.</li>
+      </ul>
+
+      <%= f.govuk_submit class: "govuk-!-margin-top-7" %>
+    <% end %>
+  </div>
+</div>

spec/services/bulk_upload/lettings/validator_spec.rb

@@ -114,7 +114,7 @@ RSpec.describe BulkUpload::Lettings::Validator do
 
       context "when uploading a 2022 template for 2023 bulk upload" do
         let(:bulk_upload) { create(:bulk_upload, user:, year: 2023) }
-        let(:log) { build(:lettings_log, :completed, startdate: Time.zone.local(2022, 5, 6)) }
+        let(:log) { build(:lettings_log, :completed, startdate: Time.zone.local(2022, 5, 6), tenancycode: "5") }
 
         context "with no headers" do
           before do

spec/services/bulk_upload/lettings/year2024/csv_parser_spec.rb

@@ -0,0 +1,226 @@
+require "rails_helper"
+
+RSpec.describe BulkUpload::Lettings::Year2024::CsvParser do
+  subject(:service) { described_class.new(path:) }
+
+  let(:file) { Tempfile.new }
+  let(:path) { file.path }
+  let(:log) { build(:lettings_log, :completed) }
+
+  context "when parsing csv with headers" do
+    before do
+      file.write("Question\n")
+      file.write("Additional info\n")
+      file.write("Values\n")
+      file.write("Can be empty?\n")
+      file.write("Type of letting the question applies to\n")
+      file.write("Duplicate check field?\n")
+      file.write(BulkUpload::LettingsLogToCsv.new(log:).default_2024_field_numbers_row)
+      file.write(BulkUpload::LettingsLogToCsv.new(log:).to_2024_csv_row)
+      file.rewind
+    end
+
+    it "returns correct offsets" do
+      expect(service.row_offset).to eq(7)
+      expect(service.col_offset).to eq(1)
+    end
+
+    it "parses csv correctly" do
+      expect(service.row_parsers[0].field_13).to eql(log.tenancycode)
+    end
+  end
+
+  context "when parsing csv with headers with extra rows" do
+    before do
+      file.write("Section\n")
+      file.write("Question\n")
+      file.write("Additional info\n")
+      file.write("Values\n")
+      file.write("Can be empty?\n")
+      file.write("Type of letting the question applies to\n")
+      file.write("Duplicate check field?\n")
+      file.write(BulkUpload::LettingsLogToCsv.new(log:).default_2024_field_numbers_row)
+      file.write(BulkUpload::LettingsLogToCsv.new(log:).to_2024_csv_row)
+      file.rewind
+    end
+
+    it "returns correct offsets" do
+      expect(service.row_offset).to eq(8)
+      expect(service.col_offset).to eq(1)
+    end
+
+    it "parses csv correctly" do
+      expect(service.row_parsers[0].field_13).to eql(log.tenancycode)
+    end
+  end
+
+  context "when parsing csv with headers in arbitrary order" do
+    let(:seed) { rand }
+
+    before do
+      file.write("Question\n")
+      file.write("Additional info\n")
+      file.write("Values\n")
+      file.write("Can be empty?\n")
+      file.write("Type of letting the question applies to\n")
+      file.write("Duplicate check field?\n")
+      file.write(BulkUpload::LettingsLogToCsv.new(log:).default_2024_field_numbers_row(seed:))
+      file.write(BulkUpload::LettingsLogToCsv.new(log:).to_2024_csv_row(seed:))
+      file.rewind
+    end
+
+    it "returns correct offsets" do
+      expect(service.row_offset).to eq(7)
+      expect(service.col_offset).to eq(1)
+    end
+
+    it "parses csv correctly" do
+      expect(service.row_parsers[0].field_13).to eql(log.tenancycode)
+    end
+  end
+
+  context "when parsing csv with extra invalid headers" do
+    let(:seed) { rand }
+    let(:log_to_csv) { BulkUpload::LettingsLogToCsv.new(log:) }
+    let(:field_numbers) { log_to_csv.default_2024_field_numbers + %w[invalid_field_number] }
+    let(:field_values) { log_to_csv.to_2024_row + %w[value_for_invalid_field_number] }
+
+    before do
+      file.write("Question\n")
+      file.write("Additional info\n")
+      file.write("Values\n")
+      file.write("Can be empty?\n")
+      file.write("Type of letting the question applies to\n")
+      file.write("Duplicate check field?\n")
+      file.write(log_to_csv.custom_field_numbers_row(seed:, field_numbers:))
+      file.write(log_to_csv.to_custom_csv_row(seed:, field_values:))
+      file.rewind
+    end
+
+    it "parses csv correctly" do
+      expect(service.row_parsers[0].field_13).to eql(log.tenancycode)
+    end
+
+    it "counts the number of valid field numbers correctly" do
+      expect(service).to be_correct_field_count
+    end
+  end
+
+  context "when parsing csv without headers" do
+    before do
+      file.write(BulkUpload::LettingsLogToCsv.new(log:, col_offset: 0).to_2024_csv_row)
+      file.rewind
+    end
+
+    it "returns correct offsets" do
+      expect(service.row_offset).to eq(0)
+      expect(service.col_offset).to eq(0)
+    end
+
+    it "parses csv correctly" do
+      expect(service.row_parsers[0].field_13).to eql(log.tenancycode)
+    end
+  end
+
+  context "when parsing with BOM aka byte order mark" do
+    let(:bom) { "\uFEFF" }
+
+    before do
+      file.write(bom)
+      file.write(BulkUpload::LettingsLogToCsv.new(log:, col_offset: 0).to_2024_csv_row)
+      file.rewind
+    end
+
+    it "parses csv correctly" do
+      expect(service.row_parsers[0].field_13).to eql(log.tenancycode)
+    end
+  end
+
+  context "when an invalid byte sequence" do
+    let(:invalid_sequence) { "\x81" }
+
+    before do
+      file.write(invalid_sequence)
+      file.write(BulkUpload::LettingsLogToCsv.new(log:, col_offset: 0).to_2024_csv_row)
+      file.rewind
+    end
+
+    it "parses csv correctly" do
+      expect(service.row_parsers[0].field_13).to eql(log.tenancycode)
+    end
+  end
+
+  context "when parsing csv with carriage returns" do
+    before do
+      file.write("Question\r\n")
+      file.write("Additional info\r")
+      file.write("Values\r\n")
+      file.write("Can be empty?\r")
+      file.write("Type of letting the question applies to\r\n")
+      file.write("Duplicate check field?\r")
+      file.write(BulkUpload::LettingsLogToCsv.new(log:).default_2024_field_numbers_row)
+      file.write(BulkUpload::LettingsLogToCsv.new(log:).to_2024_csv_row)
+      file.rewind
+    end
+
+    it "parses csv correctly" do
+      expect(service.row_parsers[0].field_13).to eql(log.tenancycode)
+    end
+  end
+
+  describe "#column_for_field", aggregate_failures: true do
+    context "when with headers using default ordering" do
+      before do
+        file.write("Question\n")
+        file.write("Additional info\n")
+        file.write("Values\n")
+        file.write("Can be empty?\n")
+        file.write("Type of letting the question applies to\n")
+        file.write("Duplicate check field?\n")
+        file.write(BulkUpload::LettingsLogToCsv.new(log:).default_2024_field_numbers_row)
+        file.write(BulkUpload::LettingsLogToCsv.new(log:).to_2024_csv_row)
+        file.rewind
+      end
+
+      it "returns correct column" do
+        expect(service.column_for_field("field_5")).to eql("F")
+        expect(service.column_for_field("field_22")).to eql("W")
+      end
+    end
+
+    context "when without headers using default ordering" do
+      before do
+        file.write(BulkUpload::LettingsLogToCsv.new(log:, col_offset: 0).to_2024_csv_row)
+        file.rewind
+      end
+
+      it "returns correct column" do
+        expect(service.column_for_field("field_5")).to eql("E")
+        expect(service.column_for_field("field_22")).to eql("V")
+      end
+    end
+
+    context "when with headers using custom ordering" do
+      let(:seed) { 123 }
+
+      before do
+        file.write("Question\n")
+        file.write("Additional info\n")
+        file.write("Values\n")
+        file.write("Can be empty?\n")
+        file.write("Type of letting the question applies to\n")
+        file.write("Duplicate check field?\n")
+        file.write(BulkUpload::LettingsLogToCsv.new(log:).default_2024_field_numbers_row(seed:))
+        file.write(BulkUpload::LettingsLogToCsv.new(log:).to_2024_csv_row(seed:))
+        file.rewind
+      end
+
+      it "returns correct column" do
+        expect(service.column_for_field("field_5")).to eql("B")
+        expect(service.column_for_field("field_22")).to eql("AS")
+        expect(service.column_for_field("field_26")).to eql("DH")
+        expect(service.column_for_field("field_25")).to eql("I")
+      end
+    end
+  end
+end

spec/support/bulk_upload/lettings_log_to_csv.rb

@@ -42,6 +42,15 @@ class BulkUpload::LettingsLogToCsv
     end
   end
 
+  def to_2024_csv_row(seed: nil)
+    if seed
+      row = to_2024_row.shuffle(random: Random.new(seed))
+      (row_prefix + row).flatten.join(",") + line_ending
+    else
+      (row_prefix + to_2024_row).flatten.join(",") + line_ending
+    end
+  end
+
   def to_2023_row
     to_2022_row + [
       log.needstype,
@@ -62,10 +71,169 @@ class BulkUpload::LettingsLogToCsv
     end.flatten.join(",") + line_ending
   end
 
+  def default_2024_field_numbers_row(seed: nil)
+    if seed
+      ["Bulk upload field number"] + default_2024_field_numbers.shuffle(random: Random.new(seed))
+    else
+      ["Bulk upload field number"] + default_2024_field_numbers
+    end.flatten.join(",") + line_ending
+  end
+
   def default_2023_field_numbers
     [5, nil, nil, 15, 16, nil, 13, 40, 41, 42, 43, 46, 52, 56, 60, 64, 68, 72, 76, 47, 53, 57, 61, 65, 69, 73, 77, 51, 55, 59, 63, 67, 71, 75, 50, 54, 58, 62, 66, 70, 74, 78, 48, 49, 79, 81, 82, 123, 124, 122, 120, 102, 103, nil, 83, 84, 85, 86, 87, 88, 104, 109, 107, 108, 106, 100, 101, 105, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 126, 128, 129, 130, 131, 132, 127, 125, 133, 134, 33, 34, 35, 36, 37, 38, nil, 7, 8, 9, 28, 14, 32, 29, 30, 31, 26, 27, 25, 23, 24, nil, 1, 3, 2, 80, nil, 121, 44, 89, 98, 92, 95, 90, 91, 93, 94, 97, 96, 99, 10, 11, 12, 45, 39, 6, 4, 17, 18, 19, 20, 21, 22]
   end
 
+  def default_2024_field_numbers
+    (1..130).to_a
+  end
+
+  def to_2024_row
+    [
+      log.owning_organisation&.old_visible_id, # 1
+      log.managing_organisation&.old_visible_id,
+      log.created_by&.email,
+      log.needstype,
+      "S#{log.scheme&.id}",
+      log.location&.id,
+      renewal,
+      log.startdate&.day,
+      log.startdate&.month,
+      log.startdate&.strftime("%y"), # 10
+
+      london_affordable_rent,
+      log.irproduct_other,
+      log.tenancycode,
+      log.propcode,
+      log.declaration,
+      log.uprn,
+      log.address_line1,
+      log.address_line2,
+      log.town_or_city,
+      log.county, # 20
+
+      ((log.postcode_full || "").split(" ") || [""]).first,
+      ((log.postcode_full || "").split(" ") || [""]).last,
+      log.la,
+      log.rsnvac,
+      log.unitletas,
+      log.unittype_gn,
+      log.builtype,
+      log.wchair,
+      log.beds,
+      log.voiddate&.day, # 30
+
+      log.voiddate&.month,
+      log.voiddate&.strftime("%y"),
+      log.mrcdate&.day,
+      log.mrcdate&.month,
+      log.mrcdate&.strftime("%y"),
+      log.joint,
+      log.startertenancy,
+      log.tenancy,
+      log.tenancyother,
+      log.tenancylength, # 40
+
+      log.sheltered,
+      log.age1 || overrides[:age1],
+      log.sex1,
+      log.ethnic,
+      log.national,
+      log.ecstat1,
+      log.relat2,
+      log.age2 || overrides[:age2],
+      log.sex2,
+      log.ecstat2, # 50
+
+      log.relat3,
+      log.age3 || overrides[:age3],
+      log.sex3,
+      log.ecstat3,
+      log.relat4,
+      log.age4 || overrides[:age4],
+      log.sex4,
+      log.ecstat4,
+      log.relat5,
+      log.age5 || overrides[:age5], # 60
+
+      log.sex5,
+      log.ecstat5,
+      log.relat6,
+      log.age6 || overrides[:age6],
+      log.sex6,
+      log.ecstat6,
+      log.relat7,
+      log.age7 || overrides[:age7],
+      log.sex7,
+      log.ecstat7, # 70
+
+      log.relat8,
+      log.age8 || overrides[:age8],
+      log.sex8,
+      log.ecstat8,
+      log.armedforces,
+      log.leftreg,
+      log.reservist,
+      log.preg_occ,
+      log.housingneeds_a,
+      log.housingneeds_b, # 80
+
+      log.housingneeds_c,
+      log.housingneeds_f,
+      log.housingneeds_g,
+      log.housingneeds_h,
+      overrides[:illness] || log.illness,
+      log.illness_type_1,
+      log.illness_type_2,
+      log.illness_type_3,
+      log.illness_type_4,
+      log.illness_type_5, # 90
+
+      log.illness_type_6,
+      log.illness_type_7,
+      log.illness_type_8,
+      log.illness_type_9,
+      log.illness_type_10,
+      log.layear,
+      log.waityear,
+      log.reason,
+      log.reasonother,
+      log.prevten, # 100
+
+      homeless,
+      previous_postcode_known,
+      ((log.ppostcode_full || "").split(" ") || [""]).first,
+      ((log.ppostcode_full || "").split(" ") || [""]).last,
+      log.prevloc,
+      log.reasonpref,
+      log.rp_homeless,
+      log.rp_insan_unsat,
+      log.rp_medwel,
+      log.rp_hardship, # 110
+
+      log.rp_dontknow,
+      cbl,
+      chr,
+      cap,
+      nil, # accessible register
+      log.referral,
+      net_income_known,
+      log.earnings,
+      log.incfreq,
+      log.hb, # 120
+
+      log.benefits,
+      log.household_charge,
+      log.period,
+      log.chcharge,
+      log.brent,
+      log.scharge,
+      log.pscharge,
+      log.supcharg,
+      log.hbrentshortfall,
+      log.tshortfall, # 130
+    ]
+  end
+
   def to_2022_row
     [
       log.renttype, # 1