From 5240c60ba27f8a76902c0e37e9524c109ecc1b96 Mon Sep 17 00:00:00 2001 From: Kat <54268893+kosiakkatrina@users.noreply.github.com> Date: Wed, 12 Feb 2025 08:26:46 +0000 Subject: [PATCH] Update csv parser --- .../bulk_upload/lettings_log_to_csv.rb | 175 ++++++++++++++++++ .../lettings/year2025/csv_parser.rb | 10 +- .../lettings/year2025/csv_parser_spec.rb | 42 ++--- 3 files changed, 201 insertions(+), 26 deletions(-) diff --git a/app/helpers/bulk_upload/lettings_log_to_csv.rb b/app/helpers/bulk_upload/lettings_log_to_csv.rb index cdf38db17..11c180f3e 100644 --- a/app/helpers/bulk_upload/lettings_log_to_csv.rb +++ b/app/helpers/bulk_upload/lettings_log_to_csv.rb @@ -23,6 +23,8 @@ class BulkUpload::LettingsLogToCsv to_2023_csv_row(seed:) when 2024 to_2024_csv_row(seed:) + when 2025 + to_2025_csv_row(seed:) else raise NotImplementedError "No mapping function implemented for year #{year}" end @@ -37,6 +39,8 @@ class BulkUpload::LettingsLogToCsv to_2023_row when 2024 to_2024_row + when 2025 + to_2025_row else raise NotImplementedError "No mapping function implemented for year #{year}" end @@ -51,6 +55,8 @@ class BulkUpload::LettingsLogToCsv default_2023_field_numbers_row(seed:) when 2024 default_2024_field_numbers_row(seed:) + when 2025 + default_2025_field_numbers_row(seed:) else raise NotImplementedError "No mapping function implemented for year #{year}" end @@ -65,6 +71,8 @@ class BulkUpload::LettingsLogToCsv default_2023_field_numbers when 2024 default_2024_field_numbers + when 2025 + default_2025_field_numbers else raise NotImplementedError "No mapping function implemented for year #{year}" end @@ -109,6 +117,15 @@ class BulkUpload::LettingsLogToCsv end end + def to_2025_csv_row(seed: nil) + if seed + row = to_2025_row.shuffle(random: Random.new(seed)) + (row_prefix + row).flatten.join(",") + line_ending + else + (row_prefix + to_2025_row).flatten.join(",") + line_ending + end + end + def to_2023_row to_2022_row + [ log.needstype, @@ -129,6 +146,14 @@ class BulkUpload::LettingsLogToCsv end.flatten.join(",") + line_ending end + def default_2025_field_numbers_row(seed: nil) + if seed + ["Field number"] + default_2025_field_numbers.shuffle(random: Random.new(seed)) + else + ["Field number"] + default_2025_field_numbers + end.flatten.join(",") + line_ending + end + def default_2024_field_numbers_row(seed: nil) if seed ["Field number"] + default_2024_field_numbers.shuffle(random: Random.new(seed)) @@ -141,10 +166,160 @@ class BulkUpload::LettingsLogToCsv [5, nil, nil, 15, 16, nil, 13, 40, 41, 42, 43, 46, 52, 56, 60, 64, 68, 72, 76, 47, 53, 57, 61, 65, 69, 73, 77, 51, 55, 59, 63, 67, 71, 75, 50, 54, 58, 62, 66, 70, 74, 78, 48, 49, 79, 81, 82, 123, 124, 122, 120, 102, 103, nil, 83, 84, 85, 86, 87, 88, 104, 109, 107, 108, 106, 100, 101, 105, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 126, 128, 129, 130, 131, 132, 127, 125, 133, 134, 33, 34, 35, 36, 37, 38, nil, 7, 8, 9, 28, 14, 32, 29, 30, 31, 26, 27, 25, 23, 24, nil, 1, 3, 2, 80, nil, 121, 44, 89, 98, 92, 95, 90, 91, 93, 94, 97, 96, 99, 10, 11, 12, 45, 39, 6, 4, 17, 18, 19, 20, 21, 22] end + def default_2025_field_numbers + (1..129).to_a + end + def default_2024_field_numbers (1..130).to_a end + def to_2025_row + [ + overrides[:organisation_id] || log.owning_organisation&.old_visible_id, # 1 + overrides[:managing_organisation_id] || log.managing_organisation&.old_visible_id, + log.assigned_to&.email, + log.needstype, + log.scheme&.id ? "S#{log.scheme&.id}" : "", + log.location&.id, + renewal, + log.startdate&.day, + log.startdate&.month, + log.startdate&.strftime("%y"), # 10 + + rent_type, + log.irproduct_other, + log.tenancycode, + log.propcode, + log.declaration, + log.unittype_gn, + log.unitletas, + log.uprn, + log.address_line1&.tr(",", " "), + log.address_line2&.tr(",", " "), # 20 + log.town_or_city&.tr(",", " "), + log.county&.tr(",", " "), + + ((log.postcode_full || "").split(" ") || [""]).first, + ((log.postcode_full || "").split(" ") || [""]).last, + log.la, + log.rsnvac, + log.builtype, + log.wchair, + log.beds, + log.voiddate&.day, # 30 + + log.voiddate&.month, + log.voiddate&.strftime("%y"), + log.mrcdate&.day, + log.mrcdate&.month, + log.mrcdate&.strftime("%y"), + log.sheltered, + log.joint, + log.startertenancy, + log.tenancy, + log.tenancyother, # 40 + + log.tenancylength, + log.age1 || overrides[:age1], + log.sex1, + log.ethnic, + log.nationality_all_group, + log.ecstat1, + log.relat2, + log.age2 || overrides[:age2], + log.sex2, + log.ecstat2, # 50 + + log.relat3, + log.age3 || overrides[:age3], + log.sex3, + log.ecstat3, + log.relat4, + log.age4 || overrides[:age4], + log.sex4, + log.ecstat4, + log.relat5, + log.age5 || overrides[:age5], # 60 + + log.sex5, + log.ecstat5, + log.relat6, + log.age6 || overrides[:age6], + log.sex6, + log.ecstat6, + log.relat7, + log.age7 || overrides[:age7], + log.sex7, + log.ecstat7, # 70 + + log.relat8, + log.age8 || overrides[:age8], + log.sex8, + log.ecstat8, + log.armedforces, + log.leftreg, + log.reservist, + log.preg_occ, + log.housingneeds_a, + log.housingneeds_b, # 80 + + log.housingneeds_c, + log.housingneeds_f, + log.housingneeds_g, + log.housingneeds_h, + overrides[:illness] || log.illness, + log.illness_type_1, + log.illness_type_2, + log.illness_type_3, + log.illness_type_4, + log.illness_type_5, # 90 + + log.illness_type_6, + log.illness_type_7, + log.illness_type_8, + log.illness_type_9, + log.illness_type_10, + log.layear, + log.waityear, + log.reason, + log.reasonother, + log.prevten, # 100 + + homeless, + previous_postcode_known, + ((log.ppostcode_full || "").split(" ") || [""]).first, + ((log.ppostcode_full || "").split(" ") || [""]).last, + log.prevloc, + log.reasonpref, + log.rp_homeless, + log.rp_insan_unsat, + log.rp_medwel, + log.rp_hardship, # 110 + + log.rp_dontknow, + cbl, + chr, + cap, + accessible_register, + log.referral, + net_income_known, + log.incfreq, + log.earnings, + log.hb, # 120 + + log.benefits, + log.household_charge, + log.period, + log.brent, + log.scharge, + log.pscharge, + log.supcharg, + log.hbrentshortfall, + log.tshortfall, # 129 + ] + end + def to_2024_row [ overrides[:organisation_id] || log.owning_organisation&.old_visible_id, # 1 diff --git a/app/services/bulk_upload/lettings/year2025/csv_parser.rb b/app/services/bulk_upload/lettings/year2025/csv_parser.rb index d5c4b99a2..ec6c33b6d 100644 --- a/app/services/bulk_upload/lettings/year2025/csv_parser.rb +++ b/app/services/bulk_upload/lettings/year2025/csv_parser.rb @@ -3,9 +3,9 @@ require "csv" class BulkUpload::Lettings::Year2025::CsvParser include CollectionTimeHelper - FIELDS = 130 - MAX_COLUMNS = 131 - FORM_YEAR = 2024 + FIELDS = 129 + MAX_COLUMNS = 130 + FORM_YEAR = 2025 attr_reader :path @@ -26,7 +26,7 @@ class BulkUpload::Lettings::Year2025::CsvParser end def cols - @cols ||= ("A".."EA").to_a + @cols ||= ("A".."DZ").to_a end def row_parsers @@ -37,7 +37,7 @@ class BulkUpload::Lettings::Year2025::CsvParser hash = Hash[field_numbers.zip(stripped_row)] - BulkUpload::Lettings::Year2024::RowParser.new(hash) + BulkUpload::Lettings::Year2025::RowParser.new(hash) }.compact end diff --git a/spec/services/bulk_upload/lettings/year2025/csv_parser_spec.rb b/spec/services/bulk_upload/lettings/year2025/csv_parser_spec.rb index 3661fc725..924171724 100644 --- a/spec/services/bulk_upload/lettings/year2025/csv_parser_spec.rb +++ b/spec/services/bulk_upload/lettings/year2025/csv_parser_spec.rb @@ -15,8 +15,8 @@ RSpec.describe BulkUpload::Lettings::Year2025::CsvParser do file.write("Can be empty?\n") file.write("Type of letting the question applies to\n") file.write("Duplicate check field?\n") - file.write(BulkUpload::LettingsLogToCsv.new(log:).default_2024_field_numbers_row) - file.write(BulkUpload::LettingsLogToCsv.new(log:).to_2024_csv_row) + file.write(BulkUpload::LettingsLogToCsv.new(log:).default_2025_field_numbers_row) + file.write(BulkUpload::LettingsLogToCsv.new(log:).to_2025_csv_row) file.rewind end @@ -38,8 +38,8 @@ RSpec.describe BulkUpload::Lettings::Year2025::CsvParser do file.write("\n") file.write("Type of letting the question applies to\n") file.write("Duplicate check field?\n") - file.write(BulkUpload::LettingsLogToCsv.new(log:).default_2024_field_numbers_row) - file.write(BulkUpload::LettingsLogToCsv.new(log:).to_2024_csv_row) + file.write(BulkUpload::LettingsLogToCsv.new(log:).default_2025_field_numbers_row) + file.write(BulkUpload::LettingsLogToCsv.new(log:).to_2025_csv_row) file.rewind end @@ -62,8 +62,8 @@ RSpec.describe BulkUpload::Lettings::Year2025::CsvParser do file.write("Can be empty?\n") file.write("Type of letting the question applies to\n") file.write("Duplicate check field?\n") - file.write(BulkUpload::LettingsLogToCsv.new(log:).default_2024_field_numbers_row) - file.write(BulkUpload::LettingsLogToCsv.new(log:).to_2024_csv_row) + file.write(BulkUpload::LettingsLogToCsv.new(log:).default_2025_field_numbers_row) + file.write(BulkUpload::LettingsLogToCsv.new(log:).to_2025_csv_row) file.write("\n") file.rewind end @@ -92,8 +92,8 @@ RSpec.describe BulkUpload::Lettings::Year2025::CsvParser do file.write("Can be empty?\n") file.write("Type of letting the question applies to\n") file.write("Duplicate check field?\n") - file.write(BulkUpload::LettingsLogToCsv.new(log:).default_2024_field_numbers_row(seed:)) - file.write(BulkUpload::LettingsLogToCsv.new(log:).to_2024_csv_row(seed:)) + file.write(BulkUpload::LettingsLogToCsv.new(log:).default_2025_field_numbers_row(seed:)) + file.write(BulkUpload::LettingsLogToCsv.new(log:).to_2025_csv_row(seed:)) file.rewind end @@ -110,8 +110,8 @@ RSpec.describe BulkUpload::Lettings::Year2025::CsvParser do context "when parsing csv with extra invalid headers" do let(:seed) { rand } let(:log_to_csv) { BulkUpload::LettingsLogToCsv.new(log:) } - let(:field_numbers) { log_to_csv.default_2024_field_numbers + %w[invalid_field_number] } - let(:field_values) { log_to_csv.to_2024_row + %w[value_for_invalid_field_number] } + let(:field_numbers) { log_to_csv.default_2025_field_numbers + %w[invalid_field_number] } + let(:field_values) { log_to_csv.to_2025_row + %w[value_for_invalid_field_number] } before do file.write("Question\n") @@ -136,7 +136,7 @@ RSpec.describe BulkUpload::Lettings::Year2025::CsvParser do context "when parsing csv without headers" do before do - file.write(BulkUpload::LettingsLogToCsv.new(log:, col_offset: 0).to_2024_csv_row) + file.write(BulkUpload::LettingsLogToCsv.new(log:, col_offset: 0).to_2025_csv_row) file.rewind end @@ -155,7 +155,7 @@ RSpec.describe BulkUpload::Lettings::Year2025::CsvParser do before do file.write(bom) - file.write(BulkUpload::LettingsLogToCsv.new(log:, col_offset: 0).to_2024_csv_row) + file.write(BulkUpload::LettingsLogToCsv.new(log:, col_offset: 0).to_2025_csv_row) file.rewind end @@ -169,7 +169,7 @@ RSpec.describe BulkUpload::Lettings::Year2025::CsvParser do before do file.write(invalid_sequence) - file.write(BulkUpload::LettingsLogToCsv.new(log:, col_offset: 0).to_2024_csv_row) + file.write(BulkUpload::LettingsLogToCsv.new(log:, col_offset: 0).to_2025_csv_row) file.rewind end @@ -186,8 +186,8 @@ RSpec.describe BulkUpload::Lettings::Year2025::CsvParser do file.write("Can be empty?\r") file.write("Type of letting the question applies to\r\n") file.write("Duplicate check field?\r") - file.write(BulkUpload::LettingsLogToCsv.new(log:).default_2024_field_numbers_row) - file.write(BulkUpload::LettingsLogToCsv.new(log:).to_2024_csv_row) + file.write(BulkUpload::LettingsLogToCsv.new(log:).default_2025_field_numbers_row) + file.write(BulkUpload::LettingsLogToCsv.new(log:).to_2025_csv_row) file.rewind end @@ -205,8 +205,8 @@ RSpec.describe BulkUpload::Lettings::Year2025::CsvParser do file.write("Can be empty?\n") file.write("Type of letting the question applies to\n") file.write("Duplicate check field?\n") - file.write(BulkUpload::LettingsLogToCsv.new(log:).default_2024_field_numbers_row) - file.write(BulkUpload::LettingsLogToCsv.new(log:).to_2024_csv_row) + file.write(BulkUpload::LettingsLogToCsv.new(log:).default_2025_field_numbers_row) + file.write(BulkUpload::LettingsLogToCsv.new(log:).to_2025_csv_row) file.rewind end @@ -218,7 +218,7 @@ RSpec.describe BulkUpload::Lettings::Year2025::CsvParser do context "when without headers using default ordering" do before do - file.write(BulkUpload::LettingsLogToCsv.new(log:, col_offset: 0).to_2024_csv_row) + file.write(BulkUpload::LettingsLogToCsv.new(log:, col_offset: 0).to_2025_csv_row) file.rewind end @@ -238,15 +238,15 @@ RSpec.describe BulkUpload::Lettings::Year2025::CsvParser do file.write("Can be empty?\n") file.write("Type of letting the question applies to\n") file.write("Duplicate check field?\n") - file.write(BulkUpload::LettingsLogToCsv.new(log:).default_2024_field_numbers_row(seed:)) - file.write(BulkUpload::LettingsLogToCsv.new(log:).to_2024_csv_row(seed:)) + file.write(BulkUpload::LettingsLogToCsv.new(log:).default_2025_field_numbers_row(seed:)) + file.write(BulkUpload::LettingsLogToCsv.new(log:).to_2025_csv_row(seed:)) file.rewind end it "returns correct column" do expect(service.column_for_field("field_5")).to eql("B") expect(service.column_for_field("field_22")).to eql("AS") - expect(service.column_for_field("field_26")).to eql("DH") + expect(service.column_for_field("field_26")).to eql("DG") expect(service.column_for_field("field_25")).to eql("I") end end