baarkerlounger
/
submit-social-housing-lettings-and-sales-data
mirror of https://github.com/communitiesuk/submit-social-housing-lettings-and-sales-data.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Update csv parser

pull/2939/head
Kat 3 months ago
parent
f8a818dfc3
commit
5240c60ba2
3 changed files with 201 additions and 26 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 175
      app/helpers/bulk_upload/lettings_log_to_csv.rb
  2. 10
      app/services/bulk_upload/lettings/year2025/csv_parser.rb
  3. 42
      spec/services/bulk_upload/lettings/year2025/csv_parser_spec.rb

175
app/helpers/bulk_upload/lettings_log_to_csv.rb
Unescape View File

@ -23,6 +23,8 @@ class BulkUpload::LettingsLogToCsv
to_2023_csv_row(seed:)
when 2024
to_2024_csv_row(seed:)
when 2025
to_2025_csv_row(seed:)
else
raise NotImplementedError "No mapping function implemented for year #{year}"
end
@ -37,6 +39,8 @@ class BulkUpload::LettingsLogToCsv
to_2023_row
when 2024
to_2024_row
when 2025
to_2025_row
else
raise NotImplementedError "No mapping function implemented for year #{year}"
end
@ -51,6 +55,8 @@ class BulkUpload::LettingsLogToCsv
default_2023_field_numbers_row(seed:)
when 2024
default_2024_field_numbers_row(seed:)
when 2025
default_2025_field_numbers_row(seed:)
else
raise NotImplementedError "No mapping function implemented for year #{year}"
end
@ -65,6 +71,8 @@ class BulkUpload::LettingsLogToCsv
default_2023_field_numbers
when 2024
default_2024_field_numbers
when 2025
default_2025_field_numbers
else
raise NotImplementedError "No mapping function implemented for year #{year}"
end
@ -109,6 +117,15 @@ class BulkUpload::LettingsLogToCsv
end
end
def to_2025_csv_row(seed: nil)
if seed
row = to_2025_row.shuffle(random: Random.new(seed))
(row_prefix + row).flatten.join(",") + line_ending
else
(row_prefix + to_2025_row).flatten.join(",") + line_ending
end
end
def to_2023_row
to_2022_row + [
log.needstype,
@ -129,6 +146,14 @@ class BulkUpload::LettingsLogToCsv
end.flatten.join(",") + line_ending
end
def default_2025_field_numbers_row(seed: nil)
if seed
["Field number"] + default_2025_field_numbers.shuffle(random: Random.new(seed))
else
["Field number"] + default_2025_field_numbers
end.flatten.join(",") + line_ending
end
def default_2024_field_numbers_row(seed: nil)
if seed
["Field number"] + default_2024_field_numbers.shuffle(random: Random.new(seed))
@ -141,10 +166,160 @@ class BulkUpload::LettingsLogToCsv
[5, nil, nil, 15, 16, nil, 13, 40, 41, 42, 43, 46, 52, 56, 60, 64, 68, 72, 76, 47, 53, 57, 61, 65, 69, 73, 77, 51, 55, 59, 63, 67, 71, 75, 50, 54, 58, 62, 66, 70, 74, 78, 48, 49, 79, 81, 82, 123, 124, 122, 120, 102, 103, nil, 83, 84, 85, 86, 87, 88, 104, 109, 107, 108, 106, 100, 101, 105, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 126, 128, 129, 130, 131, 132, 127, 125, 133, 134, 33, 34, 35, 36, 37, 38, nil, 7, 8, 9, 28, 14, 32, 29, 30, 31, 26, 27, 25, 23, 24, nil, 1, 3, 2, 80, nil, 121, 44, 89, 98, 92, 95, 90, 91, 93, 94, 97, 96, 99, 10, 11, 12, 45, 39, 6, 4, 17, 18, 19, 20, 21, 22]
end
def default_2025_field_numbers
(1..129).to_a
end
def default_2024_field_numbers
(1..130).to_a
end
def to_2025_row
[
overrides[:organisation_id] || log.owning_organisation&.old_visible_id, # 1
overrides[:managing_organisation_id] || log.managing_organisation&.old_visible_id,
log.assigned_to&.email,
log.needstype,
log.scheme&.id ? "S#{log.scheme&.id}" : "",
log.location&.id,
renewal,
log.startdate&.day,
log.startdate&.month,
log.startdate&.strftime("%y"), # 10
rent_type,
log.irproduct_other,
log.tenancycode,
log.propcode,
log.declaration,
log.unittype_gn,
log.unitletas,
log.uprn,
log.address_line1&.tr(",", " "),
log.address_line2&.tr(",", " "), # 20
log.town_or_city&.tr(",", " "),
log.county&.tr(",", " "),
((log.postcode_full || "").split(" ") || [""]).first,
((log.postcode_full || "").split(" ") || [""]).last,
log.la,
log.rsnvac,
log.builtype,
log.wchair,
log.beds,
log.voiddate&.day, # 30
log.voiddate&.month,
log.voiddate&.strftime("%y"),
log.mrcdate&.day,
log.mrcdate&.month,
log.mrcdate&.strftime("%y"),
log.sheltered,
log.joint,
log.startertenancy,
log.tenancy,
log.tenancyother, # 40
log.tenancylength,
log.age1 || overrides[:age1],
log.sex1,
log.ethnic,
log.nationality_all_group,
log.ecstat1,
log.relat2,
log.age2 || overrides[:age2],
log.sex2,
log.ecstat2, # 50
log.relat3,
log.age3 || overrides[:age3],
log.sex3,
log.ecstat3,
log.relat4,
log.age4 || overrides[:age4],
log.sex4,
log.ecstat4,
log.relat5,
log.age5 || overrides[:age5], # 60
log.sex5,
log.ecstat5,
log.relat6,
log.age6 || overrides[:age6],
log.sex6,
log.ecstat6,
log.relat7,
log.age7 || overrides[:age7],
log.sex7,
log.ecstat7, # 70
log.relat8,
log.age8 || overrides[:age8],
log.sex8,
log.ecstat8,
log.armedforces,
log.leftreg,
log.reservist,
log.preg_occ,
log.housingneeds_a,
log.housingneeds_b, # 80
log.housingneeds_c,
log.housingneeds_f,
log.housingneeds_g,
log.housingneeds_h,
overrides[:illness] || log.illness,
log.illness_type_1,
log.illness_type_2,
log.illness_type_3,
log.illness_type_4,
log.illness_type_5, # 90
log.illness_type_6,
log.illness_type_7,
log.illness_type_8,
log.illness_type_9,
log.illness_type_10,
log.layear,
log.waityear,
log.reason,
log.reasonother,
log.prevten, # 100
homeless,
previous_postcode_known,
((log.ppostcode_full || "").split(" ") || [""]).first,
((log.ppostcode_full || "").split(" ") || [""]).last,
log.prevloc,
log.reasonpref,
log.rp_homeless,
log.rp_insan_unsat,
log.rp_medwel,
log.rp_hardship, # 110
log.rp_dontknow,
cbl,
chr,
cap,
accessible_register,
log.referral,
net_income_known,
log.incfreq,
log.earnings,
log.hb, # 120
log.benefits,
log.household_charge,
log.period,
log.brent,
log.scharge,
log.pscharge,
log.supcharg,
log.hbrentshortfall,
log.tshortfall, # 129
]
end
def to_2024_row
[
overrides[:organisation_id] || log.owning_organisation&.old_visible_id, # 1

10
app/services/bulk_upload/lettings/year2025/csv_parser.rb
Unescape View File

@ -3,9 +3,9 @@ require "csv"
class BulkUpload::Lettings::Year2025::CsvParser
include CollectionTimeHelper
FIELDS = 130
MAX_COLUMNS = 131
FORM_YEAR = 2024
FIELDS = 129
MAX_COLUMNS = 130
FORM_YEAR = 2025
attr_reader :path
@ -26,7 +26,7 @@ class BulkUpload::Lettings::Year2025::CsvParser
end
def cols
@cols ||= ("A".."EA").to_a
@cols ||= ("A".."DZ").to_a
end
def row_parsers
@ -37,7 +37,7 @@ class BulkUpload::Lettings::Year2025::CsvParser
hash = Hash[field_numbers.zip(stripped_row)]
BulkUpload::Lettings::Year2024::RowParser.new(hash)
BulkUpload::Lettings::Year2025::RowParser.new(hash)
}.compact
end

42
spec/services/bulk_upload/lettings/year2025/csv_parser_spec.rb
Unescape View File

@ -15,8 +15,8 @@ RSpec.describe BulkUpload::Lettings::Year2025::CsvParser do
file.write("Can be empty?\n")
file.write("Type of letting the question applies to\n")
file.write("Duplicate check field?\n")
file.write(BulkUpload::LettingsLogToCsv.new(log:).default_2024_field_numbers_row)
file.write(BulkUpload::LettingsLogToCsv.new(log:).to_2024_csv_row)
file.write(BulkUpload::LettingsLogToCsv.new(log:).default_2025_field_numbers_row)
file.write(BulkUpload::LettingsLogToCsv.new(log:).to_2025_csv_row)
file.rewind
end
@ -38,8 +38,8 @@ RSpec.describe BulkUpload::Lettings::Year2025::CsvParser do
file.write("\n")
file.write("Type of letting the question applies to\n")
file.write("Duplicate check field?\n")
file.write(BulkUpload::LettingsLogToCsv.new(log:).default_2024_field_numbers_row)
file.write(BulkUpload::LettingsLogToCsv.new(log:).to_2024_csv_row)
file.write(BulkUpload::LettingsLogToCsv.new(log:).default_2025_field_numbers_row)
file.write(BulkUpload::LettingsLogToCsv.new(log:).to_2025_csv_row)
file.rewind
end
@ -62,8 +62,8 @@ RSpec.describe BulkUpload::Lettings::Year2025::CsvParser do
file.write("Can be empty?\n")
file.write("Type of letting the question applies to\n")
file.write("Duplicate check field?\n")
file.write(BulkUpload::LettingsLogToCsv.new(log:).default_2024_field_numbers_row)
file.write(BulkUpload::LettingsLogToCsv.new(log:).to_2024_csv_row)
file.write(BulkUpload::LettingsLogToCsv.new(log:).default_2025_field_numbers_row)
file.write(BulkUpload::LettingsLogToCsv.new(log:).to_2025_csv_row)
file.write("\n")
file.rewind
end
@ -92,8 +92,8 @@ RSpec.describe BulkUpload::Lettings::Year2025::CsvParser do
file.write("Can be empty?\n")
file.write("Type of letting the question applies to\n")
file.write("Duplicate check field?\n")
file.write(BulkUpload::LettingsLogToCsv.new(log:).default_2024_field_numbers_row(seed:))
file.write(BulkUpload::LettingsLogToCsv.new(log:).to_2024_csv_row(seed:))
file.write(BulkUpload::LettingsLogToCsv.new(log:).default_2025_field_numbers_row(seed:))
file.write(BulkUpload::LettingsLogToCsv.new(log:).to_2025_csv_row(seed:))
file.rewind
end
@ -110,8 +110,8 @@ RSpec.describe BulkUpload::Lettings::Year2025::CsvParser do
context "when parsing csv with extra invalid headers" do
let(:seed) { rand }
let(:log_to_csv) { BulkUpload::LettingsLogToCsv.new(log:) }
let(:field_numbers) { log_to_csv.default_2024_field_numbers + %w[invalid_field_number] }
let(:field_values) { log_to_csv.to_2024_row + %w[value_for_invalid_field_number] }
let(:field_numbers) { log_to_csv.default_2025_field_numbers + %w[invalid_field_number] }
let(:field_values) { log_to_csv.to_2025_row + %w[value_for_invalid_field_number] }
before do
file.write("Question\n")
@ -136,7 +136,7 @@ RSpec.describe BulkUpload::Lettings::Year2025::CsvParser do
context "when parsing csv without headers" do
before do
file.write(BulkUpload::LettingsLogToCsv.new(log:, col_offset: 0).to_2024_csv_row)
file.write(BulkUpload::LettingsLogToCsv.new(log:, col_offset: 0).to_2025_csv_row)
file.rewind
end
@ -155,7 +155,7 @@ RSpec.describe BulkUpload::Lettings::Year2025::CsvParser do
before do
file.write(bom)
file.write(BulkUpload::LettingsLogToCsv.new(log:, col_offset: 0).to_2024_csv_row)
file.write(BulkUpload::LettingsLogToCsv.new(log:, col_offset: 0).to_2025_csv_row)
file.rewind
end
@ -169,7 +169,7 @@ RSpec.describe BulkUpload::Lettings::Year2025::CsvParser do
before do
file.write(invalid_sequence)
file.write(BulkUpload::LettingsLogToCsv.new(log:, col_offset: 0).to_2024_csv_row)
file.write(BulkUpload::LettingsLogToCsv.new(log:, col_offset: 0).to_2025_csv_row)
file.rewind
end
@ -186,8 +186,8 @@ RSpec.describe BulkUpload::Lettings::Year2025::CsvParser do
file.write("Can be empty?\r")
file.write("Type of letting the question applies to\r\n")
file.write("Duplicate check field?\r")
file.write(BulkUpload::LettingsLogToCsv.new(log:).default_2024_field_numbers_row)
file.write(BulkUpload::LettingsLogToCsv.new(log:).to_2024_csv_row)
file.write(BulkUpload::LettingsLogToCsv.new(log:).default_2025_field_numbers_row)
file.write(BulkUpload::LettingsLogToCsv.new(log:).to_2025_csv_row)
file.rewind
end
@ -205,8 +205,8 @@ RSpec.describe BulkUpload::Lettings::Year2025::CsvParser do
file.write("Can be empty?\n")
file.write("Type of letting the question applies to\n")
file.write("Duplicate check field?\n")
file.write(BulkUpload::LettingsLogToCsv.new(log:).default_2024_field_numbers_row)
file.write(BulkUpload::LettingsLogToCsv.new(log:).to_2024_csv_row)
file.write(BulkUpload::LettingsLogToCsv.new(log:).default_2025_field_numbers_row)
file.write(BulkUpload::LettingsLogToCsv.new(log:).to_2025_csv_row)
file.rewind
end
@ -218,7 +218,7 @@ RSpec.describe BulkUpload::Lettings::Year2025::CsvParser do
context "when without headers using default ordering" do
before do
file.write(BulkUpload::LettingsLogToCsv.new(log:, col_offset: 0).to_2024_csv_row)
file.write(BulkUpload::LettingsLogToCsv.new(log:, col_offset: 0).to_2025_csv_row)
file.rewind
end
@ -238,15 +238,15 @@ RSpec.describe BulkUpload::Lettings::Year2025::CsvParser do
file.write("Can be empty?\n")
file.write("Type of letting the question applies to\n")
file.write("Duplicate check field?\n")
file.write(BulkUpload::LettingsLogToCsv.new(log:).default_2024_field_numbers_row(seed:))
file.write(BulkUpload::LettingsLogToCsv.new(log:).to_2024_csv_row(seed:))
file.write(BulkUpload::LettingsLogToCsv.new(log:).default_2025_field_numbers_row(seed:))
file.write(BulkUpload::LettingsLogToCsv.new(log:).to_2025_csv_row(seed:))
file.rewind
end
it "returns correct column" do
expect(service.column_for_field("field_5")).to eql("B")
expect(service.column_for_field("field_22")).to eql("AS")
expect(service.column_for_field("field_26")).to eql("DH")
expect(service.column_for_field("field_26")).to eql("DG")
expect(service.column_for_field("field_25")).to eql("I")
end
end

Write Preview
Loading…
Cancel
Save