From 19335a2367b58c20a24f2af1f047642d9bb17de2 Mon Sep 17 00:00:00 2001 From: natdeanlewissoftwire <94526761+natdeanlewissoftwire@users.noreply.github.com> Date: Mon, 12 Jun 2023 11:03:48 +0100 Subject: [PATCH] CLDC-2438 Fix carriage return csv bug (#1685) * feat: add extra subs to row_parsers * feat: add tests * refactor: gsub not tr * feat: remove redundant sub * feat: simplify replacement * feat: scrub before tr to fix invalid byte error * feat: add ! --- .../lettings/year2022/csv_parser.rb | 1 + .../lettings/year2023/csv_parser.rb | 1 + .../bulk_upload/sales/year2022/csv_parser.rb | 1 + .../bulk_upload/sales/year2023/csv_parser.rb | 1 + .../lettings/year2022/csv_parser_spec.rb | 18 +++++++++++++++ .../lettings/year2023/csv_parser_spec.rb | 18 +++++++++++++++ .../sales/year2022/csv_parser_spec.rb | 22 +++++++++++++++++++ .../sales/year2023/csv_parser_spec.rb | 18 +++++++++++++++ 8 files changed, 80 insertions(+) diff --git a/app/services/bulk_upload/lettings/year2022/csv_parser.rb b/app/services/bulk_upload/lettings/year2022/csv_parser.rb index 663964195..f5bba15bf 100644 --- a/app/services/bulk_upload/lettings/year2022/csv_parser.rb +++ b/app/services/bulk_upload/lettings/year2022/csv_parser.rb @@ -93,6 +93,7 @@ private @normalised_string = File.read(path, encoding: "bom|utf-8") @normalised_string.gsub!("\r\n", "\n") @normalised_string.scrub!("") + @normalised_string.tr!("\r", "\n") @normalised_string end diff --git a/app/services/bulk_upload/lettings/year2023/csv_parser.rb b/app/services/bulk_upload/lettings/year2023/csv_parser.rb index 632e18329..a51f215fb 100644 --- a/app/services/bulk_upload/lettings/year2023/csv_parser.rb +++ b/app/services/bulk_upload/lettings/year2023/csv_parser.rb @@ -98,6 +98,7 @@ private @normalised_string = File.read(path, encoding: "bom|utf-8") @normalised_string.gsub!("\r\n", "\n") @normalised_string.scrub!("") + @normalised_string.tr!("\r", "\n") @normalised_string end diff --git a/app/services/bulk_upload/sales/year2022/csv_parser.rb b/app/services/bulk_upload/sales/year2022/csv_parser.rb index 76604d5fe..4bae79d33 100644 --- a/app/services/bulk_upload/sales/year2022/csv_parser.rb +++ b/app/services/bulk_upload/sales/year2022/csv_parser.rb @@ -67,6 +67,7 @@ private @normalised_string = File.read(path, encoding: "bom|utf-8") @normalised_string.gsub!("\r\n", "\n") @normalised_string.scrub!("") + @normalised_string.tr!("\r", "\n") @normalised_string end diff --git a/app/services/bulk_upload/sales/year2023/csv_parser.rb b/app/services/bulk_upload/sales/year2023/csv_parser.rb index 6a1929b5c..abbc51632 100644 --- a/app/services/bulk_upload/sales/year2023/csv_parser.rb +++ b/app/services/bulk_upload/sales/year2023/csv_parser.rb @@ -93,6 +93,7 @@ private @normalised_string = File.read(path, encoding: "bom|utf-8") @normalised_string.gsub!("\r\n", "\n") @normalised_string.scrub!("") + @normalised_string.tr!("\r", "\n") @normalised_string end diff --git a/spec/services/bulk_upload/lettings/year2022/csv_parser_spec.rb b/spec/services/bulk_upload/lettings/year2022/csv_parser_spec.rb index f2b9dda6a..5ae063567 100644 --- a/spec/services/bulk_upload/lettings/year2022/csv_parser_spec.rb +++ b/spec/services/bulk_upload/lettings/year2022/csv_parser_spec.rb @@ -169,4 +169,22 @@ RSpec.describe BulkUpload::Lettings::Year2022::CsvParser do end end end + + context "when parsing csv with carriage returns" do + before do + file.write("Question\r\n") + file.write("Additional info\r") + file.write("Values\r\n") + file.write("Can be empty?\r") + file.write("Type of letting the question applies to\r\n") + file.write("Duplicate check field?\r") + file.write(BulkUpload::LettingsLogToCsv.new(log:).default_2022_field_numbers_row) + file.write(BulkUpload::LettingsLogToCsv.new(log:).to_2022_csv_row) + file.rewind + end + + it "parses csv correctly" do + expect(service.row_parsers[0].field_12.to_i).to eq(35) + end + end end diff --git a/spec/services/bulk_upload/lettings/year2023/csv_parser_spec.rb b/spec/services/bulk_upload/lettings/year2023/csv_parser_spec.rb index 936cf1103..43e2f262a 100644 --- a/spec/services/bulk_upload/lettings/year2023/csv_parser_spec.rb +++ b/spec/services/bulk_upload/lettings/year2023/csv_parser_spec.rb @@ -150,6 +150,24 @@ RSpec.describe BulkUpload::Lettings::Year2023::CsvParser do end end + context "when parsing csv with carriage returns" do + before do + file.write("Question\r\n") + file.write("Additional info\r") + file.write("Values\r\n") + file.write("Can be empty?\r") + file.write("Type of letting the question applies to\r\n") + file.write("Duplicate check field?\r") + file.write(BulkUpload::LettingsLogToCsv.new(log:).default_2023_field_numbers_row) + file.write(BulkUpload::LettingsLogToCsv.new(log:).to_2023_csv_row) + file.rewind + end + + it "parses csv correctly" do + expect(service.row_parsers[0].field_13).to eql(log.tenancycode) + end + end + describe "#column_for_field", aggregate_failures: true do context "when with headers using default ordering" do before do diff --git a/spec/services/bulk_upload/sales/year2022/csv_parser_spec.rb b/spec/services/bulk_upload/sales/year2022/csv_parser_spec.rb index 2d65075b5..76504b974 100644 --- a/spec/services/bulk_upload/sales/year2022/csv_parser_spec.rb +++ b/spec/services/bulk_upload/sales/year2022/csv_parser_spec.rb @@ -94,4 +94,26 @@ RSpec.describe BulkUpload::Sales::Year2022::CsvParser do end end end + + context "when parsing csv with carriage returns" do + let(:file) { Tempfile.new } + let(:path) { file.path } + let(:log) { build(:sales_log, :completed) } + + before do + file.write("Question\r\n") + file.write("Additional info\r") + file.write("Values\r\n") + file.write("Can be empty?\r") + file.write("Type of letting the question applies to\r\n") + file.write("Duplicate check field?\r") + file.write(BulkUpload::SalesLogToCsv.new(log:).to_2022_csv_row) + file.rewind + end + + it "parses csv correctly" do + expect(service.column_for_field("field_1")).to eql("A") + expect(service.column_for_field("field_125")).to eql("DU") + end + end end diff --git a/spec/services/bulk_upload/sales/year2023/csv_parser_spec.rb b/spec/services/bulk_upload/sales/year2023/csv_parser_spec.rb index 54af60e3d..72c19e5d1 100644 --- a/spec/services/bulk_upload/sales/year2023/csv_parser_spec.rb +++ b/spec/services/bulk_upload/sales/year2023/csv_parser_spec.rb @@ -145,4 +145,22 @@ RSpec.describe BulkUpload::Sales::Year2023::CsvParser do end end end + + context "when parsing csv with carriage returns" do + before do + file.write("Question\r\n") + file.write("Additional info\r") + file.write("Values\r\n") + file.write("Can be empty?\r") + file.write("Type of letting the question applies to\r\n") + file.write("Duplicate check field?\r") + file.write(BulkUpload::SalesLogToCsv.new(log:).default_2023_field_numbers_row) + file.write(BulkUpload::SalesLogToCsv.new(log:).to_2023_csv_row) + file.rewind + end + + it "parses csv correctly" do + expect(service.row_parsers[0].field_19).to eql(log.uprn) + end + end end