CLDC-1898 Bulk upload headers (#1210)

* fix bulk upload age data types

* bulk upload handles both with and without headers

- headers are from the spreadsheet template
- otherwise assume cell A1 is start of the dataset

app/services/bulk_upload/lettings/csv_parser.rb

@@ -8,11 +8,11 @@ class BulkUpload::Lettings::CsvParser
   end
 
   def row_offset
-    5
+    with_headers? ? 5 : 0
   end
 
   def col_offset
-    1
+    with_headers? ? 1 : 0
   end
 
   def cols
@@ -21,7 +21,7 @@ class BulkUpload::Lettings::CsvParser
 
   def row_parsers
     @row_parsers ||= body_rows.map do |row|
-      stripped_row = row[1..]
+      stripped_row = row[col_offset..]
       headers = ("field_1".."field_134").to_a
       hash = Hash[headers.zip(stripped_row)]
 
@@ -39,6 +39,10 @@ class BulkUpload::Lettings::CsvParser
 
 private
 
+  def with_headers?
+    rows[0][0]&.match?(/\D+/)
+  end
+
   def row_sep
     "\n"
   end

app/services/bulk_upload/lettings/row_parser.rb

@@ -15,14 +15,14 @@ class BulkUpload::Lettings::RowParser
   attribute :field_9, :integer
   attribute :field_10, :string
   attribute :field_11, :integer
-  attribute :field_12, :string
+  attribute :field_12, :integer
-  attribute :field_13, :string
+  attribute :field_13, :integer
-  attribute :field_14, :string
+  attribute :field_14, :integer
-  attribute :field_15, :string
+  attribute :field_15, :integer
-  attribute :field_16, :string
+  attribute :field_16, :integer
-  attribute :field_17, :string
+  attribute :field_17, :integer
-  attribute :field_18, :string
+  attribute :field_18, :integer
-  attribute :field_19, :string
+  attribute :field_19, :integer
   attribute :field_20, :string
   attribute :field_21, :string
   attribute :field_22, :string

spec/services/bulk_upload/lettings/csv_parser_spec.rb

@@ -0,0 +1,38 @@
+require "rails_helper"
+
+RSpec.describe BulkUpload::Lettings::CsvParser do
+  subject(:service) { described_class.new(path:) }
+
+  let(:path) { file_fixture("2022_23_lettings_bulk_upload.csv") }
+
+  context "when parsing csv with headers" do
+    it "returns correct offsets" do
+      expect(service.row_offset).to eq(5)
+      expect(service.col_offset).to eq(1)
+    end
+
+    it "parses csv correctly" do
+      expect(service.row_parsers[0].field_12).to eq(55)
+    end
+  end
+
+  context "when parsing csv without headers" do
+    let(:file) { Tempfile.new }
+    let(:path) { file.path }
+    let(:log) { build(:lettings_log, :completed) }
+
+    before do
+      file.write(BulkUpload::LogToCsv.new(log:, col_offset: 0).to_csv_row)
+      file.rewind
+    end
+
+    it "returns correct offsets" do
+      expect(service.row_offset).to eq(0)
+      expect(service.col_offset).to eq(0)
+    end
+
+    it "parses csv correctly" do
+      expect(service.row_parsers[0].field_12).to eql(log.age1)
+    end
+  end
+end

spec/services/bulk_upload/lettings/log_creator_spec.rb

@@ -44,8 +44,7 @@ RSpec.describe BulkUpload::Lettings::LogCreator do
       end
 
       before do
-        5.times { file.write "\n" }
+        file.write(BulkUpload::LogToCsv.new(log:, col_offset: 0).to_csv_row)
-        file.write(BulkUpload::LogToCsv.new(log:).to_csv_row)
         file.rewind
       end
 

spec/services/bulk_upload/lettings/validator_spec.rb

@@ -70,8 +70,7 @@ RSpec.describe BulkUpload::Lettings::Validator do
       let(:path) { file.path }
 
       before do
-        file.write("\r\n" * 5)
+        file.write(BulkUpload::LogToCsv.new(log:, line_ending: "\r\n", col_offset: 0).to_csv_row)
-        file.write(BulkUpload::LogToCsv.new(log:, line_ending: "\r\n").to_csv_row)
         file.rewind
       end
 

spec/support/bulk_upload/log_to_csv.rb

@@ -1,14 +1,15 @@
 class BulkUpload::LogToCsv
-  attr_reader :log, :line_ending
+  attr_reader :log, :line_ending, :col_offset
 
-  def initialize(log:, line_ending: "\n")
+  def initialize(log:, line_ending: "\n", col_offset: 1)
     @log = log
     @line_ending = line_ending
+    @col_offset = col_offset
   end
 
   def to_csv_row
     [
-      nil, # 0
+      [nil] * col_offset, # 0
       log.renttype, # 1
       nil,
       nil,
@@ -154,7 +155,7 @@ class BulkUpload::LogToCsv
       log.joint,
       renewal,
       line_ending,
-    ].join(",")
+    ].flatten.join(",")
   end
 
   def renewal