CLDC-1896 Bulk upload line endings (#1208)

* add class to create logs from bulk upload

* create logs when processing bulk uploads

* remove bulk_upload_id from csv output

* create bulk upload logs only if all valid

- this will be changed later to allow for partial logs
- and only to create logs when a threshold has been met

* add method to blank invalid non setup fields

* bulk upload log creation blanks invalid fields

* fix incorrect logic for bulk upload renewal

* fix linting

* bulk upload log creation fail logs to sentry

* fix bulk upload line ending parsing

* extract bulk uploading csv parsing to class

* use csv parser in log creator

* change handle line endings mechanism

- we now strip all windows line endings for unix based line endings
- this normalises things making it simpler

app/models/bulk_upload.rb

@@ -2,7 +2,10 @@ class BulkUpload < ApplicationRecord
   enum log_type: { lettings: "lettings", sales: "sales" }
 
   belongs_to :user
+
   has_many :bulk_upload_errors
+  has_many :lettings_logs
+  has_many :sales_logs
 
   after_initialize :generate_identifier, unless: :identifier
 

app/models/log.rb

@@ -5,6 +5,8 @@ class Log < ApplicationRecord
   belongs_to :managing_organisation, class_name: "Organisation", optional: true
   belongs_to :created_by, class_name: "User", optional: true
   belongs_to :updated_by, class_name: "User", optional: true
+  belongs_to :bulk_upload, optional: true
+
   before_save :update_status!
 
   STATUS = { "not_started" => 0, "in_progress" => 1, "completed" => 2 }.freeze
@@ -50,6 +52,16 @@ class Log < ApplicationRecord
     form.end_date > Time.zone.today
   end
 
+  def blank_invalid_non_setup_fields!
+    setup_ids = form.setup_sections.flat_map(&:subsections).flat_map(&:questions).map(&:id)
+
+    errors.each do |error|
+      next if setup_ids.include?(error.attribute.to_s)
+
+      public_send("#{error.attribute}=", nil)
+    end
+  end
+
 private
 
   def update_status!

app/services/bulk_upload/lettings/csv_parser.rb

@@ -0,0 +1,54 @@
+require "csv"
+
+class BulkUpload::Lettings::CsvParser
+  attr_reader :path
+
+  def initialize(path:)
+    @path = path
+  end
+
+  def row_offset
+    5
+  end
+
+  def col_offset
+    1
+  end
+
+  def cols
+    @cols ||= ("A".."EE").to_a
+  end
+
+  def row_parsers
+    @row_parsers ||= body_rows.map do |row|
+      stripped_row = row[1..]
+      headers = ("field_1".."field_134").to_a
+      hash = Hash[headers.zip(stripped_row)]
+
+      BulkUpload::Lettings::RowParser.new(hash)
+    end
+  end
+
+  def body_rows
+    rows[row_offset..]
+  end
+
+  def rows
+    @rows ||= CSV.parse(normalised_string, row_sep:)
+  end
+
+private
+
+  def row_sep
+    "\n"
+  end
+
+  def normalised_string
+    return @normalised_string if @normalised_string
+
+    @normalised_string = File.read(path)
+    @normalised_string.gsub!("\r\n", "\n")
+
+    @normalised_string
+  end
+end

app/services/bulk_upload/lettings/log_creator.rb

@@ -0,0 +1,57 @@
+class BulkUpload::Lettings::LogCreator
+  attr_reader :bulk_upload, :path
+
+  def initialize(bulk_upload:, path:)
+    @bulk_upload = bulk_upload
+    @path = path
+  end
+
+  def call
+    row_parsers.each do |row_parser|
+      row_parser.valid?
+
+      row_parser.log.blank_invalid_non_setup_fields!
+      row_parser.log.bulk_upload = bulk_upload
+
+      begin
+        row_parser.log.save!
+      rescue StandardError => e
+        Sentry.capture_exception(e)
+      end
+    end
+  end
+
+private
+
+  def csv_parser
+    @csv_parser ||= BulkUpload::Lettings::CsvParser.new(path:)
+  end
+
+  def row_offset
+    csv_parser.row_offset
+  end
+
+  def col_offset
+    csv_parser.col_offset
+  end
+
+  def row_parsers
+    return @row_parsers if @row_parsers
+
+    @row_parsers = csv_parser.row_parsers
+
+    @row_parsers.each do |row_parser|
+      row_parser.bulk_upload = bulk_upload
+    end
+
+    @row_parsers
+  end
+
+  def body_rows
+    csv_parser.body_rows
+  end
+
+  def rows
+    csv_parser.rows
+  end
+end

app/services/bulk_upload/lettings/row_parser.rb

@@ -663,7 +663,7 @@ private
     when 2
       0
     when nil
-      field_116 == 14 ? 1 : 0
+      rsnvac == 14 ? 1 : 0
     end
   end
 

app/services/bulk_upload/lettings/validator.rb

@@ -169,18 +169,26 @@ class BulkUpload::Lettings::Validator
     end
   end
 
+  def create_logs?
+    row_parsers.all?(&:valid?)
+  end
+
   def self.question_for_field(field)
     QUESTIONS[field]
   end
 
 private
 
+  def csv_parser
+    @csv_parser ||= BulkUpload::Lettings::CsvParser.new(path:)
+  end
+
   def row_offset
-    5
+    csv_parser.row_offset
   end
 
   def col_offset
-    1
+    csv_parser.col_offset
   end
 
   def field_number_for_attribute(attribute)
@@ -188,46 +196,27 @@ private
   end
 
   def cols
-    @cols ||= ("A".."EE").to_a
+    csv_parser.cols
   end
 
   def row_parsers
-    @row_parsers ||= body_rows.map do |row|
+    return @row_parsers if @row_parsers
-      stripped_row = row[1..]
-      headers = ("field_1".."field_134").to_a
-      hash = Hash[headers.zip(stripped_row)]
-      hash[:bulk_upload] = bulk_upload
 
-      BulkUpload::Lettings::RowParser.new(hash)
+    @row_parsers = csv_parser.row_parsers
-    end
-  end
-
-  # determine the row seperator from CSV
-  # Windows will use \r\n
-  def row_sep
-    contents = ""
 
-    File.open(path, "r") do |f|
+    @row_parsers.each do |row_parser|
-      f.seek(9900)
+      row_parser.bulk_upload = bulk_upload
-      contents = f.read
     end
 
-    rn_count = contents.scan("\r\n").count
+    @row_parsers
-    n_count = contents.scan(/[^\r]\n/).count
-
-    if rn_count > n_count
-      "\r\n"
-    else
-      "\n"
-    end
   end
 
   def rows
-    @rows ||= CSV.read(path, row_sep:)
+    csv_parser.rows
   end
 
   def body_rows
-    rows[row_offset..]
+    csv_parser.body_rows
   end
 
   def validate_file_not_empty

app/services/bulk_upload/processor.rb

@@ -8,12 +8,31 @@ class BulkUpload::Processor
   def call
     download
     validator.call
+    create_logs if validator.create_logs?
   ensure
     downloader.delete_local_file!
   end
 
 private
 
+  def create_logs
+    log_creator_class.new(
+      bulk_upload:,
+      path: downloader.path,
+    ).call
+  end
+
+  def log_creator_class
+    case bulk_upload.log_type
+    when "lettings"
+      BulkUpload::Lettings::LogCreator
+    when "sales"
+      BulkUpload::Sales::LogCreator
+    else
+      raise "Log creator not found for #{bulk_upload.log_type}"
+    end
+  end
+
   def downloader
     @downloader ||= BulkUpload::Downloader.new(bulk_upload:)
   end

app/services/csv/lettings_log_csv_service.rb

@@ -1,6 +1,6 @@
 module Csv
   class LettingsLogCsvService
-    CSV_FIELDS_TO_OMIT = %w[hhmemb net_income_value_check first_time_property_let_as_social_housing renttype needstype postcode_known is_la_inferred totchild totelder totadult net_income_known is_carehome previous_la_known is_previous_la_inferred age1_known age2_known age3_known age4_known age5_known age6_known age7_known age8_known letting_allocation_unknown details_known_2 details_known_3 details_known_4 details_known_5 details_known_6 details_known_7 details_known_8 rent_type_detail wrent wscharge wpschrge wsupchrg wtcharge wtshortfall rent_value_check old_form_id old_id retirement_value_check tshortfall_known pregnancy_value_check hhtype new_old vacdays la prevloc unresolved updated_by_id].freeze
+    CSV_FIELDS_TO_OMIT = %w[hhmemb net_income_value_check first_time_property_let_as_social_housing renttype needstype postcode_known is_la_inferred totchild totelder totadult net_income_known is_carehome previous_la_known is_previous_la_inferred age1_known age2_known age3_known age4_known age5_known age6_known age7_known age8_known letting_allocation_unknown details_known_2 details_known_3 details_known_4 details_known_5 details_known_6 details_known_7 details_known_8 rent_type_detail wrent wscharge wpschrge wsupchrg wtcharge wtshortfall rent_value_check old_form_id old_id retirement_value_check tshortfall_known pregnancy_value_check hhtype new_old vacdays la prevloc unresolved updated_by_id bulk_upload_id].freeze
 
     def initialize(user)
       @user = user
@@ -40,7 +40,7 @@ module Csv
 
     def set_csv_attributes
       metadata_fields = %w[id status created_at updated_at created_by_name is_dpo owning_organisation_name managing_organisation_name collection_start_year]
-      metadata_id_fields = %w[managing_organisation_id owning_organisation_id created_by_id]
+      metadata_id_fields = %w[managing_organisation_id owning_organisation_id created_by_id bulk_upload_id]
       scheme_and_location_ids = %w[scheme_id location_id]
       scheme_attributes = %w[scheme_code scheme_service_name scheme_sensitive scheme_type scheme_registered_under_care_act scheme_owning_organisation_name scheme_primary_client_group scheme_has_other_client_group scheme_secondary_client_group scheme_support_type scheme_intended_stay scheme_created_at]
       location_attributes = %w[location_code location_postcode location_name location_units location_type_of_unit location_mobility_type location_admin_district location_startdate]

db/migrate/20230113154518_add_bulk_upload_to_logs.rb

@@ -0,0 +1,6 @@
+class AddBulkUploadToLogs < ActiveRecord::Migration[7.0]
+  def change
+    add_reference :lettings_logs, :bulk_upload
+    add_reference :sales_logs, :bulk_upload
+  end
+end

db/schema.rb

@@ -265,6 +265,8 @@ ActiveRecord::Schema[7.0].define(version: 2023_01_16_151942) do
     t.integer "housingneeds_other"
     t.boolean "unresolved"
     t.bigint "updated_by_id"
+    t.bigint "bulk_upload_id"
+    t.index ["bulk_upload_id"], name: "index_lettings_logs_on_bulk_upload_id"
     t.index ["created_by_id"], name: "index_lettings_logs_on_created_by_id"
     t.index ["location_id"], name: "index_lettings_logs_on_location_id"
     t.index ["managing_organisation_id"], name: "index_lettings_logs_on_managing_organisation_id"
@@ -499,6 +501,8 @@ ActiveRecord::Schema[7.0].define(version: 2023_01_16_151942) do
     t.string "postcode_full"
     t.boolean "is_la_inferred"
     t.integer "hodate_check"
+    t.bigint "bulk_upload_id"
+    t.index ["bulk_upload_id"], name: "index_sales_logs_on_bulk_upload_id"
     t.index ["created_by_id"], name: "index_sales_logs_on_created_by_id"
     t.index ["managing_organisation_id"], name: "index_sales_logs_on_managing_organisation_id"
     t.index ["owning_organisation_id"], name: "index_sales_logs_on_owning_organisation_id"

spec/fixtures/files/2022_23_lettings_bulk_upload.csv

@@ -69,4 +69,6 @@ if 87 = 1, then a value must be entered",No,,,Yes,,,,No,,,"If the property is be
 or 106 = 15 - 17",No,"Only if 1 = 2, 4, 6, 8, 10 or 12",,,,No,,No,"Yes, if 45 = 2, 3 or 6",,"Yes, if 50 = 1","Only if 1 = 1, 3, 5, 7, 9 or 11",No,Yes,,,,,,,,,,Only if 1 = 1 - 4 or 9 - 12.,Only if 1 = 1 - 8.,Only if 130 is not 3,No,No,Yes,
 Bulk upload format and duplicate check,All lettings,Question removed from 22/23 onwards,,Supported housing only,,Question Removed from 2020/21,,,,,,Duplicate check field,,,,,,,,Duplicate check field,,,,,,,,,,,,,,,Duplicate check field,,,,,,,,,,,,,,,,,,,Question removed from 22/23 onwards,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,Duplicate check field,,,,,,,,,,,Question removed from 22/23 onwards,Duplicate check fields,,,,Duplicate check field,General Needs lettings only,,,All lettings,General Needs lettings only,All lettings,General Needs lettings only,,,Question removed from 2020/21,Duplicate check field, “Username does not exist”. ,,,Question removed from 21/22 onwards,,Supported Housing lettings only.,,,,,,,,,,,,Affordable Rent Lettings only,Intermediate Rent Lettings only,,All lettings,All lettings,,
 Bulk upload field number,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,
+,1,,,,,,123,1,4,,2,55,54,,,,,,,F,,,,,,,,,,,,,,,1,,,,,,,,17,13,2,,2,3,4,,2,7,,,,,,,,,3,,,,2,1,2,1,3,,,,,,,,,16,4,1000,100,100,100,1300,,,,,,,,,,,,13,1,23,,,4,1,1,2,,,,EC1N,2TD,,3,,3,,,,,2,,,,,,,,,,,,,,1,2,2
 ,1,,,,,,123,1,2,,6,55,54,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+,1,,,,,,123,1,2,,,55,54,,,,,,,"A",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,13,1,23,,,,,,,,,,,,,123,,123,,,,,,,,,,,,,,,,,,,,,2,

spec/models/lettings_log_spec.rb

@@ -2536,4 +2536,24 @@ RSpec.describe LettingsLog do
       end
     end
   end
+
+  describe "#blank_invalid_non_setup_fields!" do
+    context "when a setup field is invalid" do
+      subject(:model) { described_class.new(needstype: 404) }
+
+      it "does not blank it" do
+        model.valid?
+        expect { model.blank_invalid_non_setup_fields! }.not_to change(model, :needstype)
+      end
+    end
+
+    context "when a non setup field is invalid" do
+      subject(:model) { described_class.new(beds: 404) }
+
+      it "blanks it" do
+        model.valid?
+        expect { model.blank_invalid_non_setup_fields! }.to change(model, :beds)
+      end
+    end
+  end
 end

spec/rails_helper.rb

@@ -32,8 +32,8 @@ Capybara.javascript_driver = :headless
 # of increasing the boot-up time by auto-requiring all files in the support
 # directory. Alternatively, in the individual `*_spec.rb` files, manually
 # require only the support files necessary.
-#
+
-# Dir[Rails.root.join('spec', 'support', '**', '*.rb')].sort.each { |f| require f }
+Dir[Rails.root.join("spec/support/**/*.rb")].sort.each { |f| require f }
 
 # Checks for pending migrations and applies them before tests are run.
 # If you are not using ActiveRecord, you can remove these lines.

spec/services/bulk_upload/lettings/log_creator_spec.rb

@@ -0,0 +1,68 @@
+require "rails_helper"
+
+RSpec.describe BulkUpload::Lettings::LogCreator do
+  subject(:service) { described_class.new(bulk_upload:, path:) }
+
+  let(:owning_org) { create(:organisation, old_visible_id: 123) }
+  let(:user) { create(:user, organisation: owning_org) }
+
+  let(:bulk_upload) { create(:bulk_upload, :lettings, user:) }
+  let(:path) { file_fixture("2022_23_lettings_bulk_upload.csv") }
+
+  describe "#call" do
+    context "when a valid csv with new log" do
+      it "creates a new log" do
+        expect { service.call }.to change(LettingsLog, :count)
+      end
+
+      it "associates log with bulk upload" do
+        service.call
+
+        log = LettingsLog.last
+        expect(log.bulk_upload).to eql(bulk_upload)
+        expect(bulk_upload.lettings_logs).to include(log)
+      end
+    end
+
+    context "when a valid csv with row with one invalid non setup field" do
+      let(:file) { Tempfile.new }
+      let(:path) { file.path }
+      let(:log) do
+        build(
+          :lettings_log,
+          :completed,
+          renttype: 3,
+          age1: 5,
+          owning_organisation: owning_org,
+          managing_organisation: owning_org,
+          national: 18,
+          waityear: 9,
+          joint: 2,
+          tenancy: 9,
+          ppcodenk: 0,
+        )
+      end
+
+      before do
+        5.times { file.write "\n" }
+        file.write(BulkUpload::LogToCsv.new(log:).to_csv_row)
+        file.rewind
+      end
+
+      it "creates the log" do
+        expect { service.call }.to change(LettingsLog, :count).by(1)
+      end
+
+      it "blanks invalid field" do
+        service.call
+
+        record = LettingsLog.last
+        expect(record.age1).to be_blank
+      end
+    end
+
+    context "when valid csv with existing log" do
+      xit "what should happen?"
+    end
+  end
+end

spec/services/bulk_upload/lettings/row_parser_spec.rb

@@ -377,8 +377,8 @@ RSpec.describe BulkUpload::Lettings::RowParser do
         end
       end
 
-      context "when field_134 is null but rsnvac/field_116 is 14" do
+      context "when field_134 is null but rsnvac/field_106 is 14" do
-        let(:attributes) { { bulk_upload:, field_134: "", field_116: "14" } }
+        let(:attributes) { { bulk_upload:, field_134: "", field_106: "14" } }
 
         it "sets renewal to 1" do
           expect(parser.log.renewal).to eq(1)

spec/services/bulk_upload/lettings/validator_spec.rb

@@ -3,7 +3,9 @@ require "rails_helper"
 RSpec.describe BulkUpload::Lettings::Validator do
   subject(:validator) { described_class.new(bulk_upload:, path:) }
 
-  let(:bulk_upload) { create(:bulk_upload) }
+  let(:organisation) { create(:organisation, old_visible_id: "3") }
+  let(:user) { create(:user, organisation:) }
+  let(:bulk_upload) { create(:bulk_upload, user:) }
   let(:path) { file.path }
   let(:file) { Tempfile.new }
 
@@ -29,18 +31,79 @@ RSpec.describe BulkUpload::Lettings::Validator do
     context "when incorrect headers"
   end
 
-  context "when a valid csv" do
+  describe "#call" do
-    let(:path) { file_fixture("2022_23_lettings_bulk_upload.csv") }
+    context "when a valid csv" do
+      let(:path) { file_fixture("2022_23_lettings_bulk_upload.csv") }
 
-    it "creates validation errors" do
+      it "creates validation errors" do
-      expect { validator.call }.to change(BulkUploadError, :count)
+        expect { validator.call }.to change(BulkUploadError, :count)
+      end
+
+      it "create validation error with correct values" do
+        validator.call
+
+        error = BulkUploadError.first
+        expect(error.row).to eq("7")
+      end
+    end
+
+    context "with unix line endings" do
+      let(:fixture_path) { file_fixture("2022_23_lettings_bulk_upload.csv") }
+      let(:file) { Tempfile.new }
+      let(:path) { file.path }
+
+      before do
+        string = File.read(fixture_path)
+        string.gsub!("\r\n", "\n")
+        file.write(string)
+        file.rewind
+      end
+
+      it "creates validation errors" do
+        expect { validator.call }.to change(BulkUploadError, :count)
+      end
+    end
+
+    context "without headers" do
+      let(:log) { build(:lettings_log, :completed) }
+      let(:file) { Tempfile.new }
+      let(:path) { file.path }
+
+      before do
+        file.write("\r\n" * 5)
+        file.write(BulkUpload::LogToCsv.new(log:, line_ending: "\r\n").to_csv_row)
+        file.rewind
+      end
+
+      it "creates validation errors" do
+        expect { validator.call }.to change(BulkUploadError, :count)
+      end
     end
+  end
 
-    it "create validation error with correct values" do
+  describe "#should_create_logs?" do
-      validator.call
+    context "when all logs are valid" do
+      let(:target_path) { file_fixture("2022_23_lettings_bulk_upload.csv") }
 
-      error = BulkUploadError.first
+      before do
-      expect(error.row).to eq("6")
+        target_array = File.open(target_path).readlines
+        target_array[0..71].each do |line|
+          file.write line
+        end
+        file.rewind
+      end
+
+      it "returns truthy" do
+        expect(validator).to be_create_logs
+      end
+    end
+
+    context "when there is an invalid log" do
+      let(:path) { file_fixture("2022_23_lettings_bulk_upload.csv") }
+
+      it "returns falsey" do
+        expect(validator).not_to be_create_logs
+      end
     end
   end
 end

spec/services/bulk_upload/processor_spec.rb

@@ -5,8 +5,8 @@ RSpec.describe BulkUpload::Processor do
 
   let(:bulk_upload) { create(:bulk_upload, :lettings) }
 
-  context "when processing a bulk upload with errors" do
+  describe "#call" do
-    describe "#call" do
+    context "when processing a bulk upload with errors" do
       let(:mock_downloader) do
         instance_double(
           BulkUpload::Downloader,
@@ -30,5 +30,44 @@ RSpec.describe BulkUpload::Processor do
         expect(mock_downloader).to have_received(:delete_local_file!)
       end
     end
+
+    context "when processing a bulk with perfect data" do
+      let(:path) { file_fixture("2022_23_lettings_bulk_upload.csv") }
+
+      let(:mock_downloader) do
+        instance_double(
+          BulkUpload::Downloader,
+          call: nil,
+          path:,
+          delete_local_file!: nil,
+        )
+      end
+
+      let(:mock_validator) do
+        instance_double(
+          BulkUpload::Lettings::Validator,
+          call: nil,
+          create_logs?: true,
+        )
+      end
+
+      let(:mock_creator) do
+        instance_double(
+          BulkUpload::Lettings::LogCreator,
+          call: nil,
+          path:,
+        )
+      end
+
+      it "creates logs" do
+        allow(BulkUpload::Downloader).to receive(:new).with(bulk_upload:).and_return(mock_downloader)
+        allow(BulkUpload::Lettings::Validator).to receive(:new).and_return(mock_validator)
+        allow(BulkUpload::Lettings::LogCreator).to receive(:new).with(bulk_upload:, path:).and_return(mock_creator)
+
+        processor.call
+
+        expect(mock_creator).to have_received(:call)
+      end
+    end
   end
 end

spec/support/bulk_upload/log_to_csv.rb

@@ -0,0 +1,255 @@
+class BulkUpload::LogToCsv
+  attr_reader :log, :line_ending
+
+  def initialize(log:, line_ending: "\n")
+    @log = log
+    @line_ending = line_ending
+  end
+
+  def to_csv_row
+    [
+      nil, # 0
+      log.renttype, # 1
+      nil,
+      nil,
+      log.scheme&.old_visible_id,
+      log.location&.old_visible_id,
+      nil,
+      log.tenancycode,
+      log.startertenancy,
+      log.tenancy,
+      log.tenancyother, # 10
+      log.tenancylength,
+      log.age1,
+      log.age2,
+      log.age3,
+      log.age4,
+      log.age5,
+      log.age6,
+      log.age7,
+      log.age8,
+
+      log.sex1, # 20
+      log.sex2,
+      log.sex3,
+      log.sex4,
+      log.sex5,
+      log.sex6,
+      log.sex7,
+      log.sex8,
+
+      log.relat2,
+      log.relat3,
+      log.relat4, # 30
+      log.relat5,
+      log.relat6,
+      log.relat7,
+      log.relat8,
+
+      log.ecstat1,
+      log.ecstat2,
+      log.ecstat3,
+      log.ecstat4,
+      log.ecstat5,
+      log.ecstat6, # 40
+      log.ecstat7,
+      log.ecstat8,
+
+      log.ethnic,
+      log.national,
+      log.armedforces,
+      log.reservist,
+      log.preg_occ,
+      log.hb,
+      log.benefits,
+      log.earnings, # 50
+      net_income_known,
+      nil,
+      log.reasonother,
+      nil,
+      nil,
+      nil,
+      nil,
+      nil,
+      nil,
+      nil, # 60
+      log.prevten,
+      log.prevloc,
+      ((log.ppostcode_full || "").split(" ") || [""]).first,
+      ((log.ppostcode_full || "").split(" ") || [""]).last,
+      previous_postcode_known,
+      log.layear,
+      log.waityear,
+      homeless,
+      log.reasonpref,
+      log.rp_homeless, # 70
+      log.rp_insan_unsat,
+      log.rp_medwel,
+      log.rp_hardship,
+      log.rp_dontknow,
+      cbl,
+      chr,
+      cap,
+      log.referral,
+      log.period,
+
+      log.brent, # 80
+      log.scharge,
+      log.pscharge,
+      log.supcharg,
+      log.tcharge,
+      log.chcharge,
+      log.household_charge,
+      log.hbrentshortfall,
+      log.tshortfall,
+      log.voiddate&.day,
+
+      log.voiddate&.month, # 90
+      log.voiddate&.strftime("%y"),
+      log.mrcdate&.day,
+      log.mrcdate&.month,
+      log.mrcdate&.strftime("%y"),
+      nil,
+      log.startdate&.day,
+      log.startdate&.month,
+      log.startdate&.strftime("%y"),
+      log.offered,
+
+      log.propcode, # 100
+      log.beds,
+      log.unittype_gn,
+      log.builtype,
+      log.wchair,
+      log.unitletas,
+      log.rsnvac,
+      log.la,
+      ((log.postcode_full || "").split(" ") || [""]).first,
+      ((log.postcode_full || "").split(" ") || [""]).last,
+
+      nil, # 110
+      log.owning_organisation&.old_visible_id,
+      nil,
+      log.managing_organisation&.old_visible_id,
+      leftreg,
+      nil,
+      log.incfreq,
+      log.sheltered,
+      log.illness,
+      log.illness_type_1,
+
+      log.illness_type_2, # 120
+      log.illness_type_3,
+      log.illness_type_4,
+      log.illness_type_5,
+      log.illness_type_6,
+      log.illness_type_7,
+      log.illness_type_8,
+      log.illness_type_9,
+      log.illness_type_10,
+      london_affordable_rent,
+
+      intermediate_rent_type, # 130
+      log.irproduct_other,
+      log.declaration,
+      log.joint,
+      renewal,
+      line_ending,
+    ].join(",")
+  end
+
+  def renewal
+    case log.renewal
+    when 1
+      1
+    when 0
+      2
+    end
+  end
+
+  def london_affordable_rent
+    case log.renttype
+    when Imports::LettingsLogsImportService::RENT_TYPE[:london_affordable_rent]
+      1
+    end
+  end
+
+  def intermediate_rent_type
+    case log.renttype
+    when Imports::LettingsLogsImportService::RENT_TYPE[:rent_to_buy]
+      1
+    when Imports::LettingsLogsImportService::RENT_TYPE[:london_living_rent]
+      2
+    when Imports::LettingsLogsImportService::RENT_TYPE[:other_intermediate_rent_product]
+      3
+    end
+  end
+
+  def leftreg
+    case log.leftreg
+    when 3
+      3
+    when 1
+      4
+    when 2
+      5
+    when 0
+      6
+    end
+  end
+
+  def net_income_known
+    case log.net_income_known
+    when 0
+      1
+    when 1
+      2
+    when 2
+      4
+    end
+  end
+
+  def previous_postcode_known
+    case log.ppcodenk
+    when 1
+      1
+    when 0
+      2
+    end
+  end
+
+  def homeless
+    case log.homeless
+    when 1
+      1
+    when 11
+      12
+    end
+  end
+
+  def cbl
+    case log.cbl
+    when 0
+      2
+    when 1
+      1
+    end
+  end
+
+  def chr
+    case log.chr
+    when 0
+      2
+    when 1
+      1
+    end
+  end
+
+  def cap
+    case log.cap
+    when 0
+      2
+    when 1
+      1
+    end
+  end
+end