CLDC-2031 Sales bulk upload (#1574)

* Add sales csv parser

* Create log creator and update row parser for sales

* Add validations

* Add status_cache to sales logs

* Parse completed log

* Add basic validations to the row parser

* Fix details known and proplen mapping

* Add setup section errors

* Update sales log validator

* tests

* Add sales resume and summary pages

* send the correct emails

app/controllers/bulk_upload_sales_results_controller.rb

@@ -6,4 +6,34 @@ class BulkUploadSalesResultsController < ApplicationController
   def show
     @bulk_upload = current_user.bulk_uploads.sales.find(params[:id])
   end
+
+  def resume
+    @bulk_upload = current_user.bulk_uploads.sales.find(params[:id])
+
+    if @bulk_upload.sales_logs.in_progress.count.positive?
+      set_bulk_upload_logs_filters
+
+      redirect_to(sales_logs_path(bulk_upload_id: [@bulk_upload.id]))
+    else
+      reset_logs_filters
+    end
+  end
+
+  def summary
+    @bulk_upload = current_user.bulk_uploads.sales.find(params[:id])
+  end
+
+  def reset_logs_filters
+    session["logs_filters"] = {}.to_json
+  end
+
+  def set_bulk_upload_logs_filters
+    hash = {
+      years: [""],
+      status: ["", "in_progress"],
+      user: "all",
+    }
+
+    session["logs_filters"] = hash.to_json
+  end
 end

app/controllers/bulk_upload_sales_resume_controller.rb

@@ -0,0 +1,42 @@
+class BulkUploadSalesResumeController < ApplicationController
+  before_action :authenticate_user!
+
+  def start
+    @bulk_upload = current_user.bulk_uploads.find(params[:id])
+
+    redirect_to page_bulk_upload_sales_resume_path(@bulk_upload, page: "fix-choice")
+  end
+
+  def show
+    @bulk_upload = current_user.bulk_uploads.find(params[:id])
+
+    render form.view_path
+  end
+
+  def update
+    @bulk_upload = current_user.bulk_uploads.find(params[:id])
+
+    if form.valid? && form.save!
+      redirect_to form.next_path
+    else
+      render form.view_path
+    end
+  end
+
+private
+
+  def form
+    @form ||= case params[:page]
+              when "fix-choice"
+                Forms::BulkUploadSalesResume::FixChoice.new(form_params.merge(bulk_upload: @bulk_upload))
+              when "confirm"
+                Forms::BulkUploadSalesResume::Confirm.new(form_params.merge(bulk_upload: @bulk_upload))
+              else
+                raise "invalid form"
+              end
+  end
+
+  def form_params
+    params.fetch(:form, {}).permit(:choice)
+  end
+end

app/mailers/bulk_upload_mailer.rb

@@ -10,7 +10,7 @@ class BulkUploadMailer < NotifyMailer
   def send_how_fix_upload_mail(bulk_upload:)
     title = "We found #{pluralize(bulk_upload.bulk_upload_errors.count, 'error')} in your bulk upload"
     description = "There was a problem with your #{bulk_upload.year_combo} #{bulk_upload.log_type} data. Check the error report below to fix these errors."
-    cta_link = start_bulk_upload_lettings_resume_url(bulk_upload)
+    cta_link = bulk_upload.sales? ? start_bulk_upload_sales_resume_url(bulk_upload) : start_bulk_upload_lettings_resume_url(bulk_upload)
 
     send_email(
       bulk_upload.user.email,
@@ -53,9 +53,9 @@ class BulkUploadMailer < NotifyMailer
 
   def send_correct_and_upload_again_mail(bulk_upload:)
     summary_report_link = if BulkUploadErrorSummaryTableComponent.new(bulk_upload:).errors?
-                            summary_bulk_upload_lettings_result_url(bulk_upload)
+                            bulk_upload.sales? ? summary_bulk_upload_sales_result_url(bulk_upload) : summary_bulk_upload_lettings_result_url(bulk_upload)
                           else
-                            bulk_upload_lettings_result_url(bulk_upload)
+                            bulk_upload.sales? ? bulk_upload_sales_result_url(bulk_upload) : bulk_upload_lettings_result_url(bulk_upload)
                           end
 
     send_email(
@@ -73,9 +73,9 @@ class BulkUploadMailer < NotifyMailer
 
   def send_bulk_upload_failed_file_setup_error_mail(bulk_upload:)
     bulk_upload_link = if BulkUploadErrorSummaryTableComponent.new(bulk_upload:).errors?
-                         summary_bulk_upload_lettings_result_url(bulk_upload)
+                         bulk_upload.sales? ? summary_bulk_upload_sales_result_url(bulk_upload) : summary_bulk_upload_lettings_result_url(bulk_upload)
                        else
-                         bulk_upload_lettings_result_url(bulk_upload)
+                         bulk_upload.sales? ? bulk_upload_sales_result_url(bulk_upload) : bulk_upload_lettings_result_url(bulk_upload)
                        end
 
     send_email(

app/models/forms/bulk_upload_sales_resume/confirm.rb

@@ -0,0 +1,30 @@
+module Forms
+  module BulkUploadSalesResume
+    class Confirm
+      include ActiveModel::Model
+      include ActiveModel::Attributes
+      include Rails.application.routes.url_helpers
+
+      attribute :bulk_upload
+
+      def view_path
+        "bulk_upload_sales_resume/confirm"
+      end
+
+      def back_path
+        page_bulk_upload_sales_resume_path(bulk_upload, page: "fix-choice")
+      end
+
+      def next_path
+        resume_bulk_upload_sales_result_path(bulk_upload)
+      end
+
+      def save!
+        processor = BulkUpload::Processor.new(bulk_upload:)
+        processor.approve
+
+        true
+      end
+    end
+  end
+end

app/models/forms/bulk_upload_sales_resume/fix_choice.rb

@@ -0,0 +1,53 @@
+module Forms
+  module BulkUploadSalesResume
+    class FixChoice
+      include ActiveModel::Model
+      include ActiveModel::Attributes
+      include Rails.application.routes.url_helpers
+
+      attribute :bulk_upload
+      attribute :choice, :string
+
+      validates :choice, presence: true,
+                         inclusion: { in: %w[create-fix-inline upload-again] }
+
+      def options
+        [
+          OpenStruct.new(id: "create-fix-inline", name: "Upload these logs and fix errors on CORE site"),
+          OpenStruct.new(id: "upload-again", name: "Fix errors in the CSV and re-upload"),
+        ]
+      end
+
+      def view_path
+        "bulk_upload_sales_resume/fix_choice"
+      end
+
+      def next_path
+        case choice
+        when "create-fix-inline"
+          page_bulk_upload_sales_resume_path(bulk_upload, page: "confirm")
+        when "upload-again"
+          if BulkUploadErrorSummaryTableComponent.new(bulk_upload:).errors?
+            summary_bulk_upload_sales_result_path(bulk_upload)
+          else
+            bulk_upload_sales_result_path(bulk_upload)
+          end
+        else
+          raise "invalid choice"
+        end
+      end
+
+      def recommendation
+        if BulkUploadErrorSummaryTableComponent.new(bulk_upload:).errors?
+          "For this many errors we recommend to fix errors in the CSV and re-upload as you may be able to edit many fields at once in a CSV."
+        else
+          "For this many errors we recommend to upload logs and fix errors on site as you can easily see the questions and select the appropriate answer."
+        end
+      end
+
+      def save!
+        true
+      end
+    end
+  end
+end

app/services/bulk_upload/sales/log_creator.rb

@@ -0,0 +1,70 @@
+class BulkUpload::Sales::LogCreator
+  attr_reader :bulk_upload, :path
+
+  def initialize(bulk_upload:, path:)
+    @bulk_upload = bulk_upload
+    @path = path
+  end
+
+  def call
+    row_parsers.each do |row_parser|
+      row_parser.valid?
+
+      next if row_parser.blank_row?
+
+      row_parser.log.blank_invalid_non_setup_fields!
+      row_parser.log.bulk_upload = bulk_upload
+      row_parser.log.skip_update_status = true
+      row_parser.log.status = "pending"
+
+      row_parser.log.status_cache = row_parser.log.calculate_status
+
+      begin
+        row_parser.log.save!
+      rescue StandardError => e
+        Sentry.capture_exception(e)
+      end
+    end
+  end
+
+private
+
+  def csv_parser
+    @csv_parser ||= case bulk_upload.year
+                    when 2022
+                      BulkUpload::Sales::Year2022::CsvParser.new(path:)
+                    when 2023
+                      BulkUpload::Sales::Year2023::CsvParser.new(path:)
+                    else
+                      raise "csv parser not found"
+                    end
+  end
+
+  def row_offset
+    csv_parser.row_offset
+  end
+
+  def col_offset
+    csv_parser.col_offset
+  end
+
+  def row_parsers
+    return @row_parsers if @row_parsers
+
+    @row_parsers = csv_parser.row_parsers
+
+    @row_parsers.each do |row_parser|
+      row_parser.bulk_upload = bulk_upload
+    end
+
+    @row_parsers
+  end
+
+  def body_rows
+    csv_parser.body_rows
+  end
+
+  def rows
+    csv_parser.rows
+  end
+end

app/services/bulk_upload/sales/validator.rb

@@ -4,6 +4,7 @@ class BulkUpload::Sales::Validator
   attr_reader :bulk_upload, :path
 
   validate :validate_file_not_empty
+  validate :validate_min_columns
   validate :validate_max_columns
 
   def initialize(bulk_upload:, path:)
@@ -18,56 +19,83 @@ class BulkUpload::Sales::Validator
       row = index + row_offset + 1
 
       row_parser.errors.each do |error|
+        col = csv_parser.column_for_field(error.attribute.to_s)
+
         bulk_upload.bulk_upload_errors.create!(
           field: error.attribute,
-          error: error.type,
+          error: error.message,
           purchaser_code: row_parser.field_1,
           row:,
-          cell: "#{cols[field_number_for_attribute(error.attribute) + col_offset - 1]}#{row}",
+          cell: "#{col}#{row}",
+          col:,
+          category: error.options[:category],
         )
       end
     end
   end
 
-private
+  def create_logs?
+    return false if any_setup_errors?
+    return false if row_parsers.any?(&:block_log_creation?)
 
-  def field_number_for_attribute(attribute)
-    attribute.to_s.split("_").last.to_i
+    row_parsers.all? { |row_parser| row_parser.log.valid? }
   end
 
-  def rows
-    @rows ||= CSV.read(path, row_sep:)
+  def any_setup_errors?
+    bulk_upload
+      .bulk_upload_errors
+      .where(category: "setup")
+      .count
+      .positive?
   end
 
-  def body_rows
-    rows[row_offset..]
+private
+
+  def csv_parser
+    @csv_parser ||= case bulk_upload.year
+                    when 2022
+                      BulkUpload::Sales::Year2022::CsvParser.new(path:)
+                    when 2023
+                      BulkUpload::Sales::Year2023::CsvParser.new(path:)
+                    else
+                      raise "csv parser not found"
+                    end
   end
 
   def row_offset
-    5
+    csv_parser.row_offset
   end
 
   def col_offset
-    1
+    csv_parser.col_offset
+  end
+
+  def field_number_for_attribute(attribute)
+    attribute.to_s.split("_").last.to_i
   end
 
   def cols
-    @cols ||= ("A".."DV").to_a
+    csv_parser.cols
   end
 
   def row_parsers
-    @row_parsers ||= body_rows.map do |row|
-      stripped_row = row[col_offset..]
-      headers = ("field_1".."field_125").to_a
-      hash = Hash[headers.zip(stripped_row)]
+    return @row_parsers if @row_parsers
 
-      BulkUpload::Sales::Year2022::RowParser.new(hash)
+    @row_parsers = csv_parser.row_parsers
+
+    @row_parsers.each do |row_parser|
+      row_parser.bulk_upload = bulk_upload
     end
+
+    @row_parsers
+  end
+
+  def rows
+    csv_parser.rows
   end
 
-  def row_sep
-    "\r\n"
-    # "\n"
+  def body_rows
+    csv_parser.body_rows
   end
 
   def validate_file_not_empty
@@ -78,12 +106,20 @@ private
     end
   end
 
+  def validate_min_columns
+    return if halt_validations?
+
+    column_count = rows.map(&:size).min
+
+    errors.add(:base, :under_min_column_count) if column_count < csv_parser.class::MIN_COLUMNS
+  end
+
   def validate_max_columns
     return if halt_validations?
 
-    max_row_size = rows.map(&:size).max
+    column_count = rows.map(&:size).max
 
-    errors.add(:file, :max_row_size) if max_row_size > 126
+    errors.add(:base, :over_max_column_count) if column_count > csv_parser.class::MAX_COLUMNS
   end
 
   def halt_validations!

app/services/bulk_upload/sales/year2022/csv_parser.rb

@@ -0,0 +1,70 @@
+require "csv"
+
+class BulkUpload::Sales::Year2022::CsvParser
+  MIN_COLUMNS = 125
+  MAX_COLUMNS = 126
+
+  attr_reader :path
+
+  def initialize(path:)
+    @path = path
+  end
+
+  def row_offset
+    with_headers? ? 5 : 0
+  end
+
+  def col_offset
+    with_headers? ? 1 : 0
+  end
+
+  def cols
+    @cols ||= ("A".."DV").to_a
+  end
+
+  def row_parsers
+    @row_parsers ||= body_rows.map do |row|
+      stripped_row = row[col_offset..]
+      headers = ("field_1".."field_125").to_a
+      hash = Hash[headers.zip(stripped_row)]
+
+      BulkUpload::Sales::Year2022::RowParser.new(hash)
+    end
+  end
+
+  def body_rows
+    rows[row_offset..]
+  end
+
+  def rows
+    @rows ||= CSV.parse(normalised_string, row_sep:)
+  end
+
+  def column_for_field(field)
+    cols[headers.find_index(field) + col_offset]
+  end
+
+private
+
+  def headers
+    @headers ||= ("field_1".."field_125").to_a
+  end
+
+  def with_headers?
+    rows.map { |r| r[0] }.any? { |cell| cell&.match?(/field number/i) }
+  end
+
+  def row_sep
+    "\n"
+  end
+
+  def normalised_string
+    return @normalised_string if @normalised_string
+
+    @normalised_string = File.read(path, encoding: "bom|utf-8")
+    @normalised_string.gsub!("\r\n", "\n")
+    @normalised_string.scrub!("")
+
+    @normalised_string
+  end
+end

app/services/bulk_upload/sales/year2022/row_parser.rb

@@ -130,18 +130,21 @@ class BulkUpload::Sales::Year2022::RowParser
     field_125: "Was a mortgage used for the purchase of this property? - Outright sale",
   }.freeze
 
+  attribute :bulk_upload
+  attribute :block_log_creation, :boolean, default: -> { false }
+
   attribute :field_1, :string
   attribute :field_2, :integer
   attribute :field_3, :integer
   attribute :field_4, :integer
   attribute :field_5
   attribute :field_6, :integer
-  attribute :field_7, :integer
-  attribute :field_8, :integer
-  attribute :field_9, :integer
-  attribute :field_10, :integer
-  attribute :field_11, :integer
-  attribute :field_12, :integer
+  attribute :field_7, :string
+  attribute :field_8, :string
+  attribute :field_9, :string
+  attribute :field_10, :string
+  attribute :field_11, :string
+  attribute :field_12, :string
   attribute :field_13, :string
   attribute :field_14, :string
   attribute :field_15, :string
@@ -149,10 +152,10 @@ class BulkUpload::Sales::Year2022::RowParser
   attribute :field_17, :string
   attribute :field_18, :string
   attribute :field_19, :string
-  attribute :field_20, :integer
-  attribute :field_21, :integer
-  attribute :field_22, :integer
-  attribute :field_23, :integer
+  attribute :field_20, :string
+  attribute :field_21, :string
+  attribute :field_22, :string
+  attribute :field_23, :string
   attribute :field_24, :integer
   attribute :field_25, :integer
   attribute :field_26, :integer
@@ -221,7 +224,7 @@ class BulkUpload::Sales::Year2022::RowParser
   attribute :field_89, :integer
   attribute :field_90, :integer
   attribute :field_91, :integer
-  attribute :field_92, :integer
+  attribute :field_92, :string
   attribute :field_93, :string
   attribute :field_94
   attribute :field_95, :integer
@@ -256,53 +259,689 @@ class BulkUpload::Sales::Year2022::RowParser
   attribute :field_124, :integer
   attribute :field_125, :integer
 
-  # validates :field_1, presence: true, numericality: { in: (1..12) }
-  # validates :field_4, numericality: { in: (1..999), allow_blank: true }
-  # validates :field_4, presence: true, if: :field_4_presence_check
+  validates :field_2, presence: { message: I18n.t("validations.not_answered", question: "sale completion date (day)") }, on: :after_log
+  validates :field_3, presence: { message: I18n.t("validations.not_answered", question: "sale completion date (month)") }, on: :after_log
+  validates :field_4, presence: { message: I18n.t("validations.not_answered", question: "sale completion date (year)") }, on: :after_log
+  validates :field_4, format: { with: /\A\d{2}\z/, message: I18n.t("validations.setup.saledate.year_not_two_digits") }, on: :after_log
+
+  validates :field_113, presence: { message: I18n.t("validations.not_answered", question: "ownership type") }, on: :after_log
+  validates :field_57, presence: { message: I18n.t("validations.not_answered", question: "shared ownership type") }, if: :shared_ownership?, on: :after_log
+  validates :field_76, presence: { message: I18n.t("validations.not_answered", question: "shared ownership type") }, if: :discounted_ownership?, on: :after_log
+  validates :field_84, presence: { message: I18n.t("validations.not_answered", question: "shared ownership type") }, if: :outright_sale?, on: :after_log
+  validates :field_115, presence: { message: I18n.t("validations.not_answered", question: "will the buyers live in the property") }, if: :outright_sale?, on: :after_log
+  validates :field_116, presence: { message: I18n.t("validations.not_answered", question: "joint purchase") }, if: :joint_purchase_asked?, on: :after_log
+  validates :field_114, presence: { message: I18n.t("validations.not_answered", question: "company buyer") }, if: :outright_sale?, on: :after_log
+  validates :field_109, presence: { message: I18n.t("validations.not_answered", question: "more than 2 buyers") }, if: :joint_purchase?, on: :after_log
+
+  validate :validate_nulls, on: :after_log
+  validate :validate_valid_radio_option, on: :before_log
 
-  validate :validate_possible_answers
+  validate :validate_owning_org_data_given, on: :after_log
+  validate :validate_owning_org_exists, on: :after_log
+  validate :validate_owning_org_permitted, on: :after_log
 
-  # delegate :valid?, to: :native_object
-  # delegate :errors, to: :native_object
+  validate :validate_created_by_exists, on: :after_log
+  validate :validate_created_by_related, on: :after_log
+  validate :validate_relevant_collection_window, on: :after_log
 
   def self.question_for_field(field)
     QUESTIONS[field]
   end
 
+  def attribute_set
+    @attribute_set ||= instance_variable_get(:@attributes)
+  end
+
+  def blank_row?
+    attribute_set
+      .to_hash
+      .reject { |k, _| %w[bulk_upload block_log_creation].include?(k) }
+      .values
+      .compact
+      .empty?
+  end
+
+  def log
+    @log ||= SalesLog.new(attributes_for_log)
+  end
+
+  def valid?
+    errors.clear
+
+    return true if blank_row?
+
+    super(:before_log)
+    before_errors = errors.dup
+
+    log.valid?
+
+    super(:after_log)
+    errors.merge!(before_errors)
+
+    log.errors.each do |error|
+      fields = field_mapping_for_errors[error.attribute] || []
+
+      fields.each do |field|
+        unless errors.include?(field)
+          errors.add(field, error.message)
+        end
+      end
+    end
+
+    errors.blank?
+  end
+
+  def block_log_creation?
+    block_log_creation
+  end
+
 private
 
-  def native_object
-    @native_object ||= SalesLog.new(attributes_for_log)
+  def shared_ownership?
+    field_113 == 1
   end
 
-  def field_mapping
+  def discounted_ownership?
+    field_113 == 2
+  end
+
+  def outright_sale?
+    field_113 == 3
+  end
+
+  def joint_purchase?
+    field_116 == 1
+  end
+
+  def joint_purchase_asked?
+    shared_ownership? || discounted_ownership? || field_114 == 2
+  end
+
+  def field_mapping_for_errors
     {
-      field_117: :buy1livein,
+      purchid: %i[field_1],
+      saledate: %i[field_2 field_3 field_4],
+      noint: %i[field_6],
+      age1_known: %i[field_7],
+      age1: %i[field_7],
+      age2_known: %i[field_8],
+      age2: %i[field_8],
+      age3_known: %i[field_9],
+      age3: %i[field_9],
+      age4_known: %i[field_10],
+      age4: %i[field_10],
+      age5_known: %i[field_11],
+      age5: %i[field_11],
+      age6_known: %i[field_12],
+      age6: %i[field_12],
+      sex1: %i[field_13],
+      sex2: %i[field_14],
+      sex3: %i[field_15],
+      sex4: %i[field_16],
+      sex5: %i[field_17],
+      sex6: %i[field_18],
+      relat2: %i[field_19],
+      relat3: %i[field_20],
+      relat4: %i[field_21],
+      relat5: %i[field_22],
+      relat6: %i[field_23],
+      ecstat1: %i[field_24],
+      ecstat2: %i[field_25],
+      ecstat3: %i[field_26],
+      ecstat4: %i[field_27],
+      ecstat5: %i[field_28],
+      ecstat6: %i[field_29],
+      ethnic_group: %i[field_30],
+      ethnic: %i[field_30],
+      national: %i[field_31],
+      income1nk: %i[field_32],
+      income1: %i[field_32],
+      income2nk: %i[field_33],
+      income2: %i[field_33],
+      inc1mort: %i[field_34],
+      inc2mort: %i[field_35],
+      savingsnk: %i[field_36],
+      savings: %i[field_36],
+      prevown: %i[field_37],
+      prevten: %i[field_39],
+      prevloc: %i[field_40],
+      previous_la_known: %i[field_40],
+      ppcodenk: %i[field_43],
+      ppostcode_full: %i[field_41 field_42],
+      pregyrha: %i[field_44],
+      pregla: %i[field_45],
+      pregghb: %i[field_46],
+      pregother: %i[field_47],
+      pregblank: %i[field_44 field_45 field_46 field_47],
+      disabled: %i[field_48],
+      wheel: %i[field_49],
+      beds: %i[field_50],
+      proptype: %i[field_51],
+      builtype: %i[field_52],
+      la_known: %i[field_53],
+      la: %i[field_53],
+      is_la_inferred: %i[field_53],
+      pcodenk: %i[field_54 field_55],
+      postcode_full: %i[field_54 field_55],
+      wchair: %i[field_56],
+      type: %i[field_57 field_76 field_84 field_113],
+      resale: %i[field_58],
+      hodate: %i[field_59 field_60 field_61],
+      exdate: %i[field_62 field_63 field_64],
+      lanomagr: %i[field_65],
+      frombeds: %i[field_66],
+      fromprop: %i[field_67],
+      value: %i[field_68 field_77 field_87],
+      equity: %i[field_69],
+      mortgage: %i[field_70 field_80 field_88],
+      extrabor: %i[field_71 field_81 field_89],
+      deposit: %i[field_72 field_82 field_90],
+      cashdis: %i[field_73],
+      mrent: %i[field_74],
+      has_mscharge: %i[field_75 field_83 field_91],
+      mscharge: %i[field_75 field_83 field_91],
+      grant: %i[field_78],
+      discount: %i[field_79],
+      othtype: %i[field_85],
+      owning_organisation_id: %i[field_92],
+      created_by: %i[field_93],
+      hhregres: %i[field_95],
+      hhregresstill: %i[field_95],
+      armedforcesspouse: %i[field_97],
+      mortgagelender: %i[field_98 field_100 field_102],
+      mortgagelenderother: %i[field_99 field_101 field_103],
+      hb: %i[field_104],
+      mortlen: %i[field_105 field_106 field_107],
+      proplen: %i[field_108 field_110],
+      jointmore: %i[field_109],
+      staircase: %i[field_111],
+      privacynotice: %i[field_112],
+      ownershipsch: %i[field_113],
+      companybuy: %i[field_114],
+      buylivein: %i[field_115],
+      jointpur: %i[field_116],
+      buy1livein: %i[field_117],
+      buy2livein: %i[field_118],
+      hholdcount: %i[field_119],
+      stairbought: %i[field_120],
+      stairowned: %i[field_121],
+      socprevten: %i[field_122],
+      mortgageused: %i[field_123 field_124 field_125],
+      soctenant: %i[field_39 field_113],
     }
   end
 
-  def validate_possible_answers
-    field_mapping.each do |field, attribute|
-      possible_answers = FormHandler.instance.current_sales_form.questions.find { |q| q.id == attribute.to_s }.answer_options.keys
+  def attributes_for_log
+    attributes = {}
+    attributes["purchid"] = field_1
+    attributes["saledate"] = saledate
+
+    attributes["noint"] = 2 if field_6 == 1
+
+    attributes["details_known_2"] = details_known?(2)
+    attributes["details_known_3"] = details_known?(3)
+    attributes["details_known_4"] = details_known?(4)
+    attributes["details_known_5"] = details_known?(5)
+    attributes["details_known_6"] = details_known?(6)
+
+    attributes["age1_known"] = age1_known?
+    attributes["age1"] = field_7 if attributes["age1_known"].zero? && field_7&.match(/\A\d{1,3}\z|\AR\z/)
+
+    attributes["age2_known"] = age2_known?
+    attributes["age2"] = field_8 if attributes["age2_known"].zero? && field_8&.match(/\A\d{1,3}\z|\AR\z/)
+
+    attributes["age3_known"] = age3_known?
+    attributes["age3"] = field_9 if attributes["age3_known"].zero? && field_9&.match(/\A\d{1,3}\z|\AR\z/)
+
+    attributes["age4_known"] = age4_known?
+    attributes["age4"] = field_10 if attributes["age4_known"].zero? && field_10&.match(/\A\d{1,3}\z|\AR\z/)
+
+    attributes["age5_known"] = age5_known?
+    attributes["age5"] = field_11 if attributes["age5_known"].zero? && field_11&.match(/\A\d{1,3}\z|\AR\z/)
+
+    attributes["age6_known"] = age6_known?
+    attributes["age6"] = field_12 if attributes["age6_known"].zero? && field_12&.match(/\A\d{1,3}\z|\AR\z/)
+
+    attributes["sex1"] = field_13
+    attributes["sex2"] = field_14
+    attributes["sex3"] = field_15
+    attributes["sex4"] = field_16
+    attributes["sex5"] = field_17
+    attributes["sex6"] = field_18
+
+    attributes["relat2"] = field_19
+    attributes["relat3"] = field_20
+    attributes["relat4"] = field_21
+    attributes["relat5"] = field_22
+    attributes["relat6"] = field_23
+
+    attributes["ecstat1"] = field_24
+    attributes["ecstat2"] = field_25
+    attributes["ecstat3"] = field_26
+    attributes["ecstat4"] = field_27
+    attributes["ecstat5"] = field_28
+    attributes["ecstat6"] = field_29
+
+    attributes["ethnic_group"] = ethnic_group_from_ethnic
+    attributes["ethnic"] = field_30
+    attributes["national"] = field_31
+    attributes["income1nk"] = field_32.present? ? 0 : 1
+    attributes["income1"] = field_32
+    attributes["income2nk"] = field_33.present? ? 0 : 1
+    attributes["income2"] = field_33
+    attributes["inc1mort"] = field_34
+    attributes["inc2mort"] = field_35
+    attributes["savingsnk"] = field_36.present? ? 0 : 1
+    attributes["savings"] = field_36
+    attributes["prevown"] = field_37
+
+    attributes["prevten"] = field_39
+    attributes["prevloc"] = field_40
+    attributes["previous_la_known"] = previous_la_known
+    attributes["ppcodenk"] = field_43
+    attributes["ppostcode_full"] = ppostcode_full
+
+    attributes["pregyrha"] = field_44
+    attributes["pregla"] = field_45
+    attributes["pregghb"] = field_46
+    attributes["pregother"] = field_47
+    attributes["pregblank"] = 1 if [field_44, field_45, field_46, field_47].all?(&:blank?)
+
+    attributes["disabled"] = field_48
+    attributes["wheel"] = field_49
+    attributes["beds"] = field_50
+    attributes["proptype"] = field_51
+    attributes["builtype"] = field_52
+    attributes["la_known"] = field_53.present? ? 1 : 0
+    attributes["la"] = field_53
+    attributes["is_la_inferred"] = false
+    attributes["pcodenk"] = 0 if postcode_full.present?
+    attributes["postcode_full"] = postcode_full
+    attributes["wchair"] = field_56
+
+    attributes["type"] = sale_type
+
+    attributes["resale"] = field_58
+
+    attributes["hodate"] = hodate
+    attributes["exdate"] = exdate
+
+    attributes["lanomagr"] = field_65
+
+    attributes["frombeds"] = field_66
+    attributes["fromprop"] = field_67
+
+    attributes["value"] = value
+    attributes["equity"] = field_69
+    attributes["mortgage"] = mortgage
+    attributes["extrabor"] = extrabor
+    attributes["deposit"] = deposit
+    attributes["cashdis"] = field_73
+    attributes["mrent"] = field_74
+    attributes["has_mscharge"] = mscharge.present? ? 1 : 0
+    attributes["mscharge"] = mscharge
+    attributes["grant"] = field_78
+    attributes["discount"] = field_79
+
+    attributes["othtype"] = field_85
+
+    attributes["owning_organisation_id"] = owning_organisation_id
+    attributes["created_by"] = created_by || bulk_upload.user
+    attributes["hhregres"] = hhregres
+    attributes["hhregresstill"] = hhregresstill
+    attributes["armedforcesspouse"] = field_97
+
+    attributes["mortgagelender"] = mortgagelender
+    attributes["mortgagelenderother"] = mortgagelenderother
+
+    attributes["hb"] = field_104
+
+    attributes["mortlen"] = mortlen
+
+    attributes["proplen"] = proplen
+    attributes["jointmore"] = field_109
+    attributes["staircase"] = field_111
+    attributes["privacynotice"] = field_112
+    attributes["ownershipsch"] = field_113
+    attributes["companybuy"] = field_114
+    attributes["buylivein"] = field_115
+    attributes["jointpur"] = field_116
+    attributes["buy1livein"] = field_117
+    attributes["buy2livein"] = field_118
+    attributes["hholdcount"] = field_119
+    attributes["stairbought"] = field_120
+    attributes["stairowned"] = field_121
+    attributes["socprevten"] = field_122
+    attributes["mortgageused"] = mortgageused
+    attributes["soctenant"] = soctenant
+
+    attributes
+  end
+
+  def saledate
+    Date.new(field_4 + 2000, field_3, field_2) if field_2.present? && field_3.present? && field_4.present?
+  rescue Date::Error
+    Date.new
+  end
+
+  def hodate
+    Date.new(field_61 + 2000, field_60, field_59) if field_59.present? && field_60.present? && field_61.present?
+  rescue Date::Error
+    Date.new
+  end
+
+  def exdate
+    Date.new(field_64 + 2000, field_63, field_62) if field_62.present? && field_63.present? && field_64.present?
+  rescue Date::Error
+    Date.new
+  end
 
-      unless possible_answers.include?(public_send(field))
-        errors.add(field, "Value supplied is not one of the permitted values")
+  def age1_known?
+    return 1 if field_7 == "R"
+    return 1 if field_7.blank?
+
+    0
+  end
+
+  [
+    { person: 2, field: :field_8 },
+    { person: 3, field: :field_9 },
+    { person: 4, field: :field_10 },
+    { person: 5, field: :field_11 },
+    { person: 6, field: :field_12 },
+  ].each do |hash|
+    define_method("age#{hash[:person]}_known?") do
+      return 1 if public_send(hash[:field]) == "R"
+      return 0 if send("person_#{hash[:person]}_present?")
+      return 1 if public_send(hash[:field]).blank?
+
+      0
+    end
+  end
+
+  def person_2_present?
+    field_8.present? || field_14.present? || field_19.present?
+  end
+
+  def person_3_present?
+    field_9.present? || field_15.present? || field_20.present?
+  end
+
+  def person_4_present?
+    field_10.present? || field_16.present? || field_21.present?
+  end
+
+  def person_5_present?
+    field_11.present? || field_17.present? || field_22.present?
+  end
+
+  def person_6_present?
+    field_12.present? || field_18.present? || field_23.present?
+  end
+
+  def details_known?(person_n)
+    send("person_#{person_n}_present?") ? 1 : 2
+  end
+
+  def ethnic_group_from_ethnic
+    return nil if field_30.blank?
+
+    case field_30
+    when 1, 2, 3, 18
+      0
+    when 4, 5, 6, 7
+      1
+    when 8, 9, 10, 11, 15
+      2
+    when 12, 13, 14
+      3
+    when 16, 19
+      4
+    when 17
+      17
+    end
+  end
+
+  def postcode_full
+    "#{field_54} #{field_55}" if field_54 && field_55
+  end
+
+  def ppostcode_full
+    "#{field_41} #{field_42}" if field_41 && field_42
+  end
+
+  def sale_type
+    return field_57 if shared_ownership?
+    return field_76 if discounted_ownership?
+    return field_84 if outright_sale?
+  end
+
+  def value
+    return field_68 if shared_ownership?
+    return field_77 if discounted_ownership?
+    return field_87 if outright_sale?
+  end
+
+  def mortgage
+    return field_70 if shared_ownership?
+    return field_80 if discounted_ownership?
+    return field_88 if outright_sale?
+  end
+
+  def extrabor
+    return field_71 if shared_ownership?
+    return field_81 if discounted_ownership?
+    return field_89 if outright_sale?
+  end
+
+  def deposit
+    return field_72 if shared_ownership?
+    return field_82 if discounted_ownership?
+    return field_90 if outright_sale?
+  end
+
+  def mscharge
+    return field_75 if shared_ownership?
+    return field_83 if discounted_ownership?
+    return field_91 if outright_sale?
+  end
+
+  def mortgagelender
+    return field_98 if shared_ownership?
+    return field_100 if discounted_ownership?
+    return field_102 if outright_sale?
+  end
+
+  def mortgagelenderother
+    return field_99 if shared_ownership?
+    return field_101 if discounted_ownership?
+    return field_103 if outright_sale?
+  end
+
+  def mortlen
+    return field_105 if shared_ownership?
+    return field_106 if discounted_ownership?
+    return field_107 if outright_sale?
+  end
+
+  def proplen
+    return field_110 if shared_ownership?
+    return field_108 if discounted_ownership?
+  end
+
+  def mortgageused
+    return field_123 if shared_ownership?
+    return field_124 if discounted_ownership?
+    return field_125 if outright_sale?
+  end
+
+  def owning_organisation
+    Organisation.find_by_id_on_multiple_fields(field_92)
+  end
+
+  def owning_organisation_id
+    owning_organisation&.id
+  end
+
+  def created_by
+    @created_by ||= User.find_by(email: field_93)
+  end
+
+  def hhregres
+    case field_95
+    when 3 then 3
+    when 4, 5, 6 then 1
+    when 7 then 7
+    when 8 then 8
+    end
+  end
+
+  def hhregresstill
+    return unless hhregres == 1
+
+    field_95
+  end
+
+  def previous_la_known
+    field_40.present? ? 1 : 0
+  end
+
+  def soctenant
+    return unless field_39 && field_113
+
+    if (field_39 == 1 || fields_39 == 2) && field_113 == 1
+      1
+    elsif field_113 == 1
+      2
+    end
+  end
+
+  def block_log_creation!
+    self.block_log_creation = true
+  end
+
+  def questions
+    @questions ||= log.form.subsections.flat_map { |ss| ss.applicable_questions(log) }
+  end
+
+  def validate_owning_org_data_given
+    if field_92.blank?
+      block_log_creation!
+
+      if errors[:field_92].blank?
+        errors.add(:field_92, "The owning organisation code is incorrect", category: :setup)
       end
     end
   end
 
-  def attributes_for_log
-    hash = field_mapping.invert
-    attributes = {}
+  def validate_owning_org_exists
+    if owning_organisation.nil?
+      block_log_creation!
 
-    hash.map do |k, v|
-      attributes[k] = public_send(v)
+      if errors[:field_92].blank?
+        errors.add(:field_92, "The owning organisation code is incorrect", category: :setup)
+      end
     end
+  end
 
-    attributes
+  def validate_owning_org_owns_stock
+    if owning_organisation && !owning_organisation.holds_own_stock?
+      block_log_creation!
+
+      if errors[:field_92].blank?
+        errors.add(:field_92, "The owning organisation code provided is for an organisation that does not own stock", category: :setup)
+      end
+    end
+  end
+
+  def validate_owning_org_permitted
+    if owning_organisation && !bulk_upload.user.organisation.affiliated_stock_owners.include?(owning_organisation)
+      block_log_creation!
+
+      if errors[:field_92].blank?
+        errors.add(:field_92, "You do not have permission to add logs for this owning organisation", category: :setup)
+      end
+    end
+  end
+
+  def validate_created_by_exists
+    return if field_93.blank?
+
+    unless created_by
+      errors.add(:field_93, "User with the specified email could not be found")
+    end
   end
 
-  # def field_4_presence_check
-  #   [1, 3, 5, 7, 9, 11].include?(field_1)
-  # end
+  def validate_created_by_related
+    return unless created_by
+
+    unless created_by.organisation == owning_organisation
+      block_log_creation!
+      errors.add(:field_93, "User must be related to owning organisation")
+    end
+  end
+
+  def setup_question?(question)
+    log.form.setup_sections[0].subsections[0].questions.include?(question)
+  end
+
+  def validate_nulls
+    field_mapping_for_errors.each do |error_key, fields|
+      question_id = error_key.to_s
+      question = questions.find { |q| q.id == question_id }
+
+      next unless question
+      next if log.optional_fields.include?(question.id)
+      next if question.completed?(log)
+
+      if setup_question?(question)
+        fields.each do |field|
+          if errors[field].present?
+            errors.add(field, I18n.t("validations.not_answered", question: question.check_answer_label&.downcase), category: :setup)
+          end
+        end
+      else
+        fields.each do |field|
+          unless errors.any? { |e| fields.include?(e.attribute) }
+            errors.add(field, I18n.t("validations.not_answered", question: question.check_answer_label&.downcase))
+          end
+        end
+      end
+    end
+  end
+
+  def validate_valid_radio_option
+    log.attributes.each do |question_id, _v|
+      question = log.form.get_question(question_id, log)
+
+      next unless question&.type == "radio"
+      next if log[question_id].blank? || question.answer_options.key?(log[question_id].to_s) || !question.page.routed_to?(log, nil)
+
+      fields = field_mapping_for_errors[question_id.to_sym] || []
+
+      if setup_question?(question)
+        fields.each do |field|
+          if errors[field].present?
+            errors.add(field, I18n.t("validations.invalid_option", question: QUESTIONS[field]), category: :setup)
+          end
+        end
+      else
+        fields.each do |field|
+          unless errors.any? { |e| fields.include?(e.attribute) }
+            errors.add(field, I18n.t("validations.invalid_option", question: QUESTIONS[field]))
+          end
+        end
+      end
+    end
+  end
+
+  def validate_relevant_collection_window
+    return if saledate.blank? || bulk_upload.form.blank?
+
+    unless bulk_upload.form.valid_start_date_for_form?(saledate)
+      errors.add(:field_2, I18n.t("validations.date.outside_collection_window"))
+      errors.add(:field_3, I18n.t("validations.date.outside_collection_window"))
+      errors.add(:field_4, I18n.t("validations.date.outside_collection_window"))
+    end
+  end
 end

app/views/bulk_upload_sales_results/resume.html.erb

@@ -0,0 +1,11 @@
+<div class="govuk-grid-row">
+  <div class="govuk-grid-column-two-thirds">
+    <h1 class="govuk-heading-xl">There are no more logs that need updating</h1>
+  </div>
+</div>
+
+<p class="govuk-body-l">
+  You’ve completed all the logs that had errors from your bulk upload.
+</p>
+
+<%= govuk_button_link_to "Back to all logs", sales_logs_path, button: true %>

app/views/bulk_upload_sales_results/summary.html.erb

@@ -0,0 +1,30 @@
+<div class="govuk-grid-row">
+  <div class="govuk-grid-column-two-thirds">
+    <span class="govuk-caption-l">Bulk upload for sales (<%= @bulk_upload.year_combo %>)</span>
+    <h1 class="govuk-heading-l">Fix <%= pluralize(@bulk_upload.bulk_upload_errors.count, "error") %> and upload file again</h1>
+
+    <p class="govuk-body-l">
+      We could not create logs from your bulk upload. Below is a list of everything that you need to fix your spreadsheet. You can download the <%= govuk_link_to "specification", Forms::BulkUploadSales::PrepareYourFile.new(year: @bulk_upload.year).specification_path, target: "_blank" %> to help you fix the cells in your CSV file.
+    </p>
+
+    <p class="govuk-body-l">
+      Filename: <%= @bulk_upload.filename %>
+    </p>
+  </div>
+</div>
+
+<div class="govuk-grid-row">
+  <%= govuk_tabs(title: "Error reports") do |c| %>
+    <% c.with_tab(label: "Summary") do %>
+      <%= render BulkUploadErrorSummaryTableComponent.new(bulk_upload: @bulk_upload) %>
+    <% end %>
+
+    <% c.with_tab(label: "Full error report") do %>
+      <% @bulk_upload.bulk_upload_errors.order_by_cell.group_by(&:row).each do |_row, errors_for_row| %>
+        <%= render BulkUploadErrorRowComponent.new(bulk_upload_errors: errors_for_row) %>
+      <% end %>
+    <% end %>
+  <% end %>
+</div>
+
+<%= govuk_button_link_to "Upload your file again", start_bulk_upload_sales_logs_path %>

app/views/bulk_upload_sales_resume/confirm.html.erb

@@ -0,0 +1,22 @@
+<% content_for :before_content do %>
+  <%= govuk_back_link href: @form.back_path %>
+<% end %>
+
+<div class="govuk-grid-row">
+  <div class="govuk-grid-column-two-thirds">
+    <span class="govuk-caption-l">Bulk upload for sales (<%= @bulk_upload.year_combo %>)</span>
+    <h1 class="govuk-heading-l">Are you sure you want to upload all logs from this bulk upload?</h1>
+
+    <p class="govuk-body">There are <%= pluralize(@bulk_upload.logs.count, "log") %> in this bulk upload with <%= pluralize(@bulk_upload.bulk_upload_errors.count, "error") %> that still need to be fixed after upload.</p>
+
+    <%= govuk_warning_text(icon_fallback_text: "Danger") do %>
+      You can not delete logs once you create them
+    <% end %>
+
+    <%= form_with model: @form, scope: :form, url: page_bulk_upload_sales_resume_path(@bulk_upload, page: "confirm"), method: :patch do |f| %>
+      <%= f.govuk_submit %>
+
+      <%= govuk_button_link_to "Cancel", @form.back_path, secondary: true %>
+    <% end %>
+  </div>
+</div>

app/views/bulk_upload_sales_resume/fix_choice.html.erb

@@ -0,0 +1,36 @@
+<div class="govuk-grid-row">
+  <div class="govuk-grid-column-two-thirds">
+    <%= form_with model: @form, scope: :form, url: page_bulk_upload_sales_resume_path(@bulk_upload, page: "fix-choice"), method: :patch do |f| %>
+      <%= f.govuk_error_summary %>
+
+      <span class="govuk-caption-l">Bulk upload for sales (<%= @bulk_upload.year_combo %>)</span>
+      <h1 class="govuk-heading-l">How would you like to fix <%= pluralize(@bulk_upload.bulk_upload_errors.count, "error") %>?</h1>
+
+      <div class="govuk-body-l">
+        <%= @bulk_upload.filename %>
+      </div>
+
+      <div class="govuk-body">
+        <%= @form.recommendation %>
+      </div>
+
+      <%= govuk_details(summary_text: "How to choose between fixing errors on the CORE site or in the CSV") do %>
+        <p class="govuk-body">When it comes to fixing errors, there are pros and cons to doing it on a CSV versus doing it on a website.</p>
+
+        <p class="govuk-body">Fixing errors on a CSV file can be beneficial because it allows you to easily make changes to multiple records at once, and you can use tools like Excel to quickly identify and correct errors. However, if the CSV file is not properly formatted, it can be difficult to identify which records contain errors.</p>
+
+        <p class="govuk-body">Fixing errors on a website can be convenient because you can see the data in context and make changes in real-time. However, this approach can be time-consuming if you need to make changes to multiple records, and it may be more difficult to identify errors in a large dataset.</p>
+
+        <p class="govuk-body">Ultimately, the best approach will depend on the specific situation and the nature of the errors that need to be fixed.</p>
+      <% end %>
+
+      <%= f.govuk_collection_radio_buttons :choice,
+            @form.options,
+            :id,
+            :name,
+            legend: { hidden: true } %>
+
+      <%= f.govuk_submit %>
+    <% end %>
+  </div>
+</div>

config/locales/en.yml

@@ -170,6 +170,7 @@ en:
           Enter a date within the %{current_start_year_short}/%{current_end_year_short} collection year, which is between %{current_start_year_long} and %{current_end_year_long}
         previous_and_current_collection_year:
           "Enter a date within the %{previous_start_year_short}/%{previous_end_year_short} or %{previous_end_year_short}/%{current_end_year_short} collection years, which is between %{previous_start_year_long} and %{current_end_year_long}"
+        year_not_two_digits: "Sale completion year must be 2 digits"
       type:
         percentage_bought_must_be_at_least_threshold: "The minimum increase in equity while staircasing is %{threshold}% for this shared ownership type"
 

config/routes.rb

@@ -199,7 +199,21 @@ Rails.application.routes.draw do
         end
       end
 
-      resources :bulk_upload_sales_results, path: "bulk-upload-results", only: [:show]
+      resources :bulk_upload_sales_results, path: "bulk-upload-results", only: [:show] do
+        member do
+          get :resume
+          get :summary
+        end
+      end
+
+      resources :bulk_upload_sales_resume, path: "bulk-upload-resume", only: %i[show update] do
+        member do
+          get :start
+
+          get "*page", to: "bulk_upload_sales_resume#show", as: "page"
+          patch "*page", to: "bulk_upload_sales_resume#update"
+        end
+      end
     end
 
     member do

db/migrate/20230419153741_add_status_cache.rb

@@ -0,0 +1,5 @@
+class AddStatusCache < ActiveRecord::Migration[7.0]
+  def change
+    add_column :sales_logs, :status_cache, :integer, null: false, default: 0
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2023_04_18_095819) do
+ActiveRecord::Schema[7.0].define(version: 2023_04_19_153741) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
@@ -590,6 +590,7 @@ ActiveRecord::Schema[7.0].define(version: 2023_04_18_095819) do
     t.integer "student_not_child_value_check"
     t.integer "percentage_discount_value_check"
     t.integer "buyer_livein_value_check"
+    t.integer "status_cache", default: 0, null: false
     t.index ["bulk_upload_id"], name: "index_sales_logs_on_bulk_upload_id"
     t.index ["created_by_id"], name: "index_sales_logs_on_created_by_id"
     t.index ["old_id"], name: "index_sales_logs_on_old_id", unique: true

spec/factories/sales_log.rb

@@ -8,6 +8,7 @@ FactoryBot.define do
       purchid { "PC123" }
       ownershipsch { 2 }
       type { 8 }
+      jointpur { 2 }
       saledate { Time.utc(2023, 2, 2, 10, 36, 49) }
     end
     trait :shared_ownership do

spec/fixtures/files/2022_23_sales_bulk_upload.csv

@@ -116,4 +116,4 @@ OR
 If field 39 = 3 - 9",If field 113 = 2 or 3,If field 113 = 1 or 3,If field 113 = 1 or 2
 Bulk upload format and duplicate check,Yes,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
 Field number,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125
-,1,1,1,23,,1,30,,,,,,M,,,,,,,,,,,1,,,,,,1,18,20000,,1,,10000,2,,3,,EC1N,2TD,2,2,2,2,2,2,2,2,1,1,,,,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+,22 test BU,22,2,23,,1,32,32,,,,,M,F,,,,,R,,,,,1,2,,,,,12,18,30000,15000,1,1,20000,3,,1,E09000008,A1,1AA,1,,1,1,,3,3,2,1,1,E09000008,CR0,4BB,3,2,2,23,3,22,30,3,22,3,1,1,250000,25,42500,3,20000,,800,200,,,,,,,,,,,,,,,,,,,,3,,5,1,,,,,,4,20,,,,2,5,1,1,1,,1,1,1,1,0,10,10,1,1,,

spec/fixtures/files/completed_2022_23_sales_bulk_upload.csv

@@ -0,0 +1,119 @@
+Question,What is the purchaser code?,What is the day of the sale completion date? - DD,What is the month of the sale completion date? - MM,What is the year of the sale completion date? - YY,[BLANK],Was the buyer interviewed for any of the answers you will provide on this log?,Age of Buyer 1,Age of Buyer 2 or Person 2,Age of Person 3,Age of Person 4,Age of Person 5,Age of Person 6,Gender identity of Buyer 1,Gender identity of Buyer 2 or Person 2,Gender identity of Person 3,Gender identity of Person 4,Gender identity of Person 5,Gender identity of Person 6,Person 2's relationship to lead tenant,Person 3's relationship to lead tenant,Person 4's relationship to lead tenant,Person 5's relationship to lead tenant,Person 6's relationship to lead tenant,Working situation of Buyer 1,Working situation of Buyer 2 or Person 2,Working situation of Person 3,Working situation of Person 4,Working situation of Person 5,Working situation of Person 6,What is the buyer 1's ethnic group?,What is buyer 1's nationality?,What is buyer 1's gross annual income?,What is buyer 2's gross annual income?,Was buyer 1's income used for a mortgage application?,Was buyer 2's income used for a mortgage application?,"What is the total amount the buyers had in savings before they paid any deposit for the property?
+
+To the nearest £10",Have any of the buyers previously owned a property?,[BLANK],What was buyer 1's previous tenure?,What is the local authority of buyer 1's last settled home,Part 1 of postcode of buyer 1's last settled home,Part 2 of postcode of buyer 1's last settled home,Do you know the postcode of buyer 1's last settled home?,Was the buyer registered with their PRP (HA)?,Was the buyer registered with the local authority?,Was the buyer registered with a Help to Buy agent?,Was the buyer registered with another PRP (HA)?,Does anyone in the household consider themselves to have a disability?,Does anyone in the household use a wheelchair?,How many bedrooms does the property have?,What type of unit is the property?,Which type of building is the property?,What is the local authority of the property?,Part 1 of postcode of property,Part 2 of postcode of property,Is the property built or adapted to wheelchair-user standards?,What is the type of shared ownership sale?,"Is this a resale?
+
+Shared ownership","What is the day of the practical completion or handover date? - DD
+
+Shared ownership","What is the month of the practical completion or handover date? - MM
+
+Shared ownership","What is the year of the practical completion or handover date? - YY
+
+Shared ownership","What is the day of the exchange of contracts date? - DD
+
+Shared ownership","What is the month of the exchange of contracts date? - MM
+
+Shared ownership","What is the year of the exchange of contracts date? - YY
+
+Shared ownership","Was the household re-housed under a local authority nominations agreement?
+
+Shared ownership","How many bedrooms did the buyer's previous property have?
+
+Shared ownership","What was the type of the buyer's previous property?
+
+Shared ownership","What was the full purchase price?
+
+Shared ownership","What was the initial percentage equity stake purchased?
+
+Shared ownership","What is the mortgage amount?
+
+Shared ownership","Does this include any extra borrowing?
+
+Shared ownership","How much was the cash deposit paid on the property?
+
+Shared ownership","How much cash discount was given through Social Homebuy?
+
+Shared ownership","What is the basic monthly rent?
+
+Shared ownership","What are the total monthly leasehold charges for the property?
+
+Shared ownership",What is the type of discounted ownership sale?,"What was the full purchase price?
+
+Discounted ownership","What was the amount of any loan, grant, discount or subsidy given?
+
+Discounted ownership","What was the percentage discount?
+
+Discounted ownership","What is the mortgage amount?
+
+Discounted ownership","Does this include any extra borrowing?
+
+Discounted ownership","How much was the cash deposit paid on the property?
+
+Discounted ownership","What are the total monthly leasehold charges for the property?
+
+Discounted ownership",What is the type of outright sale?,"What is the 'other' type of outright sale?
+
+Outright sale",[BLANK],"What is the full purchase price?
+
+Outright sale","What is the mortgage amount?
+
+Outright sale","Does this include any extra borrowing?
+
+Outright sale","How much was the cash deposit paid on the property?
+
+Outright sale","What are the total monthly leasehold charges for the property?
+
+Outright sale","Which organisation owned this property before the sale?
+
+Organisation's CORE ID",Username,BLANK,Has the buyer ever served in the UK Armed Forces and for how long?,[BLANK],Are any of the buyers a spouse or civil partner of a UK Armed Forces regular who died in service within the last 2 years?,"What is the name of the mortgage lender?
+
+Shared ownership","What is the name of the 'other' mortgage lender?
+
+Shared ownership","What is the name of the mortgage lender?
+
+Discounted ownership","What is the name of the 'other' mortgage lender?
+
+Discounted ownership","What is the name of the mortgage lender?
+
+Outright sale","What is the name of the 'other' mortgage lender?
+
+Outright sale",Were the buyers receiving any of these housing-related benefits immediately before buying this property?,"What is the length of the mortgage in years?
+
+Shared ownership","What is the length of the mortgage in years?
+
+Discounted ownership","What is the length of the mortgage in years?
+
+Outright sale","How long have the buyers been living in the property before the purchase?
+
+Discounted ownership",Are there more than two joint purchasers of this property?,"How long have the buyers been living in the property before the purchase?
+
+Shared ownership",Is this a staircasing transaction?,Data Protection question,Was this purchase made through an ownership scheme?,"Is the buyer a company?
+
+Outright sale",Will the buyers live in the property?,Is this a joint purchase?,Will buyer 1 live in the property?,Will buyer 2 live in the property?,"Besides the buyers, how many people live in the property?","What percentage of the property has been bought in this staircasing transaction?
+
+Shared ownership","What percentage of the property does the buyer now own in total?
+
+Shared ownership","What was the rent type of the buyer's previous property?
+
+Shared ownership","Was a mortgage used for the purchase of this property?
+
+Shared ownership","Was a mortgage used for the purchase of this property?
+
+Discounted ownership","Was a mortgage used for the purchase of this property?
+
+Outright sale"
+Values,Max 9 digits,1 - 31,1 - 12,19 - 23,,1 or null,"15 - 110
+or R",1 - 110 or R,,,,,"M, F, X or R",,,,,,"P, C, X or R",,,,,0 - 10,,,,,,1 - 19,"12 -13, 17 -19",0 - 99999,,1 or 2,1 or 2,0 - 999990,1 - 3,,1 - 7 or 9,ONS CODE - E + 9 digits,XXX(X),XXX,1 or null,,,,,1 - 3,1 - 3,1 - 9,1 - 4 or 9,1 or 2,ONS CODE E + 9 digits,XXX(X),XXX,1 - 3,"2, 16, 18, 24, 28 or 30-31",1 or 2,1 - 31,1 - 12,19 - 23,1 - 31,1 - 12,19 - 23,1 - 3,1 - 9,1 - 4 or 9,0 - 999999,0 - 100,0 - 999999,1 - 3,0 - 999999,,0 - 999.99,,"8, 9, 14, 21, 22, 27 or 29",0 - 999999,,0 - 100,0 - 999999,1 - 3,0 - 999999,0 - 999.99,10 or 12,,,0 - 999999,,1-3,0 - 999999,0-999.99,Up to 7 digits,Username of CORE account this sales log should be assigned to,,3 - 8,,4 - 7,1 - 40,,1 - 40,,1 - 40,,1 - 4, Integer <=60, Integer <=60, Integer <=60, Integer <=80,1 - 3, Integer <=80,1 - 3,1,1 - 3,1 - 2,1 - 2,1 - 2,1 - 2,1 - 2,0 - 5,1 - 100,1 - 100,1-3 or 9-10,1 - 2,1 - 2,1 - 2
+Can be Null?,No,,,,,No,No,"If fields 14, 19 and 25 are all also null","If fields 15, 20 and 26 are all also null","If fields 16, 21 and 27 are all also null","If fields 17, 22 and 28 are all also null","If fields 18, 23 and 29 are all also null",No,"If fields 8, 19 and 25 are all also null","If fields 9, 20 and 26 are also null","If fields 10, 21 and 27 are all also null","If fields 11, 22 and 28 are all also null","If fields 12, 23 and 29 are all also null","If fields 8, 14 and 25 are all also null","If fields 9, 15 and 26 are all also null","If fields 10, 16 and 27 are all also null","If fields 11, 17 and 28 are all also null","If fields 12, 18 and 29 are all also null",If field 6 = 1,"If fields 8, 14 and 19 are all also null","If fields 9, 15 and 20 are all also null","If fields 10, 16 and 21 are all also null","If fields 11, 17 and 22 are all also null","If fields 12, 18 and 23 are all also null",If field 6 =  1,,,If field 116 = 2,If field 32  is null,If field 116 = 2,If field 6 = 1,,,If field 6 = 1,No,If field 43 = 1,,If fields 41 and 42 BOTH have valid entries,Yes,,,,If field 6 = 1,,No,,,,,,,If field 113 = 2 or 3,,,,,,,,,"If field 113 = 2 or 3
+OR
+field 39 = 3 - 7 or 9",,If field 113 = 2 or 3,,,,,"If field 57 is null, 2, 16, 24 or 28",If field 113 = 2 or 3,,If field 113 = 1 or 3,If field 76 is null,"If field 76 is null, 9 or 14","If field 76 is null, 8, 21 or 22",If field 113 = 1 or 3,,,,If field 113 = 1 or 2,If field 84 is null or 10,,If field 113 = 1 or 2,,,,,No,Yes,,No,,No,If field 113 = 2 or 3,"If field 113 = 2 or 3
+OR
+If field 98 is not 40",If field 113 = 1 or 3,"If field 113 = 1 or 3
+OR
+If field 100 is not 40",If field 113 = 1 or 2,"If field 113 = 1 or 2
+OR
+If field 102 is not 40",No,If field 113 = 2 or 3,If field 113 = 1 or 3,If field 113 = 1 or 2,If field 113 = 1 or 3,If field 116 = 2,If field 113 = 2 or 3,If field 113 = 2 or 3,No,No,If field 113 = 1 or 2,If field 113 = 1 or 2,No,No,If field 116 = 2,No,If field 113 = 2 or 3,If field 113 = 2 or 3,"If field 113 = 1 or 2
+OR
+If field 39 = 3 - 9",If field 113 = 2 or 3,If field 113 = 1 or 3,If field 113 = 1 or 2
+Bulk upload format and duplicate check,Yes,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Field number,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125
+,22 test BU,22,2,23,,1,32,32,,,,,M,F,,,,,R,,,,,1,2,,,,,12,18,30000,15000,1,1,20000,3,,1,E09000008,A1,1AA,1,,1,1,,3,3,2,1,1,E09000008,CR0,4BB,3,2,2,23,3,22,30,3,22,3,1,1,250000,25,42500,3,20000,,800,200,,,,,,,,,,,,,,,,,3,,,3,,5,1,,,,,,4,20,,,,2,5,1,1,1,,1,1,1,1,0,10,10,1,1,,

spec/services/bulk_upload/sales/log_creator_spec.rb

@@ -0,0 +1,99 @@
+require "rails_helper"
+
+RSpec.describe BulkUpload::Sales::LogCreator do
+  subject(:service) { described_class.new(bulk_upload:, path:) }
+
+  let(:owning_org) { create(:organisation, old_visible_id: 123) }
+  let(:user) { create(:user, organisation: owning_org) }
+
+  let(:bulk_upload) { create(:bulk_upload, :sales, user:) }
+  let(:path) { file_fixture("completed_2022_23_sales_bulk_upload.csv") }
+
+  describe "#call" do
+    context "when a valid csv with new log" do
+      it "creates a new log" do
+        expect { service.call }.to change(SalesLog, :count)
+      end
+
+      it "create a log with pending status" do
+        service.call
+        expect(SalesLog.last.status).to eql("pending")
+      end
+
+      it "associates log with bulk upload" do
+        service.call
+
+        log = SalesLog.last
+        expect(log.bulk_upload).to eql(bulk_upload)
+        expect(bulk_upload.sales_logs).to include(log)
+      end
+    end
+
+    context "when a valid csv with several blank rows" do
+      let(:file) { Tempfile.new }
+      let(:path) { file.path }
+      let(:log) { SalesLog.new }
+
+      before do
+        file.write(BulkUpload::LogToCsv.new(log:, col_offset: 0).to_2022_sales_csv_row)
+        file.write(BulkUpload::LogToCsv.new(log:, col_offset: 0).to_2022_sales_csv_row)
+        file.write(BulkUpload::LogToCsv.new(log:, col_offset: 0).to_2022_sales_csv_row)
+        file.rewind
+      end
+
+      it "ignores them and does not create the logs" do
+        expect { service.call }.not_to change(SalesLog, :count)
+      end
+    end
+
+    context "when a valid csv with row with one invalid non setup field" do
+      let(:file) { Tempfile.new }
+      let(:path) { file.path }
+      let(:log) do
+        build(
+          :sales_log,
+          :completed,
+          age1: 5,
+          owning_organisation: owning_org,
+        )
+      end
+
+      before do
+        file.write(BulkUpload::LogToCsv.new(log:, col_offset: 0).to_2022_sales_csv_row)
+        file.rewind
+      end
+
+      it "creates the log" do
+        expect { service.call }.to change(SalesLog, :count).by(1)
+      end
+
+      it "blanks invalid field" do
+        service.call
+
+        record = SalesLog.last
+        expect(record.age1).to be_blank
+      end
+    end
+
+    context "when pre-creating logs" do
+      subject(:service) { described_class.new(bulk_upload:, path:) }
+
+      it "creates a new log" do
+        expect { service.call }.to change(SalesLog, :count)
+      end
+
+      it "creates a log with correct states" do
+        service.call
+
+        last_log = SalesLog.last
+
+        expect(last_log.status).to eql("pending")
+        expect(last_log.status_cache).to eql("completed")
+      end
+    end
+
+    context "when valid csv with existing log" do
+      xit "what should happen?"
+    end
+  end
+end

spec/services/bulk_upload/sales/validator_spec.rb

@@ -3,7 +3,9 @@ require "rails_helper"
 RSpec.describe BulkUpload::Sales::Validator do
   subject(:validator) { described_class.new(bulk_upload:, path:) }
 
-  let(:bulk_upload) { create(:bulk_upload) }
+  let(:user) { create(:user, organisation:) }
+  let(:organisation) { create(:organisation, old_visible_id: "3") }
+  let(:bulk_upload) { create(:bulk_upload, user:) }
   let(:path) { file.path }
   let(:file) { Tempfile.new }
 
@@ -14,6 +16,18 @@ RSpec.describe BulkUpload::Sales::Validator do
       end
     end
 
+    context "when file has too few columns" do
+      before do
+        file.write("a," * 112)
+        file.write("\n")
+        file.rewind
+      end
+
+      it "is not valid" do
+        expect(validator).not_to be_valid
+      end
+    end
+
     context "when file has too many columns" do
       before do
         file.write((%w[a] * 127).join(","))
@@ -28,20 +42,194 @@ RSpec.describe BulkUpload::Sales::Validator do
     context "when incorrect headers"
   end
 
-  context "when a valid csv that contains errors" do
-    let(:path) { file_fixture("2022_23_sales_bulk_upload.csv") }
+  describe "#call" do
+    context "when a valid csv" do
+      let(:path) { file_fixture("2022_23_sales_bulk_upload.csv") }
+
+      it "creates validation errors" do
+        expect { validator.call }.to change(BulkUploadError, :count)
+      end
+
+      it "create validation error with correct values" do
+        validator.call
+
+        error = BulkUploadError.find_by(row: "6", field: "field_92", category: "setup")
+
+        expect(error.field).to eql("field_92")
+        expect(error.error).to eql("The owning organisation code is incorrect")
+        expect(error.purchaser_code).to eql("22 test BU")
+        expect(error.row).to eql("6")
+        expect(error.cell).to eql("CO6")
+        expect(error.col).to eql("CO")
+      end
+    end
+
+    context "with unix line endings" do
+      let(:fixture_path) { file_fixture("2022_23_sales_bulk_upload.csv") }
+      let(:file) { Tempfile.new }
+      let(:path) { file.path }
+
+      before do
+        string = File.read(fixture_path)
+        string.gsub!("\r\n", "\n")
+        file.write(string)
+        file.rewind
+      end
+
+      it "creates validation errors" do
+        expect { validator.call }.to change(BulkUploadError, :count)
+      end
+    end
+
+    context "without headers" do
+      let(:log) { build(:sales_log, :completed) }
+      let(:file) { Tempfile.new }
+      let(:path) { file.path }
+
+      before do
+        file.write(BulkUpload::LogToCsv.new(log:, line_ending: "\r\n", col_offset: 0).to_2022_sales_csv_row)
+        file.close
+      end
+
+      it "creates validation errors" do
+        expect { validator.call }.to change(BulkUploadError, :count)
+      end
+    end
+  end
+
+  describe "#create_logs?" do
+    context "when all logs are valid" do
+      let(:target_path) { file_fixture("completed_2022_23_sales_bulk_upload.csv") }
+
+      before do
+        target_array = File.open(target_path).readlines
+        target_array[0..118].each do |line|
+          file.write line
+        end
+        file.rewind
+      end
 
-    it "persists bulk upload errors" do
-      expect {
+      it "returns truthy" do
         validator.call
-      }.to change(BulkUploadError, :count).by(1)
+        expect(validator).to be_create_logs
+      end
     end
 
-    it "populates purchaser_code" do
-      validator.call
+    context "when there is an invalid log" do
+      let(:path) { file_fixture("2022_23_sales_bulk_upload.csv") }
 
-      error = BulkUploadError.last
-      expect(error.purchaser_code).to eql("1")
+      it "returns falsey" do
+        validator.call
+        expect(validator).not_to be_create_logs
+      end
+    end
+
+    context "when a log is not valid?" do
+      let(:log_1) { build(:sales_log, :completed, created_by: user) }
+      let(:log_2) { build(:sales_log, :completed, created_by: user) }
+
+      before do
+        file.write(BulkUpload::LogToCsv.new(log: log_1, line_ending: "\r\n", col_offset: 0).to_2022_sales_csv_row)
+        file.write(BulkUpload::LogToCsv.new(log: log_2, line_ending: "\r\n", col_offset: 0, overrides: { organisation_id: "random" }).to_2022_sales_csv_row)
+        file.close
+      end
+
+      it "returns false" do
+        validator.call
+        expect(validator).not_to be_create_logs
+      end
+    end
+
+    context "when all logs valid?" do
+      let(:log_1) { build(:sales_log, :completed, created_by: user) }
+      let(:log_2) { build(:sales_log, :completed, created_by: user) }
+
+      before do
+        file.write(BulkUpload::LogToCsv.new(log: log_1, line_ending: "\r\n", col_offset: 0).to_2022_sales_csv_row)
+        file.write(BulkUpload::LogToCsv.new(log: log_2, line_ending: "\r\n", col_offset: 0).to_2022_sales_csv_row)
+        file.close
+      end
+
+      it "returns true" do
+        validator.call
+        expect(validator).to be_create_logs
+      end
+    end
+
+    context "when a single log wants to block log creation" do
+      let(:unaffiliated_org) { create(:organisation) }
+
+      let(:log_1) { build(:sales_log, :completed, created_by: user, owning_organisation: unaffiliated_org) }
+
+      before do
+        file.write(BulkUpload::LogToCsv.new(log: log_1, line_ending: "\r\n", col_offset: 0).to_2022_sales_csv_row)
+        file.close
+      end
+
+      it "will not create logs" do
+        validator.call
+        expect(validator).not_to be_create_logs
+      end
+    end
+
+    context "when a log has incomplete setup secion" do
+      let(:log) { build(:sales_log, created_by: user, saledate: Time.zone.local(2022, 5, 1)) }
+
+      before do
+        file.write(BulkUpload::LogToCsv.new(log:, line_ending: "\r\n", col_offset: 0).to_2022_sales_csv_row)
+        file.close
+      end
+
+      it "returns false" do
+        validator.call
+        expect(validator).not_to be_create_logs
+      end
+    end
+
+    context "when a column has error rate below absolute threshold" do
+      context "when a column is over 60% error threshold" do
+        let(:log_1) { build(:sales_log, :completed, created_by: user) }
+        let(:log_2) { build(:sales_log, :in_progress, created_by: user, saledate: Time.zone.local(2022, 5, 1)) }
+        let(:log_3) { build(:sales_log, :in_progress, created_by: user, saledate: Time.zone.local(2022, 5, 1)) }
+        let(:log_4) { build(:sales_log, :in_progress, created_by: user, saledate: Time.zone.local(2022, 5, 1)) }
+        let(:log_5) { build(:sales_log, :in_progress, created_by: user, saledate: Time.zone.local(2022, 5, 1)) }
+
+        before do
+          file.write(BulkUpload::LogToCsv.new(log: log_1, line_ending: "\r\n", col_offset: 0).to_2022_sales_csv_row)
+          file.write(BulkUpload::LogToCsv.new(log: log_2, line_ending: "\r\n", col_offset: 0).to_2022_sales_csv_row)
+          file.write(BulkUpload::LogToCsv.new(log: log_3, line_ending: "\r\n", col_offset: 0).to_2022_sales_csv_row)
+          file.write(BulkUpload::LogToCsv.new(log: log_4, line_ending: "\r\n", col_offset: 0).to_2022_sales_csv_row)
+          file.write(BulkUpload::LogToCsv.new(log: log_5, line_ending: "\r\n", col_offset: 0).to_2022_sales_csv_row)
+          file.close
+        end
+
+        it "returns true" do
+          validator.call
+          expect(validator).to be_create_logs
+        end
+      end
+
+      context "when a column is under 60% error threshold" do
+        let(:log_1) { build(:sales_log, :completed, created_by: user) }
+        let(:log_2) { build(:sales_log, :completed, created_by: user) }
+        let(:log_3) { build(:sales_log, :in_progress, created_by: user, saledate: Time.zone.local(2022, 5, 1)) }
+        let(:log_4) { build(:sales_log, :in_progress, created_by: user, saledate: Time.zone.local(2022, 5, 1)) }
+        let(:log_5) { build(:sales_log, :in_progress, created_by: user, saledate: Time.zone.local(2022, 5, 1)) }
+
+        before do
+          file.write(BulkUpload::LogToCsv.new(log: log_1, line_ending: "\r\n", col_offset: 0).to_2022_sales_csv_row)
+          file.write(BulkUpload::LogToCsv.new(log: log_2, line_ending: "\r\n", col_offset: 0).to_2022_sales_csv_row)
+          file.write(BulkUpload::LogToCsv.new(log: log_3, line_ending: "\r\n", col_offset: 0).to_2022_sales_csv_row)
+          file.write(BulkUpload::LogToCsv.new(log: log_4, line_ending: "\r\n", col_offset: 0).to_2022_sales_csv_row)
+          file.write(BulkUpload::LogToCsv.new(log: log_5, line_ending: "\r\n", col_offset: 0).to_2022_sales_csv_row)
+          file.close
+        end
+
+        it "returns true" do
+          validator.call
+          expect(validator).to be_create_logs
+        end
+      end
     end
   end
 end

spec/services/bulk_upload/sales/year2022/csv_parser_spec.rb

@@ -0,0 +1,97 @@
+require "rails_helper"
+
+RSpec.describe BulkUpload::Sales::Year2022::CsvParser do
+  subject(:service) { described_class.new(path:) }
+
+  let(:path) { file_fixture("completed_2022_23_sales_bulk_upload.csv") }
+
+  context "when parsing csv with headers" do
+    it "returns correct offsets" do
+      expect(service.row_offset).to eq(5)
+      expect(service.col_offset).to eq(1)
+    end
+
+    it "parses csv correctly" do
+      expect(service.row_parsers[0].field_7.to_i).to eq(32)
+    end
+  end
+
+  context "when parsing csv without headers" do
+    let(:file) { Tempfile.new }
+    let(:path) { file.path }
+    let(:log) { build(:sales_log, :completed) }
+
+    before do
+      file.write(BulkUpload::LogToCsv.new(log:, col_offset: 0).to_2022_sales_csv_row)
+      file.rewind
+    end
+
+    it "returns correct offsets" do
+      expect(service.row_offset).to eq(0)
+      expect(service.col_offset).to eq(0)
+    end
+
+    it "parses csv correctly" do
+      expect(service.row_parsers[0].field_7.to_i).to eql(log.age1)
+    end
+  end
+
+  context "when parsing with BOM aka byte order mark" do
+    let(:file) { Tempfile.new }
+    let(:path) { file.path }
+    let(:log) { build(:sales_log, :completed) }
+    let(:bom) { "\uFEFF" }
+
+    before do
+      file.write(bom)
+      file.write(BulkUpload::LogToCsv.new(log:, col_offset: 0).to_2022_sales_csv_row)
+      file.close
+    end
+
+    it "parses csv correctly" do
+      expect(service.row_parsers[0].field_7.to_i).to eql(log.age1)
+    end
+  end
+
+  context "when an invalid byte sequence" do
+    let(:file) { Tempfile.new }
+    let(:path) { file.path }
+    let(:log) { build(:sales_log, :completed) }
+    let(:invalid_sequence) { "\x81" }
+
+    before do
+      file.write(invalid_sequence)
+      file.write(BulkUpload::LogToCsv.new(log:, col_offset: 0).to_2022_sales_csv_row)
+      file.close
+    end
+
+    it "parses csv correctly" do
+      expect(service.row_parsers[0].field_7.to_i).to eql(log.age1)
+    end
+  end
+
+  describe "#column_for_field", aggregate_failures: true do
+    context "when headers present" do
+      it "returns correct column" do
+        expect(service.column_for_field("field_1")).to eql("B")
+        expect(service.column_for_field("field_125")).to eql("DV")
+      end
+    end
+
+    context "when no headers" do
+      let(:file) { Tempfile.new }
+      let(:path) { file.path }
+      let(:log) { build(:sales_log, :completed) }
+
+      before do
+        file.write(BulkUpload::LogToCsv.new(log:, col_offset: 0).to_2022_sales_csv_row)
+        file.rewind
+      end
+
+      it "returns correct column" do
+        expect(service.column_for_field("field_1")).to eql("A")
+        expect(service.column_for_field("field_125")).to eql("DU")
+      end
+    end
+  end
+end

spec/services/bulk_upload/sales/year2022/row_parser_spec.rb

@@ -3,19 +3,522 @@ require "rails_helper"
 RSpec.describe BulkUpload::Sales::Year2022::RowParser do
   subject(:parser) { described_class.new(attributes) }
 
+  let(:now) { Time.zone.parse("01/03/2023") }
+
+  let(:attributes) { { bulk_upload: } }
+  let(:bulk_upload) { create(:bulk_upload, :sales, user:) }
+  let(:user) { create(:user, organisation: owning_org) }
+
+  let(:owning_org) { create(:organisation, :with_old_visible_id) }
+
+  let(:setup_section_params) do
+    {
+      bulk_upload:,
+      field_1: "test id", # purchase id
+      field_92: owning_org.old_visible_id, # organisation
+      field_93: user.email, # user
+      field_2: now.day.to_s, # sale day
+      field_3: now.month.to_s, # sale month
+      field_4: now.strftime("%g"), # sale year
+      field_113: "1", # owhershipsch
+      field_57: "2", # shared ownership sale type
+      field_116: "2", # joint purchase
+      field_115: "1", # will the buyers live in the property
+    }
+  end
+
+  let(:valid_attributes) do
+    {
+      bulk_upload:,
+      field_1: "test id",
+      field_2: "22",
+      field_3: "2",
+      field_4: "23",
+      field_6: "1",
+      field_7: "32",
+      field_8: "32",
+      field_13: "M",
+      field_14: "F",
+      field_19: "R",
+      field_24: "1",
+      field_25: "2",
+      field_30: "12",
+      field_31: "18",
+      field_32: "30000",
+      field_33: "15000",
+      field_34: "1",
+      field_35: "1",
+      field_36: "20000",
+      field_37: "3",
+      field_39: "1",
+      field_40: "E09000008",
+      field_41: "A1",
+      field_42: "1AA",
+      field_43: "1",
+      field_45: "1",
+      field_46: "1",
+      field_48: "3",
+      field_49: "3",
+      field_50: "2",
+      field_51: "1",
+      field_52: "1",
+      field_53: "E09000008",
+      field_54: "CR0",
+      field_55: "4BB",
+      field_56: "3",
+      field_57: "2",
+      field_58: "2",
+      field_59: "23",
+      field_60: "3",
+      field_61: "22",
+      field_62: "30",
+      field_63: "3",
+      field_64: "22",
+      field_65: "3",
+      field_66: "1",
+      field_67: "1",
+      field_68: "250000",
+      field_69: "25",
+      field_70: "42500",
+      field_71: "3",
+      field_72: "20000",
+      field_74: "800",
+      field_75: "200",
+      field_92: owning_org.old_visible_id,
+      field_95: "3",
+      field_97: "5",
+      field_98: "1",
+      field_104: "4",
+      field_105: "20",
+      field_109: "2",
+      field_110: "5",
+      field_111: "1",
+      field_112: "1",
+      field_113: "1",
+      field_115: "1",
+      field_116: "1",
+      field_117: "1",
+      field_118: "1",
+      field_119: "0",
+      field_120: "10",
+      field_121: "10",
+      field_122: "1",
+      field_123: "1",
+    }
+  end
+
+  around do |example|
+    FormHandler.instance.use_real_forms!
+
+    example.run
+
+    FormHandler.instance.use_fake_forms!
+  end
+
+  describe "#blank_row?" do
+    context "when a new object" do
+      it "returns true" do
+        expect(parser).to be_blank_row
+      end
+    end
+
+    context "when any field is populated" do
+      before do
+        parser.field_1 = "1"
+      end
+
+      it "returns false" do
+        expect(parser).not_to be_blank_row
+      end
+    end
+  end
+
   describe "validations" do
     before do
+      stub_request(:get, /api.postcodes.io/)
+      .to_return(status: 200, body: "{\"status\":200,\"result\":{\"admin_district\":\"Manchester\", \"codes\":{\"admin_district\": \"E08000003\"}}}", headers: {})
+
       parser.valid?
     end
 
+    describe "#valid?" do
+      context "when the row is blank" do
+        let(:attributes) { { bulk_upload: } }
+
+        it "returns true" do
+          expect(parser).to be_valid
+        end
+      end
+
+      context "when calling the method multiple times" do
+        let(:attributes) { { bulk_upload:, field_7: 2 } }
+
+        it "does not add keep adding errors to the pile" do
+          expect { parser.valid? }.not_to change(parser.errors, :count)
+        end
+      end
+
+      context "when valid row" do
+        let(:attributes) { valid_attributes }
+
+        it "returns true" do
+          expect(parser).to be_valid
+        end
+
+        it "instantiates a log with everything completed", aggregate_failures: true do
+          questions = parser.send(:questions).reject do |q|
+            parser.send(:log).optional_fields.include?(q.id) || q.completed?(parser.send(:log))
+          end
+
+          expect(questions.map(&:id).size).to eq(0)
+          expect(questions.map(&:id)).to eql([])
+        end
+      end
+    end
+
+    context "when setup section not complete and type is not given" do
+      let(:attributes) do
+        {
+          bulk_upload:,
+          field_1: "test id",
+        }
+      end
+
+      it "has errors on correct setup fields" do
+        errors = parser.errors.select { |e| e.options[:category] == :setup }.map(&:attribute)
+
+        expect(errors).to eql(%i[field_2 field_3 field_4 field_113 field_92])
+      end
+    end
+
+    context "when setup section not complete and type is shared ownership" do
+      let(:attributes) do
+        {
+          bulk_upload:,
+          field_1: "test id",
+          field_113: "1",
+        }
+      end
+
+      it "has errors on correct setup fields" do
+        errors = parser.errors.select { |e| e.options[:category] == :setup }.map(&:attribute)
+
+        expect(errors).to eql(%i[field_2 field_3 field_4 field_57 field_116 field_92])
+      end
+    end
+
+    context "when setup section not complete it's shared ownership joint purchase" do
+      let(:attributes) do
+        {
+          bulk_upload:,
+          field_1: "test id",
+          field_113: "1",
+          field_57: "2",
+          field_116: "1",
+        }
+      end
+
+      it "has errors on correct setup fields" do
+        errors = parser.errors.select { |e| e.options[:category] == :setup }.map(&:attribute)
+
+        expect(errors).to eql(%i[field_2 field_3 field_4 field_109 field_92])
+      end
+    end
+
+    context "when setup section not complete and type is discounted ownership" do
+      let(:attributes) do
+        {
+          bulk_upload:,
+          field_1: "test id",
+          field_113: "2",
+        }
+      end
+
+      it "has errors on correct setup fields" do
+        errors = parser.errors.select { |e| e.options[:category] == :setup }.map(&:attribute)
+
+        expect(errors).to eql(%i[field_2 field_3 field_4 field_76 field_116 field_92])
+      end
+    end
+
+    context "when setup section not complete it's discounted ownership joint purchase" do
+      let(:attributes) do
+        {
+          bulk_upload:,
+          field_1: "test id",
+          field_113: "2",
+          field_76: "8",
+          field_116: "1",
+        }
+      end
+
+      it "has errors on correct setup fields" do
+        errors = parser.errors.select { |e| e.options[:category] == :setup }.map(&:attribute)
+
+        expect(errors).to eql(%i[field_2 field_3 field_4 field_109 field_92])
+      end
+    end
+
+    context "when setup section not complete and type is outright sale" do
+      let(:attributes) do
+        {
+          bulk_upload:,
+          field_1: "test id",
+          field_113: "3",
+        }
+      end
+
+      it "has errors on correct setup fields" do
+        errors = parser.errors.select { |e| e.options[:category] == :setup }.map(&:attribute)
+
+        expect(errors).to eql(%i[field_2 field_3 field_4 field_84 field_114 field_92])
+      end
+    end
+
+    context "when setup section not complete outright sale buyer is not company" do
+      let(:attributes) do
+        {
+          bulk_upload:,
+          field_1: "test id",
+          field_113: "3",
+          field_84: "12",
+          field_114: "2",
+        }
+      end
+
+      it "has errors on correct setup fields" do
+        errors = parser.errors.select { |e| e.options[:category] == :setup }.map(&:attribute)
+
+        expect(errors).to eql(%i[field_2 field_3 field_4 field_115 field_116 field_92])
+      end
+    end
+
+    describe "#field_92" do # owning org
+      context "when no data given" do
+        let(:attributes) { { bulk_upload:, field_92: "" } }
+
+        it "is not permitted as setup error" do
+          setup_errors = parser.errors.select { |e| e.options[:category] == :setup }
+
+          expect(setup_errors.find { |e| e.attribute == :field_92 }.message).to eql("The owning organisation code is incorrect")
+        end
+
+        it "blocks log creation" do
+          expect(parser).to be_block_log_creation
+        end
+      end
+
+      context "when cannot find owning org" do
+        let(:attributes) { { bulk_upload:, field_92: "donotexist" } }
+
+        it "is not permitted as a setup error" do
+          setup_errors = parser.errors.select { |e| e.options[:category] == :setup }
+
+          expect(setup_errors.find { |e| e.attribute == :field_92 }.message).to eql("The owning organisation code is incorrect")
+        end
+
+        it "blocks log creation" do
+          expect(parser).to be_block_log_creation
+        end
+      end
+
+      context "when not affiliated with owning org" do
+        let(:unaffiliated_org) { create(:organisation, :with_old_visible_id) }
+
+        let(:attributes) { { bulk_upload:, field_92: unaffiliated_org.old_visible_id } }
+
+        it "is not permitted as setup error" do
+          setup_errors = parser.errors.select { |e| e.options[:category] == :setup }
+
+          expect(setup_errors.find { |e| e.attribute == :field_92 }.message).to eql("You do not have permission to add logs for this owning organisation")
+        end
+
+        it "blocks log creation" do
+          expect(parser).to be_block_log_creation
+        end
+      end
+    end
+
+    describe "#field_93" do # username for created_by
+      context "when blank" do
+        let(:attributes) { { bulk_upload:, field_93: "" } }
+
+        it "is permitted" do
+          expect(parser.errors[:field_93]).to be_blank
+        end
+      end
+
+      context "when user could not be found" do
+        let(:attributes) { { bulk_upload:, field_93: "idonotexist@example.com" } }
+
+        it "is not permitted" do
+          expect(parser.errors[:field_93]).to be_present
+        end
+      end
+
+      context "when an unaffiliated user" do
+        let(:other_user) { create(:user) }
+
+        let(:attributes) { { bulk_upload:, field_92: owning_org.old_visible_id, field_93: other_user.email } }
+
+        it "is not permitted" do
+          expect(parser.errors[:field_93]).to be_present
+        end
+
+        it "blocks log creation" do
+          expect(parser).to be_block_log_creation
+        end
+      end
+
+      context "when an user part of owning org" do
+        let(:other_user) { create(:user, organisation: owning_org) }
+
+        let(:attributes) { { bulk_upload:, field_92: owning_org.old_visible_id, field_93: other_user.email } }
+
+        it "is permitted" do
+          expect(parser.errors[:field_93]).to be_blank
+        end
+      end
+    end
+
+    [
+      %w[age1_known age1 field_7],
+      %w[age2_known age2 field_8],
+      %w[age3_known age3 field_9],
+      %w[age4_known age4 field_10],
+      %w[age5_known age5 field_11],
+      %w[age6_known age6 field_12],
+    ].each do |known, age, field|
+      describe "##{known} and ##{age}" do
+        context "when #{field} is blank" do
+          let(:attributes) { { bulk_upload:, field.to_s => nil } }
+
+          it "sets ##{known} 1" do
+            expect(parser.log.public_send(known)).to be(1)
+          end
+
+          it "sets ##{age} to nil" do
+            expect(parser.log.public_send(age)).to be_nil
+          end
+        end
+
+        context "when #{field} is R" do
+          let(:attributes) { setup_section_params.merge({ field.to_s => "R", field_6: "1", field_119: "5", field_112: "1" }) }
+
+          it "sets ##{known} 1" do
+            expect(parser.log.public_send(known)).to be(1)
+          end
+
+          it "sets ##{age} to nil" do
+            expect(parser.log.public_send(age)).to be_nil
+          end
+        end
+
+        context "when #{field} is a number" do
+          let(:attributes) { setup_section_params.merge({ field.to_s => "50", field_6: "1", field_119: "5", field_112: "1" }) }
+
+          it "sets ##{known} to 0" do
+            expect(parser.log.public_send(known)).to be(0)
+          end
+
+          it "sets ##{age} to given age" do
+            expect(parser.log.public_send(age)).to be(50)
+          end
+        end
+
+        context "when #{field} is a non-sensical value" do
+          let(:attributes) { setup_section_params.merge({ field.to_s => "A", field_6: "1", field_119: "5", field_112: "1" }) }
+
+          it "sets ##{known} to 0" do
+            expect(parser.log.public_send(known)).to be(0)
+          end
+
+          it "sets ##{age} to nil" do
+            expect(parser.log.public_send(age)).to be_nil
+          end
+        end
+      end
+    end
+
     describe "#field_117" do
       context "when not a possible value" do
-        let(:attributes) { { field_117: "3" } }
+        let(:attributes) { valid_attributes.merge({ field_117: "3" }) }
 
         it "is not valid" do
           expect(parser.errors).to include(:field_117)
         end
       end
     end
+
+    describe "fields 2, 3, 4 => saledate" do
+      context "when all of these fields are blank" do
+        let(:attributes) { setup_section_params.merge({ field_2: nil, field_3: nil, field_4: nil }) }
+
+        it "returns them as setup errors" do
+          setup_errors = parser.errors.select { |e| e.options[:category] == :setup }
+
+          expect(setup_errors.find { |e| e.attribute == :field_2 }).to be_present
+          expect(setup_errors.find { |e| e.attribute == :field_3 }).to be_present
+          expect(setup_errors.find { |e| e.attribute == :field_4 }).to be_present
+        end
+      end
+
+      context "when one of these fields is blank" do
+        let(:attributes) { setup_section_params.merge({ field_2: "1", field_3: "1", field_4: nil }) }
+
+        it "returns an error only on blank field" do
+          expect(parser.errors[:field_2]).to be_blank
+          expect(parser.errors[:field_3]).to be_blank
+          expect(parser.errors[:field_4]).to be_present
+        end
+      end
+
+      context "when field 4 is 4 digits instead of 2" do
+        let(:attributes) { setup_section_params.merge({ bulk_upload:, field_4: "2022" }) }
+
+        it "returns an error" do
+          expect(parser.errors[:field_4]).to include("Sale completion year must be 2 digits")
+        end
+      end
+
+      context "when invalid date given" do
+        let(:attributes) { setup_section_params.merge({ field_2: "a", field_3: "12", field_4: "2022" }) }
+
+        it "does not raise an error" do
+          expect { parser.valid? }.not_to raise_error
+        end
+      end
+
+      context "when inside of collection year" do
+        let(:attributes) { setup_section_params.merge({ field_2: "1", field_3: "10", field_4: "22" }) }
+
+        let(:bulk_upload) { create(:bulk_upload, :sales, user:, year: 2022) }
+
+        it "does not return errors" do
+          expect(parser.errors[:field_2]).not_to be_present
+          expect(parser.errors[:field_3]).not_to be_present
+          expect(parser.errors[:field_4]).not_to be_present
+        end
+      end
+
+      context "when outside of collection year" do
+        around do |example|
+          Timecop.freeze(Date.new(2022, 4, 2)) do
+            example.run
+          end
+          Timecop.return
+        end
+
+        let(:attributes) { setup_section_params.merge({ field_2: "1", field_3: "1", field_4: "22" }) }
+
+        let(:bulk_upload) { create(:bulk_upload, :sales, user:, year: 2022) }
+
+        it "returns errors" do
+          expect(parser.errors[:field_2]).to be_present
+          expect(parser.errors[:field_3]).to be_present
+          expect(parser.errors[:field_4]).to be_present
+        end
+      end
+    end
   end
 end

spec/support/bulk_upload/log_to_csv.rb

@@ -16,6 +16,10 @@ class BulkUpload::LogToCsv
     (row_prefix + to_2022_row).flatten.join(",") + line_ending
   end
 
+  def to_2022_sales_csv_row
+    (row_prefix + to_2022_sales_row).flatten.join(",") + line_ending
+  end
+
   def to_2023_csv_row(seed: nil)
     if seed
       row = to_2023_row.shuffle(random: Random.new(seed))
@@ -198,6 +202,153 @@ class BulkUpload::LogToCsv
     ]
   end
 
+  def to_2022_sales_row
+    [
+      log.purchid, # 1
+      log.saledate&.day,
+      log.saledate&.month,
+      log.saledate&.strftime("%y"),
+      nil,
+      log.noint,
+      log.age1,
+      log.age2,
+      log.age3,
+      log.age4,
+      log.age5,
+      log.age6,
+
+      log.sex1,
+      log.sex2,
+      log.sex3,
+      log.sex4,
+      log.sex5,
+      log.sex6,
+
+      log.relat2,
+      log.relat3, # 20
+      log.relat4,
+      log.relat5,
+      log.relat6,
+
+      log.ecstat1,
+      log.ecstat2,
+      log.ecstat3,
+      log.ecstat4,
+      log.ecstat5,
+      log.ecstat6,
+
+      log.ethnic, # 30
+      log.national,
+      log.income1,
+      log.income2,
+      log.inc1mort,
+      log.inc2mort,
+      log.savings,
+      log.prevown,
+      nil,
+
+      log.prevten,
+      log.prevloc, # 40
+      ((log.ppostcode_full || "").split(" ") || [""]).first,
+      ((log.ppostcode_full || "").split(" ") || [""]).last,
+      log.ppcodenk == 0 ? 1 : nil,
+
+      log.pregyrha,
+      log.pregla,
+      log.pregghb,
+      log.pregother,
+
+      log.disabled,
+      log.wheel,
+      log.beds, # 50
+      log.proptype,
+      log.builtype,
+      log.la,
+      ((log.postcode_full || "").split(" ") || [""]).first,
+      ((log.postcode_full || "").split(" ") || [""]).last,
+      log.wchair,
+
+      log.type, # shared ownership
+      log.resale,
+      log.hodate&.day,
+      log.hodate&.month, # 60
+      log.hodate&.strftime("%y"),
+      log.exdate&.day,
+      log.exdate&.month,
+      log.exdate&.strftime("%y"),
+      log.lanomagr,
+
+      log.frombeds,
+      log.fromprop,
+
+      log.value,
+      log.equity,
+      log.mortgage, # 70
+      log.extrabor,
+      log.deposit,
+      log.cashdis,
+
+      log.mrent,
+      log.mscharge,
+
+      log.type, # discounted ownership
+      log.value,
+      log.grant,
+      log.discount,
+      log.mortgage, # 80
+      log.extrabor,
+      log.deposit,
+      log.mscharge,
+
+      log.type, # outright sale
+      log.othtype,
+      nil,
+
+      log.value,
+      log.mortgage,
+      log.extrabor,
+      log.deposit, # 90
+      log.mscharge,
+
+      overrides[:organisation_id] || log.owning_organisation&.old_visible_id,
+      log.created_by&.email,
+      nil,
+      hhregres,
+      nil,
+      log.armedforcesspouse,
+      log.mortgagelender, # shared ownership
+      log.mortgagelenderother,
+      log.mortgagelender, # discounted ownership 100
+      log.mortgagelenderother,
+      log.mortgagelender, # outright ownership
+      log.mortgagelenderother,
+
+      log.hb,
+      log.mortlen, # shared ownership
+      log.mortlen, # discounted ownership
+      log.mortlen, # outright ownership
+
+      log.proplen, # discounted ownership
+      log.jointmore,
+      log.proplen, # shared ownership 110
+      log.staircase,
+      log.privacynotice,
+      log.ownershipsch,
+      log.companybuy, # outright sale
+      log.buylivein,
+      log.jointpur,
+      log.buy1livein,
+      log.buy2livein,
+      log.hholdcount,
+      log.stairbought, # 120
+      log.stairowned,
+      log.socprevten,
+      log.mortgageused, # shared ownership
+      log.mortgageused, # discounted ownership
+      log.mortgageused, # outright ownership
+    ]
+  end
+
 private
 
   def renewal
@@ -279,4 +430,12 @@ private
       1
     end
   end
+
+  def hhregres
+    if log.hhregres == 1
+      log.hhregresstill
+    else
+      log.hhregres
+    end
+  end
 end