Submit social housing lettings and sales data (CORE)
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

208 lines
5.2 KiB

class BulkUpload::Lettings::Validator
include ActiveModel::Validations
include Rails.application.routes.url_helpers
attr_reader :bulk_upload, :path
validate :validate_file_not_empty
2 years ago
validate :validate_field_numbers_count
validate :validate_missing_required_headers
2 years ago
validate :validate_max_columns_count_if_no_headers
validate :validate_correct_template
def initialize(bulk_upload:, path:)
@bulk_upload = bulk_upload
@path = path
end
def call
row_parsers.each(&:valid?)
validate_duplicate_rows if FeatureToggle.bulk_upload_duplicate_log_check_enabled?
row_parsers.each_with_index do |row_parser, index|
row = index + row_offset + 1
row_parser.errors.each do |error|
col = csv_parser.column_for_field(error.attribute.to_s)
bulk_upload.bulk_upload_errors.create!(
field: error.attribute,
error: error.message,
tenant_code: row_parser.tenant_code,
property_ref: row_parser.property_ref,
row:,
cell: "#{col}#{row}",
col:,
category: error.options[:category],
)
end
end
end
def create_logs?
return false if any_setup_errors?
if row_parsers.any?(&:block_log_creation?)
Sentry.capture_message("Bulk upload log creation blocked: #{bulk_upload.id}.")
return false
end
if any_logs_already_exist? && FeatureToggle.bulk_upload_duplicate_log_check_enabled?
Sentry.capture_message("Bulk upload log creation blocked due to duplicate logs: #{bulk_upload.id}.")
return false
end
row_parsers.each do |row_parser|
row_parser.log.blank_invalid_non_setup_fields!
end
if any_logs_invalid?
Sentry.capture_message("Bulk upload log creation blocked due to invalid logs after blanking non setup fields: #{bulk_upload.id}.")
return false
end
true
end
def self.question_for_field(field)
QUESTIONS[field]
end
def any_setup_errors?
bulk_upload
.bulk_upload_errors
.where(category: "setup")
.count
.positive?
end
def soft_validation_errors_only?
errors = bulk_upload.bulk_upload_errors
errors.count == errors.where(category: "soft_validation").count && errors.count.positive?
end
def any_logs_already_exist?
row_parsers.any?(&:log_already_exists?)
end
def total_logs_count
csv_parser.body_rows.count
end
private
# n^2 algo
def validate_duplicate_rows
row_parsers.each do |rp|
dupe = row_parsers.reject { |r| r.object_id.equal?(rp.object_id) }.any? do |rp_counter|
rp.spreadsheet_duplicate_hash == rp_counter.spreadsheet_duplicate_hash
end
if dupe
rp.add_duplicate_found_in_spreadsheet_errors
end
end
end
def any_logs_invalid?
row_parsers.any? { |row_parser| row_parser.log.invalid? }
end
def csv_parser
@csv_parser ||= case bulk_upload.year
when 2023
BulkUpload::Lettings::Year2023::CsvParser.new(path:)
when 2024
BulkUpload::Lettings::Year2024::CsvParser.new(path:)
else
raise "csv parser not found"
end
end
CLDC-1779 Bulk upload lettings validation (#1148) * able to view lettings bulk upload errors * fix linting * call service correctly in test * add bulk upload sales questions mapping * appease linter * bulk upload error shows correct question - depending on log type it will show relevant question for the field concerned * improve namespacing of classes * add job to process bulk uploads * move validation from parser to model * add validations for field_1 * add validation for field_4 * pending tests for field_4 * convert field_mapping to array of hashes * validate nulls based on form question * actually load forms when toggling between forms * validate null for startdate * row parser has access to bulk upload * csv upload validates first form section * add postcode validation * Refactor error mappings for row parser * Add unittype question * Fix null error setting and add builtype * add wchair to bulk upload * Add beds to bulk upload * Add joint to bulk upload * Add startertenancy to the bulk upload * Add tenancy for bulk upload * Add declaration to the bulk upload * Add age1 and age1_known to bulk upload * add ages to bulk upload * add sex1 to bulk upload * add ethnic_group and ethnic to bulk upload * add national to bulk upload * add ecstat1 to bulk upload * add military related fields to bulk upload * add preg_occ to bulk upload * add housingneeds to bulk upload * add illness to bulk upload * add layear to bulk upload * add waityear to bulk upload * add reason to bulk upload * add prevten to bulk upload * add homeless to bulk upload * add previous postcode to bulk upload * add reasonable preferences to bulk upload * add allocations system to bulk upload * add referral to bulk upload * add net_income_known to bulk upload * add hb to bulk upload * add benefits to bulk upload * add rent fields to bulk upload * add hhmemb to bulk upload * use 2022 csv fixtures for bulk upload * fix renewal mapping for bulk upload * placeholder test for bulk upload validation * fix bulk upload mapping for homeless field * fix leftreg mapping for bulk upload * fix user associations in bulk upload tests * add gender fields for bulk upload * add ecstatN fields to bulk upload * add #relatN fields to bulk upload * extract old_visible_id in factory to trait * map net_income_known correctly for bulk upload * fix income bugs for bulk upload * add unitletas to bulk upload * add #rsnvac to bulk upload * add #sheltered to bulk upload * add illness fields to bulk upload * add #irproduct_other to bulk upload * infer renewal from rsnvac for bulk upload * add #tenancyother to bulk upload * add #tenancylength to bulk upload * bulk upload earnings accepts pennies but rounds * add #reasonother to bulk upload * fix mapping of #ppcodenk for bulk upload * add #household_charge to bulk upload * add #chcharge to bulk upload * add #tcharge to bulk upload * add #supcharg to bulk upload * add pscharge to bulk upload * add #scharge to bulk upload * use case statement for bulk upload allocation * add offered to bulk upload * add propcode to bulk upload * add major repair fields to bulk upload * add #voiddate to bulk upload * support YY year format for bulk upload * test postcode strips whitespace for bulk upload * add #la to bulk upload * add previous la to bulk upload * fix failing test * remove duplicate line from rebase * add first time social housing to bulk upload * make methods private * fix field_4 validation for bulk upload - the null check was inverted by mistake Co-authored-by: Kat <katrina@kosiak.co.uk>
2 years ago
def row_offset
csv_parser.row_offset
CLDC-1779 Bulk upload lettings validation (#1148) * able to view lettings bulk upload errors * fix linting * call service correctly in test * add bulk upload sales questions mapping * appease linter * bulk upload error shows correct question - depending on log type it will show relevant question for the field concerned * improve namespacing of classes * add job to process bulk uploads * move validation from parser to model * add validations for field_1 * add validation for field_4 * pending tests for field_4 * convert field_mapping to array of hashes * validate nulls based on form question * actually load forms when toggling between forms * validate null for startdate * row parser has access to bulk upload * csv upload validates first form section * add postcode validation * Refactor error mappings for row parser * Add unittype question * Fix null error setting and add builtype * add wchair to bulk upload * Add beds to bulk upload * Add joint to bulk upload * Add startertenancy to the bulk upload * Add tenancy for bulk upload * Add declaration to the bulk upload * Add age1 and age1_known to bulk upload * add ages to bulk upload * add sex1 to bulk upload * add ethnic_group and ethnic to bulk upload * add national to bulk upload * add ecstat1 to bulk upload * add military related fields to bulk upload * add preg_occ to bulk upload * add housingneeds to bulk upload * add illness to bulk upload * add layear to bulk upload * add waityear to bulk upload * add reason to bulk upload * add prevten to bulk upload * add homeless to bulk upload * add previous postcode to bulk upload * add reasonable preferences to bulk upload * add allocations system to bulk upload * add referral to bulk upload * add net_income_known to bulk upload * add hb to bulk upload * add benefits to bulk upload * add rent fields to bulk upload * add hhmemb to bulk upload * use 2022 csv fixtures for bulk upload * fix renewal mapping for bulk upload * placeholder test for bulk upload validation * fix bulk upload mapping for homeless field * fix leftreg mapping for bulk upload * fix user associations in bulk upload tests * add gender fields for bulk upload * add ecstatN fields to bulk upload * add #relatN fields to bulk upload * extract old_visible_id in factory to trait * map net_income_known correctly for bulk upload * fix income bugs for bulk upload * add unitletas to bulk upload * add #rsnvac to bulk upload * add #sheltered to bulk upload * add illness fields to bulk upload * add #irproduct_other to bulk upload * infer renewal from rsnvac for bulk upload * add #tenancyother to bulk upload * add #tenancylength to bulk upload * bulk upload earnings accepts pennies but rounds * add #reasonother to bulk upload * fix mapping of #ppcodenk for bulk upload * add #household_charge to bulk upload * add #chcharge to bulk upload * add #tcharge to bulk upload * add #supcharg to bulk upload * add pscharge to bulk upload * add #scharge to bulk upload * use case statement for bulk upload allocation * add offered to bulk upload * add propcode to bulk upload * add major repair fields to bulk upload * add #voiddate to bulk upload * support YY year format for bulk upload * test postcode strips whitespace for bulk upload * add #la to bulk upload * add previous la to bulk upload * fix failing test * remove duplicate line from rebase * add first time social housing to bulk upload * make methods private * fix field_4 validation for bulk upload - the null check was inverted by mistake Co-authored-by: Kat <katrina@kosiak.co.uk>
2 years ago
end
def col_offset
csv_parser.col_offset
CLDC-1779 Bulk upload lettings validation (#1148) * able to view lettings bulk upload errors * fix linting * call service correctly in test * add bulk upload sales questions mapping * appease linter * bulk upload error shows correct question - depending on log type it will show relevant question for the field concerned * improve namespacing of classes * add job to process bulk uploads * move validation from parser to model * add validations for field_1 * add validation for field_4 * pending tests for field_4 * convert field_mapping to array of hashes * validate nulls based on form question * actually load forms when toggling between forms * validate null for startdate * row parser has access to bulk upload * csv upload validates first form section * add postcode validation * Refactor error mappings for row parser * Add unittype question * Fix null error setting and add builtype * add wchair to bulk upload * Add beds to bulk upload * Add joint to bulk upload * Add startertenancy to the bulk upload * Add tenancy for bulk upload * Add declaration to the bulk upload * Add age1 and age1_known to bulk upload * add ages to bulk upload * add sex1 to bulk upload * add ethnic_group and ethnic to bulk upload * add national to bulk upload * add ecstat1 to bulk upload * add military related fields to bulk upload * add preg_occ to bulk upload * add housingneeds to bulk upload * add illness to bulk upload * add layear to bulk upload * add waityear to bulk upload * add reason to bulk upload * add prevten to bulk upload * add homeless to bulk upload * add previous postcode to bulk upload * add reasonable preferences to bulk upload * add allocations system to bulk upload * add referral to bulk upload * add net_income_known to bulk upload * add hb to bulk upload * add benefits to bulk upload * add rent fields to bulk upload * add hhmemb to bulk upload * use 2022 csv fixtures for bulk upload * fix renewal mapping for bulk upload * placeholder test for bulk upload validation * fix bulk upload mapping for homeless field * fix leftreg mapping for bulk upload * fix user associations in bulk upload tests * add gender fields for bulk upload * add ecstatN fields to bulk upload * add #relatN fields to bulk upload * extract old_visible_id in factory to trait * map net_income_known correctly for bulk upload * fix income bugs for bulk upload * add unitletas to bulk upload * add #rsnvac to bulk upload * add #sheltered to bulk upload * add illness fields to bulk upload * add #irproduct_other to bulk upload * infer renewal from rsnvac for bulk upload * add #tenancyother to bulk upload * add #tenancylength to bulk upload * bulk upload earnings accepts pennies but rounds * add #reasonother to bulk upload * fix mapping of #ppcodenk for bulk upload * add #household_charge to bulk upload * add #chcharge to bulk upload * add #tcharge to bulk upload * add #supcharg to bulk upload * add pscharge to bulk upload * add #scharge to bulk upload * use case statement for bulk upload allocation * add offered to bulk upload * add propcode to bulk upload * add major repair fields to bulk upload * add #voiddate to bulk upload * support YY year format for bulk upload * test postcode strips whitespace for bulk upload * add #la to bulk upload * add previous la to bulk upload * fix failing test * remove duplicate line from rebase * add first time social housing to bulk upload * make methods private * fix field_4 validation for bulk upload - the null check was inverted by mistake Co-authored-by: Kat <katrina@kosiak.co.uk>
2 years ago
end
def field_number_for_attribute(attribute)
attribute.to_s.split("_").last.to_i
end
def cols
csv_parser.cols
end
def row_parsers
return @row_parsers if @row_parsers
@row_parsers = csv_parser.row_parsers
@row_parsers.each do |row_parser|
row_parser.bulk_upload = bulk_upload
end
@row_parsers
end
def rows
csv_parser.rows
end
def body_rows
csv_parser.body_rows
end
def validate_file_not_empty
if File.size(path).zero? || csv_parser.body_rows.flatten.compact.empty?
errors.add(:base, I18n.t("validations.lettings.#{@bulk_upload.year}.bulk_upload.blank_file"))
halt_validations!
end
end
2 years ago
def validate_field_numbers_count
return if halt_validations?
unless csv_parser.correct_field_count?
errors.add(:base, I18n.t("validations.lettings.#{@bulk_upload.year}.bulk_upload.wrong_template.wrong_field_numbers_count"))
halt_validations!
end
end
2 years ago
def validate_max_columns_count_if_no_headers
return if halt_validations?
if csv_parser.too_many_columns?
errors.add(:base, I18n.t("validations.lettings.#{@bulk_upload.year}.bulk_upload.wrong_template.over_max_column_count"))
halt_validations!
end
end
def validate_correct_template
return if halt_validations?
errors.add(:base, I18n.t("validations.lettings.#{@bulk_upload.year}.bulk_upload.wrong_template.wrong_template")) if csv_parser.wrong_template_for_year?
end
def validate_missing_required_headers
return if halt_validations?
if csv_parser.missing_required_headers?
errors.add :base, I18n.t("validations.lettings.#{@bulk_upload.year}.bulk_upload.wrong_template.no_headers", guidance_link: bulk_upload_lettings_log_url(id: "guidance", form: { year: bulk_upload.year }, host: ENV["APP_HOST"], anchor: "using-the-bulk-upload-template"))
halt_validations!
end
end
def halt_validations!
@halt_validations = true
end
def halt_validations?
@halt_validations ||= false
end
end