baarkerlounger
/
submit-social-housing-lettings-and-sales-data
mirror of https://github.com/communitiesuk/submit-social-housing-lettings-and-sales-data.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

CLDC-3802 sales bulk upload update 2025 (#2945) 

* add 2025 sales bulk upload parser

* lint and fix typos

* fix typos, field numbers, staircasing tests

* order field_11 values

* test log creator selects correct year's parser

* fix log to csv helper field order

* update factory so test file fully succeeds

* add 2025 BU test file method

* apply new csv syntax

* lint

* lint

* CLDC-3893 update property information field order

* commonise prepare your file page

* also update prep file page for lettings

* CLDC-3893 update test

* lint

* don't error on blank discount if not RTB

---------

Co-authored-by: Carolyn <carolyn.barker@softwire.com>
pull/2939/head
carolynbarker 4 days ago committed by GitHub
parent
fe89619b1f
commit
e5d10e219f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
20 changed files with 4137 additions and 168 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 0
      .rake_tasks~
  2. 34
      app/controllers/test_data_controller.rb
  3. 139
      app/helpers/bulk_upload/sales_log_to_csv.rb
  4. 2
      app/models/bulk_upload.rb
  5. 9
      app/models/forms/bulk_upload_form/prepare_your_file.rb
  6. 2
      app/services/bulk_upload/sales/log_creator.rb
  7. 2
      app/services/bulk_upload/sales/validator.rb
  8. 1
      app/services/bulk_upload/sales/year2024/row_parser.rb
  9. 124
      app/services/bulk_upload/sales/year2025/csv_parser.rb
  10. 1502
      app/services/bulk_upload/sales/year2025/row_parser.rb
  11. 4
      app/views/bulk_upload_lettings_logs/forms/prepare_your_file.html.erb
  12. 6
      app/views/bulk_upload_sales_logs/forms/prepare_your_file.html.erb
  13. 2
      config/locales/en.yml
  14. 46
      config/locales/validations/sales/2025/bulk_upload.en.yml
  15. BIN
      session-manager-plugin.deb
  16. 1
      spec/models/bulk_upload_spec.rb
  17. 309
      spec/services/bulk_upload/sales/log_creator_spec.rb
  18. 16
      spec/services/bulk_upload/sales/validator_spec.rb
  19. 191
      spec/services/bulk_upload/sales/year2025/csv_parser_spec.rb
  20. 1915
      spec/services/bulk_upload/sales/year2025/row_parser_spec.rb

0
.rake_tasks~
Unescape View File

34
app/controllers/test_data_controller.rb
Unescape View File

@ -47,22 +47,24 @@ class TestDataController < ApplicationController
redirect_to sales_log_path(log) redirect_to sales_log_path(log)
end end
def create_2024_test_sales_bulk_upload [2024, 2025].each do |year|
return render_not_found unless FeatureToggle.create_test_logs_enabled? define_method("create_#{year}_test_sales_bulk_upload") do
return render_not_found unless FeatureToggle.create_test_logs_enabled?
file = Tempfile.new("test_sales_log.csv")
log = FactoryBot.create(:sales_log, :completed, assigned_to: current_user, value: 180_000, deposit: 150_000) file = Tempfile.new("#{year}_test_sales_log.csv")
log_to_csv = BulkUpload::SalesLogToCsv.new(log:, line_ending: "\n", overrides: { organisation_id: "ORG#{log.owning_organisation_id}", managing_organisation_id: "ORG#{log.owning_organisation_id}" }) log = FactoryBot.create(:sales_log, :completed, assigned_to: current_user, value: 180_000, deposit: 150_000, county: "Somerset", saledate: Time.zone.local(year.to_i, rand(4..12), rand(1..28)))
file.write(log_to_csv.default_field_numbers_row) log_to_csv = BulkUpload::SalesLogToCsv.new(log:, line_ending: "\n", overrides: { organisation_id: "ORG#{log.owning_organisation_id}", managing_organisation_id: "ORG#{log.owning_organisation_id}" })
file.write(log_to_csv.to_csv_row) file.write(log_to_csv.default_field_numbers_row)
file.rewind file.write(log_to_csv.to_csv_row)
send_file file.path, type: "text/csv", file.rewind
filename: "test_sales_log.csv", send_file file.path,
disposition: "attachment", type: "text/csv",
after_send: lambda { filename: "#{year}_test_sales_log.csv",
file.close disposition: "attachment",
file.unlink after_send: lambda {
} file.close
file.unlink
}
end
end end
end end

139
app/helpers/bulk_upload/sales_log_to_csv.rb
Unescape View File

@ -19,7 +19,7 @@ class BulkUpload::SalesLogToCsv
case year case year
when 2022 when 2022
to_2022_csv_row to_2022_csv_row
when 2023, 2024 when 2023, 2024, 2025
to_year_csv_row(year, seed:) to_year_csv_row(year, seed:)
else else
raise NotImplementedError "No mapping function implemented for year #{year}" raise NotImplementedError "No mapping function implemented for year #{year}"
@ -67,6 +67,8 @@ class BulkUpload::SalesLogToCsv
[6, 3, 4, 5, nil, 28, 30, 38, 47, 51, 55, 59, 31, 39, 48, 52, 56, 60, 37, 46, 50, 54, 58, 35, 43, 49, 53, 57, 61, 32, 33, 78, 80, 79, 81, 83, 84, nil, 62, 66, 64, 65, 63, 67, 69, 70, 68, 76, 77, 16, 17, 18, 26, 24, 25, 27, 8, 91, 95, 96, 97, 92, 93, 94, 98, 100, 101, 103, 104, 106, 110, 111, 112, 113, 114, 9, 116, 117, 118, 120, 124, 125, 126, 10, 11, nil, 127, 129, 133, 134, 135, 1, 2, nil, 73, nil, 75, 107, 108, 121, 122, 130, 131, 82, 109, 123, 132, 115, 15, 86, 87, 29, 7, 12, 13, 14, 36, 44, 45, 88, 89, 102, 105, 119, 128, 19, 20, 21, 22, 23, 34, 40, 41, 42, 71, 72, 74, 85, 90, 99] [6, 3, 4, 5, nil, 28, 30, 38, 47, 51, 55, 59, 31, 39, 48, 52, 56, 60, 37, 46, 50, 54, 58, 35, 43, 49, 53, 57, 61, 32, 33, 78, 80, 79, 81, 83, 84, nil, 62, 66, 64, 65, 63, 67, 69, 70, 68, 76, 77, 16, 17, 18, 26, 24, 25, 27, 8, 91, 95, 96, 97, 92, 93, 94, 98, 100, 101, 103, 104, 106, 110, 111, 112, 113, 114, 9, 116, 117, 118, 120, 124, 125, 126, 10, 11, nil, 127, 129, 133, 134, 135, 1, 2, nil, 73, nil, 75, 107, 108, 121, 122, 130, 131, 82, 109, 123, 132, 115, 15, 86, 87, 29, 7, 12, 13, 14, 36, 44, 45, 88, 89, 102, 105, 119, 128, 19, 20, 21, 22, 23, 34, 40, 41, 42, 71, 72, 74, 85, 90, 99]
when 2024 when 2024
(1..131).to_a (1..131).to_a
when 2025
(1..121).to_a
else else
raise NotImplementedError "No mapping function implemented for year #{year}" raise NotImplementedError "No mapping function implemented for year #{year}"
end end
@ -395,6 +397,141 @@ class BulkUpload::SalesLogToCsv
] ]
end end
def to_2025_row
[
log.saledate&.day,
log.saledate&.month,
log.saledate&.strftime("%y"),
overrides[:organisation_id] || log.owning_organisation&.old_visible_id,
overrides[:managing_organisation_id] || log.managing_organisation&.old_visible_id,
log.assigned_to&.email,
log.purchid,
log.ownershipsch,
log.ownershipsch == 1 ? log.type : "", # field_9: "What is the type of shared ownership sale?",
log.staircase, # 10
log.ownershipsch == 2 ? log.type : "", # field_11: "What is the type of discounted ownership sale?",
log.jointpur,
log.jointmore,
log.noint,
log.privacynotice,
log.uprn,
log.address_line1&.tr(",", " "), # 20
log.address_line2&.tr(",", " "),
log.town_or_city&.tr(",", " "),
log.county&.tr(",", " "),
((log.postcode_full || "").split(" ") || [""]).first,
((log.postcode_full || "").split(" ") || [""]).last,
log.la,
log.proptype,
log.beds,
log.builtype,
log.wchair,
log.age1,
log.sex1,
log.ethnic, # 30
log.nationality_all_group,
log.ecstat1,
log.buy1livein,
log.relat2,
log.age2,
log.sex2,
log.ethnic_group2,
log.nationality_all_buyer2_group,
log.ecstat2,
log.buy2livein, # 40
log.hholdcount,
log.relat3,
log.age3,
log.sex3,
log.ecstat3,
log.relat4,
log.age4,
log.sex4,
log.ecstat4,
log.relat5, # 50
log.age5,
log.sex5,
log.ecstat5,
log.relat6,
log.age6,
log.sex6,
log.ecstat6,
log.prevten,
log.ppcodenk,
((log.ppostcode_full || "").split(" ") || [""]).first, # 60
((log.ppostcode_full || "").split(" ") || [""]).last,
log.prevloc,
log.buy2living,
log.prevtenbuy2,
log.hhregres,
log.hhregresstill,
log.armedforcesspouse,
log.disabled,
log.wheel,
log.income1, # 70
log.inc1mort,
log.income2,
log.inc2mort,
log.hb,
log.savings.present? || "R",
log.prevown,
log.prevshared,
log.resale,
log.proplen,
log.hodate&.day, # 80
log.hodate&.month,
log.hodate&.strftime("%y"),
log.frombeds,
log.fromprop,
log.socprevten,
log.value,
log.equity,
log.mortgageused,
log.mortgage,
log.mortlen, # 90
log.deposit,
log.cashdis,
log.mrent,
log.mscharge,
log.management_fee,
log.stairbought,
log.stairowned,
log.staircasesale,
log.firststair,
log.initialpurchase&.day, # 100
log.initialpurchase&.month,
log.initialpurchase&.strftime("%y"),
log.numstair,
log.lasttransaction&.day,
log.lasttransaction&.month,
log.lasttransaction&.strftime("%y"),
log.value,
log.equity,
log.mortgageused,
log.mrentprestaircasing, # 110
log.mrent,
log.proplen,
log.value,
log.grant,
log.discount,
log.mortgageused,
log.mortgage,
log.mortlen,
log.extrabor,
log.deposit, # 120
log.mscharge,
]
end
def custom_field_numbers_row(seed: nil, field_numbers: nil) def custom_field_numbers_row(seed: nil, field_numbers: nil)
if seed if seed
["Field number"] + field_numbers.shuffle(random: Random.new(seed)) ["Field number"] + field_numbers.shuffle(random: Random.new(seed))

2
app/models/bulk_upload.rb
Unescape View File

@ -104,6 +104,8 @@ class BulkUpload < ApplicationRecord
end end
year_class = case year year_class = case year
when 2025
"Year2025"
when 2024 when 2024
"Year2024" "Year2024"
when 2023 when 2023

9
app/models/forms/bulk_upload_form/prepare_your_file.rb
Unescape View File

@ -10,10 +10,7 @@ module Forms
attribute :organisation_id, :integer attribute :organisation_id, :integer
def view_path def view_path
case year "bulk_upload_#{log_type}_logs/forms/prepare_your_file"
when 2024
"bulk_upload_#{log_type}_logs/forms/prepare_your_file_2024"
end
end end
def back_path def back_path
@ -42,6 +39,10 @@ module Forms
"#{year} to #{year + 1}" "#{year} to #{year + 1}"
end end
def slash_year_combo
"#{year}/#{(year + 1) % 100}"
end
def save! def save!
true true
end end

2
app/services/bulk_upload/sales/log_creator.rb
Unescape View File

@ -33,6 +33,8 @@ private
BulkUpload::Sales::Year2023::CsvParser.new(path:) BulkUpload::Sales::Year2023::CsvParser.new(path:)
when 2024 when 2024
BulkUpload::Sales::Year2024::CsvParser.new(path:) BulkUpload::Sales::Year2024::CsvParser.new(path:)
when 2025
BulkUpload::Sales::Year2025::CsvParser.new(path:)
else else
raise "csv parser not found" raise "csv parser not found"
end end

2
app/services/bulk_upload/sales/validator.rb
Unescape View File

@ -108,6 +108,8 @@ private
BulkUpload::Sales::Year2023::CsvParser.new(path:) BulkUpload::Sales::Year2023::CsvParser.new(path:)
when 2024 when 2024
BulkUpload::Sales::Year2024::CsvParser.new(path:) BulkUpload::Sales::Year2024::CsvParser.new(path:)
when 2025
BulkUpload::Sales::Year2025::CsvParser.new(path:)
else else
raise "csv parser not found" raise "csv parser not found"
end end

1
app/services/bulk_upload/sales/year2024/row_parser.rb
Unescape View File

@ -375,6 +375,7 @@ class BulkUpload::Sales::Year2024::RowParser
greater_than_or_equal_to: 0, greater_than_or_equal_to: 0,
less_than_or_equal_to: 70, less_than_or_equal_to: 70,
if: :discounted_ownership?, if: :discounted_ownership?,
allow_blank: true,
}, },
on: :before_log on: :before_log

124
app/services/bulk_upload/sales/year2025/csv_parser.rb
Unescape View File

@ -0,0 +1,124 @@
require "csv"
class BulkUpload::Sales::Year2025::CsvParser
include CollectionTimeHelper
FIELDS = 121
MAX_COLUMNS = 142
FORM_YEAR = 2025
attr_reader :path
def initialize(path:)
@path = path
end
def row_offset
if with_headers?
rows.find_index { |row| row[0].present? && row[0].match(/field number/i) } + 1
else
0
end
end
def col_offset
with_headers? ? 1 : 0
end
def cols
@cols ||= ("A".."DR").to_a
end
def row_parsers
@row_parsers ||= body_rows.map { |row|
next if row.empty?
stripped_row = row[col_offset..]
hash = Hash[field_numbers.zip(stripped_row)]
BulkUpload::Sales::Year2025::RowParser.new(hash)
}.compact
end
def body_rows
rows[row_offset..]
end
def rows
@rows ||= CSV.parse(normalised_string, row_sep:)
end
def column_for_field(field)
cols[field_numbers.find_index(field) + col_offset]
end
def wrong_template_for_year?
collection_start_year_for_date(first_record_start_date) != FORM_YEAR
rescue Date::Error
false
end
def missing_required_headers?
!with_headers?
end
def correct_field_count?
valid_field_numbers_count = field_numbers.count { |f| f != "field_blank" }
valid_field_numbers_count == FIELDS
end
private
def default_field_numbers
(1..FIELDS).map do |number|
if number.to_s.match?(/^[0-9]+$/)
"field_#{number}"
else
"field_blank"
end
end
end
def field_numbers
@field_numbers ||= if with_headers?
rows[row_offset - 1][col_offset..].map { |number| number.to_s.match?(/^[0-9]+$/) ? "field_#{number}" : "field_blank" }
else
default_field_numbers
end
end
def headers
@headers ||= ("field_1".."field_#{FIELDS}").to_a
end
def with_headers?
# we will eventually want to validate that headers exist for this year
rows.map { |r| r[0] }.any? { |cell| cell&.match?(/field number/i) }
end
def row_sep
"\n"
end
def normalised_string
return @normalised_string if @normalised_string
@normalised_string = File.read(path, encoding: "bom|utf-8")
@normalised_string.gsub!("\r\n", "\n")
@normalised_string.scrub!("")
@normalised_string.tr!("\r", "\n")
@normalised_string
end
def first_record_start_date
if with_headers?
year = row_parsers.first.field_3.to_s.strip.length.between?(1, 2) ? row_parsers.first.field_3.to_i + 2000 : row_parsers.first.field_3.to_i
Date.new(year, row_parsers.first.field_2.to_i, row_parsers.first.field_1.to_i)
else
year = rows.first[2].to_s.strip.length.between?(1, 2) ? rows.first[2].to_i + 2000 : rows.first[2].to_i
Date.new(year, rows.first[1].to_i, rows.first[0].to_i)
end
end
end

1502
app/services/bulk_upload/sales/year2025/row_parser.rb
View File

File diff suppressed because it is too large Load Diff

4
app/views/bulk_upload_lettings_logs/forms/prepare_your_file_2024.html.erb → app/views/bulk_upload_lettings_logs/forms/prepare_your_file.html.erb
Unescape View File

@ -14,7 +14,7 @@
<h2 class="govuk-heading-s">Download template</h2> <h2 class="govuk-heading-s">Download template</h2>
<p class="govuk-body govuk-!-margin-bottom-2"><%= govuk_link_to "Download the lettings bulk upload template (2024 to 2025)", @form.template_path %></p> <p class="govuk-body govuk-!-margin-bottom-2"><%= govuk_link_to "Download the lettings bulk upload template (#{@form.year_combo})", @form.template_path %></p>
<p class="govuk-body govuk-!-margin-bottom-2">There are 8 rows of content in the templates. These rows are called the ‘headers’. They contain the CORE form questions and guidance about which questions are required and how to format your answers.</p> <p class="govuk-body govuk-!-margin-bottom-2">There are 8 rows of content in the templates. These rows are called the ‘headers’. They contain the CORE form questions and guidance about which questions are required and how to format your answers.</p>
<h2 class="govuk-heading-s">Create your file</h2> <h2 class="govuk-heading-s">Create your file</h2>
@ -22,7 +22,7 @@
<%= govuk_list [ <%= govuk_list [
"Fill in the template with data from your housing management system. Your data should go below the headers, with one row per log. Leave column A blank - the bulk upload fields start in column B.", "Fill in the template with data from your housing management system. Your data should go below the headers, with one row per log. Leave column A blank - the bulk upload fields start in column B.",
"Make sure each column of your data aligns with the matching headers above. You may need to reorder your data.", "Make sure each column of your data aligns with the matching headers above. You may need to reorder your data.",
"Use the #{govuk_link_to 'Lettings bulk upload Specification (2024 to 2025)', @form.specification_path} to check your data is in the correct format.".html_safe, "Use the #{govuk_link_to "Lettings bulk upload Specification (#{@form.year_combo})", @form.specification_path} to check your data is in the correct format.".html_safe,
"<strong>Username field:</strong> To assign a log to someone else, enter the email address they use to log into CORE.".html_safe, "<strong>Username field:</strong> To assign a log to someone else, enter the email address they use to log into CORE.".html_safe,
"If you have reordered the headers, keep the headers in the file.", "If you have reordered the headers, keep the headers in the file.",
], type: :bullet %> ], type: :bullet %>

6
app/views/bulk_upload_sales_logs/forms/prepare_your_file_2024.html.erb → app/views/bulk_upload_sales_logs/forms/prepare_your_file.html.erb
Unescape View File

@ -14,15 +14,15 @@
<h2 class="govuk-heading-s">Download template</h2> <h2 class="govuk-heading-s">Download template</h2>
<p class="govuk-body govuk-!-margin-bottom-2">Use one of these templates to upload logs for 2024/25:</p> <p class="govuk-body govuk-!-margin-bottom-2">Use one of these templates to upload logs for <%= @form.slash_year_combo %>:</p>
<p class="govuk-body govuk-!-margin-bottom-2"><%= govuk_link_to "Download the sales bulk upload template (2024 to 2025)", @form.template_path %>: In this template, the questions are in the same order as the 2024/25 paper form and web form.</p> <p class="govuk-body govuk-!-margin-bottom-2"><%= govuk_link_to "Download the sales bulk upload template (#{@form.year_combo})", @form.template_path %>: In this template, the questions are in the same order as the <%= @form.slash_year_combo %> paper form and web form.</p>
<p class="govuk-body govuk-!-margin-bottom-2">There are 8 rows of content in the templates. These rows are called the ‘headers’. They contain the CORE form questions and guidance about which questions are required and how to format your answers.</p> <p class="govuk-body govuk-!-margin-bottom-2">There are 8 rows of content in the templates. These rows are called the ‘headers’. They contain the CORE form questions and guidance about which questions are required and how to format your answers.</p>
<h2 class="govuk-heading-s">Create your file</h2> <h2 class="govuk-heading-s">Create your file</h2>
<%= govuk_list [ <%= govuk_list [
"Fill in the template with data from your housing management system. Your data should go below the headers, with one row per log. The bulk upload fields start at column B. Leave column A blank.", "Fill in the template with data from your housing management system. Your data should go below the headers, with one row per log. The bulk upload fields start at column B. Leave column A blank.",
"Make sure each column of your data aligns with the matching headers above. You may need to reorder your data.", "Make sure each column of your data aligns with the matching headers above. You may need to reorder your data.",
"Use the #{govuk_link_to 'Sales bulk upload Specification (2024 to 2025)', @form.specification_path} to check your data is in the correct format.".html_safe, "Use the #{govuk_link_to "Sales bulk upload Specification (#{@form.year_combo})", @form.specification_path} to check your data is in the correct format.".html_safe,
"<strong>Username field:</strong> To assign a log to someone else, enter the email address they use to log into CORE.".html_safe, "<strong>Username field:</strong> To assign a log to someone else, enter the email address they use to log into CORE.".html_safe,
"If you have reordered the headers, keep the headers in the file.", "If you have reordered the headers, keep the headers in the file.",
], type: :bullet %> ], type: :bullet %>

2
config/locales/en.yml
Unescape View File

@ -57,6 +57,8 @@ en:
<<: *bulk_upload__row_parser__base <<: *bulk_upload__row_parser__base
bulk_upload/lettings/year2023/row_parser: bulk_upload/lettings/year2023/row_parser:
<<: *bulk_upload__row_parser__base <<: *bulk_upload__row_parser__base
bulk_upload/sales/year2025/row_parser:
<<: *bulk_upload__row_parser__base
bulk_upload/sales/year2024/row_parser: bulk_upload/sales/year2024/row_parser:
<<: *bulk_upload__row_parser__base <<: *bulk_upload__row_parser__base
bulk_upload/sales/year2023/row_parser: bulk_upload/sales/year2023/row_parser:

46
config/locales/validations/sales/2025/bulk_upload.en.yml
Unescape View File

@ -0,0 +1,46 @@
en:
validations:
sales:
2025:
bulk_upload:
not_answered: "You must answer %{question}"
invalid_option: "Enter a valid value for %{question}"
spreadsheet_dupe: "This is a duplicate of a log in your file."
duplicate: "This is a duplicate log."
blank_file: "Template is blank - The template must be filled in for us to create the logs and check if data is correct."
wrong_template:
over_max_column_count: "Too many columns, please ensure you have used the correct template."
no_headers: "Your file does not contain the required header rows. Add or check the header rows and upload your file again. [Read more about using the template headers](%{guidance_link})."
wrong_field_numbers_count: "Incorrect number of fields, please ensure you have used the correct template."
wrong_template: "Incorrect sale dates, please ensure you have used the correct template."
numeric:
within_range: "%{field} must be between %{min} and %{max}."
owning_organisation:
not_found: "The owning organisation code is incorrect."
not_stock_owner: "The owning organisation code provided is for an organisation that does not own stock."
not_permitted:
support: "This owning organisation is not affiliated with %{name}."
not_support: "You do not have permission to add logs for this owning organisation."
assigned_to:
not_found: "User with the specified email could not be found."
organisation_not_related: "User must be related to owning organisation or managing organisation."
managing_organisation_not_related: "This organisation does not have a relationship with the owning organisation."
saledate:
outside_collection_window: "Enter a date within the %{year_combo} collection year, which is between 1st April %{start_year} and 31st March %{end_year}."
year_not_two_or_four_digits: "Sale completion year must be 2 or 4 digits."
ecstat1:
buyer_cannot_be_over_16_and_child: "Buyer 1's age cannot be 16 or over if their working situation is child under 16."
buyer_cannot_be_child: "Buyer 1 cannot have a working situation of child under 16."
age1:
buyer_cannot_be_over_16_and_child: "Buyer 1's age cannot be 16 or over if their working situation is child under 16."
ecstat2:
buyer_cannot_be_over_16_and_child: "Buyer 2's age cannot be 16 or over if their working situation is child under 16."
buyer_cannot_be_child: "Buyer 2 cannot have a working situation of child under 16."
age2:
buyer_cannot_be_over_16_and_child: "Buyer 2's age cannot be 16 or over if their working situation is child under 16."
address:
not_found: "We could not find this address. Check the address data in your CSV file is correct and complete, or select the correct address using the CORE site."
not_determined: "There are multiple matches for this address. Either select the correct address manually or correct the UPRN in the CSV file."
not_answered: "Enter either the UPRN or the full address."
nationality:
invalid: "Select a valid nationality."

BIN
session-manager-plugin.deb
View File

Binary file not shown.

1
spec/models/bulk_upload_spec.rb
Unescape View File

@ -41,6 +41,7 @@ RSpec.describe BulkUpload, type: :model do
[ [
{ year: 2023, expected_value: "2023 to 2024" }, { year: 2023, expected_value: "2023 to 2024" },
{ year: 2024, expected_value: "2024 to 2025" }, { year: 2024, expected_value: "2024 to 2025" },
{ year: 2025, expected_value: "2025 to 2026" },
].each do |test_case| ].each do |test_case|
context "when the bulk upload year is #{test_case[:year]}" do context "when the bulk upload year is #{test_case[:year]}" do
let(:bulk_upload) { build(:bulk_upload, year: test_case[:year]) } let(:bulk_upload) { build(:bulk_upload, year: test_case[:year]) }

309
spec/services/bulk_upload/sales/log_creator_spec.rb
Unescape View File

@ -5,162 +5,187 @@ RSpec.describe BulkUpload::Sales::LogCreator do
let(:owning_org) { create(:organisation, old_visible_id: 123) } let(:owning_org) { create(:organisation, old_visible_id: 123) }
let(:user) { create(:user, organisation: owning_org) } let(:user) { create(:user, organisation: owning_org) }
let(:bulk_upload) { create(:bulk_upload, :sales, user:, year: 2024) }
let(:csv_parser) { instance_double(BulkUpload::Sales::Year2024::CsvParser) }
let(:row_parser) { instance_double(BulkUpload::Sales::Year2024::RowParser) }
let(:log) { build(:sales_log, :completed, assigned_to: user, owning_organisation: owning_org, managing_organisation: owning_org) } let(:log) { build(:sales_log, :completed, assigned_to: user, owning_organisation: owning_org, managing_organisation: owning_org) }
before do [2023, 2024, 2025].each do |year|
allow(BulkUpload::Sales::Year2024::CsvParser).to receive(:new).and_return(csv_parser) context "when #{year}" do
allow(csv_parser).to receive(:row_parsers).and_return([row_parser]) let(:bulk_upload) { create(:bulk_upload, :sales, user:, year:) }
allow(row_parser).to receive(:log).and_return(log) let(:year_csv_parser) { instance_double("BulkUpload::Sales::Year#{year}::CsvParser".constantize) }
allow(row_parser).to receive(:bulk_upload=).and_return(true) let(:year_row_parser) { instance_double("BulkUpload::Sales::Year#{year}::RowParser".constantize) }
allow(row_parser).to receive(:valid?).and_return(true)
allow(row_parser).to receive(:blank_row?).and_return(false)
end
describe "#call" do
context "when a valid csv with new log" do
it "creates a new log" do
expect { service.call }.to change(SalesLog, :count)
end
it "create a log with pending status" do
service.call
expect(SalesLog.last.status).to eql("pending")
end
it "associates log with bulk upload" do
service.call
log = SalesLog.last
expect(log.bulk_upload).to eql(bulk_upload)
expect(bulk_upload.sales_logs).to include(log)
end
it "sets the creation method" do
service.call
expect(SalesLog.last.creation_method_bulk_upload?).to be true
end
end
context "when a valid csv with several blank rows" do
before do before do
allow(row_parser).to receive(:blank_row?).and_return(true) allow("BulkUpload::Sales::Year#{year}::CsvParser".constantize).to receive(:new).and_return(year_csv_parser)
end allow(year_csv_parser).to receive(:row_parsers).and_return([year_row_parser])
allow(year_row_parser).to receive(:log).and_return(log)
it "ignores them and does not create the logs" do allow(year_row_parser).to receive(:bulk_upload=).and_return(true)
expect { service.call }.not_to change(SalesLog, :count) allow(year_row_parser).to receive(:valid?).and_return(true)
end allow(year_row_parser).to receive(:blank_row?).and_return(false)
end
context "when a valid csv with row with one invalid non setup field" do
let(:log) do
build(
:sales_log,
:completed,
age1: 5,
owning_organisation: owning_org,
assigned_to: user,
managing_organisation: owning_org,
)
end
it "creates the log" do
expect { service.call }.to change(SalesLog, :count).by(1)
end
it "blanks invalid field" do
service.call
record = SalesLog.last
expect(record.age1).to be_blank
end
end
context "when a valid csv with row with compound errors on non setup field" do
let(:log) do
build(
:sales_log,
:completed,
owning_organisation: owning_org,
assigned_to: user,
managing_organisation: owning_org,
ownershipsch: 2,
value: 200_000,
deposit: 10_000,
mortgageused: 1,
mortgage: 100_000,
grant: 10_000,
)
end end
it "creates the log" do it "creates a parser for the correct year" do
expect { service.call }.to change(SalesLog, :count).by(1) # This would fail without parser stubs, so the parser must be for the expected year
end
it "blanks invalid field" do
service.call
record = SalesLog.last
expect(record.value).to be_blank
expect(record.deposit).to be_blank
expect(record.mortgage).to be_blank
expect(record.grant).to be_blank
end
end
context "when pre-creating logs" do
it "creates a new log" do
expect { service.call }.to change(SalesLog, :count) expect { service.call }.to change(SalesLog, :count)
end end
it "creates a log with correct states" do
service.call
last_log = SalesLog.last
expect(last_log.status).to eql("pending")
expect(last_log.status_cache).to eql("completed")
end
end end
end
context "when valid csv with existing log" do # Apart from picking the correct year's parser, everything else is year-independent
xit "what should happen?" context "when 2024" do
let(:bulk_upload) { create(:bulk_upload, :sales, user:, year: 2024) }
let(:csv_parser) { instance_double(BulkUpload::Sales::Year2024::CsvParser) }
let(:row_parser) { instance_double(BulkUpload::Sales::Year2024::RowParser) }
before do
allow(BulkUpload::Sales::Year2024::CsvParser).to receive(:new).and_return(csv_parser)
allow(csv_parser).to receive(:row_parsers).and_return([row_parser])
allow(row_parser).to receive(:log).and_return(log)
allow(row_parser).to receive(:bulk_upload=).and_return(true)
allow(row_parser).to receive(:valid?).and_return(true)
allow(row_parser).to receive(:blank_row?).and_return(false)
end end
context "with a valid csv and soft validations" do describe "#call" do
let(:log) do context "when a valid csv with new log" do
build( it "creates a new log" do
:sales_log, expect { service.call }.to change(SalesLog, :count)
:completed, end
age1: 30,
age1_known: 0, it "create a log with pending status" do
ecstat1: 5, service.call
owning_organisation: owning_org, expect(SalesLog.last.status).to eql("pending")
assigned_to: user, end
managing_organisation: owning_org,
)
end
it "creates a new log" do
expect { service.call }.to change(SalesLog, :count)
end
it "creates a log with pending status" do
service.call
expect(SalesLog.last.status).to eql("pending")
end
it "does not set unanswered soft validations" do it "associates log with bulk upload" do
service.call service.call
log = SalesLog.last log = SalesLog.last
expect(log.age1).to be(30) expect(log.bulk_upload).to eql(bulk_upload)
expect(log.ecstat1).to be(5) expect(bulk_upload.sales_logs).to include(log)
expect(log.retirement_value_check).to be(nil) end
it "sets the creation method" do
service.call
expect(SalesLog.last.creation_method_bulk_upload?).to be true
end
end
context "when a valid csv with several blank rows" do
before do
allow(row_parser).to receive(:blank_row?).and_return(true)
end
it "ignores them and does not create the logs" do
expect { service.call }.not_to change(SalesLog, :count)
end
end
context "when a valid csv with row with one invalid non setup field" do
let(:log) do
build(
:sales_log,
:completed,
age1: 5,
owning_organisation: owning_org,
assigned_to: user,
managing_organisation: owning_org,
)
end
it "creates the log" do
expect { service.call }.to change(SalesLog, :count).by(1)
end
it "blanks invalid field" do
service.call
record = SalesLog.last
expect(record.age1).to be_blank
end
end
context "when a valid csv with row with compound errors on non setup field" do
let(:log) do
build(
:sales_log,
:completed,
owning_organisation: owning_org,
assigned_to: user,
managing_organisation: owning_org,
ownershipsch: 2,
value: 200_000,
deposit: 10_000,
mortgageused: 1,
mortgage: 100_000,
grant: 10_000,
)
end
it "creates the log" do
expect { service.call }.to change(SalesLog, :count).by(1)
end
it "blanks invalid field" do
service.call
record = SalesLog.last
expect(record.value).to be_blank
expect(record.deposit).to be_blank
expect(record.mortgage).to be_blank
expect(record.grant).to be_blank
end
end
context "when pre-creating logs" do
it "creates a new log" do
expect { service.call }.to change(SalesLog, :count)
end
it "creates a log with correct states" do
service.call
last_log = SalesLog.last
expect(last_log.status).to eql("pending")
expect(last_log.status_cache).to eql("completed")
end
end
context "when valid csv with existing log" do
xit "what should happen?"
end
context "with a valid csv and soft validations" do
let(:log) do
build(
:sales_log,
:completed,
age1: 30,
age1_known: 0,
ecstat1: 5,
owning_organisation: owning_org,
assigned_to: user,
managing_organisation: owning_org,
)
end
it "creates a new log" do
expect { service.call }.to change(SalesLog, :count)
end
it "creates a log with pending status" do
service.call
expect(SalesLog.last.status).to eql("pending")
end
it "does not set unanswered soft validations" do
service.call
log = SalesLog.last
expect(log.age1).to be(30)
expect(log.ecstat1).to be(5)
expect(log.retirement_value_check).to be(nil)
end
end end
end end
end end

16
spec/services/bulk_upload/sales/validator_spec.rb
Unescape View File

@ -58,6 +58,22 @@ RSpec.describe BulkUpload::Sales::Validator do
end end
end end
context "when trying to upload 2024 logs for 2025 bulk upload" do
let(:bulk_upload) { create(:bulk_upload, user:, year: 2025) }
let(:log) { build(:sales_log, :completed, saledate: Time.zone.local(2024, 10, 10), assigned_to: user) }
before do
file.write(log_to_csv.default_field_numbers_row_for_year(2025))
file.write(log_to_csv.to_year_csv_row(2025))
file.rewind
end
it "is not valid" do
expect(validator).not_to be_valid
expect(validator.errors["base"]).to eql([I18n.t("validations.sales.2025.bulk_upload.wrong_template.wrong_template")])
end
end
[ [
{ line_ending: "\n", name: "unix" }, { line_ending: "\n", name: "unix" },
{ line_ending: "\r\n", name: "windows" }, { line_ending: "\r\n", name: "windows" },

191
spec/services/bulk_upload/sales/year2025/csv_parser_spec.rb
Unescape View File

@ -0,0 +1,191 @@
require "rails_helper"
RSpec.describe BulkUpload::Sales::Year2025::CsvParser do
subject(:service) { described_class.new(path:) }
let(:file) { Tempfile.new }
let(:path) { file.path }
let(:log) { build(:sales_log, :completed, :with_uprn) }
context "when parsing csv with headers" do
before do
file.write("Question\n")
file.write("Additional info\n")
file.write("Values\n")
file.write("Can be empty?\n")
file.write("Type of letting the question applies to\n")
file.write("Duplicate check field?\n")
file.write(BulkUpload::SalesLogToCsv.new(log:).default_field_numbers_row_for_year(2025))
file.write(BulkUpload::SalesLogToCsv.new(log:).to_year_csv_row(2025))
file.write("\n")
file.rewind
end
it "returns correct offsets" do
expect(service.row_offset).to eq(7)
expect(service.col_offset).to eq(1)
end
it "parses csv correctly" do
expect(service.row_parsers[0].field_16).to eql(log.uprn)
end
it "counts the number of valid field numbers correctly" do
expect(service).to be_correct_field_count
end
it "does not parse the last empty row" do
expect(service.row_parsers.count).to eq(1)
end
end
context "when some csv headers are empty (and we don't care about them)" do
before do
file.write("Question\n")
file.write("Additional info\n")
file.write("Values\n")
file.write("\n")
file.write("Type of letting the question applies to\n")
file.write("Duplicate check field?\n")
file.write(BulkUpload::SalesLogToCsv.new(log:).default_field_numbers_row_for_year(2025))
file.write(BulkUpload::SalesLogToCsv.new(log:).to_year_csv_row(2025))
file.write("\n")
file.rewind
end
it "returns correct offsets" do
expect(service.row_offset).to eq(7)
expect(service.col_offset).to eq(1)
end
it "parses csv correctly" do
expect(service.row_parsers[0].field_16).to eql(log.uprn)
end
it "counts the number of valid field numbers correctly" do
expect(service).to be_correct_field_count
end
it "does not parse the last empty row" do
expect(service.row_parsers.count).to eq(1)
end
end
context "when parsing csv with headers in arbitrary order" do
let(:seed) { rand }
before do
file.write("Question\n")
file.write("Additional info\n")
file.write("Values\n")
file.write("Can be empty?\n")
file.write("Type of letting the question applies to\n")
file.write("Duplicate check field?\n")
file.write(BulkUpload::SalesLogToCsv.new(log:).default_field_numbers_row_for_year(2025, seed:))
file.write(BulkUpload::SalesLogToCsv.new(log:).to_year_csv_row(2025, seed:))
file.rewind
end
it "returns correct offsets" do
expect(service.row_offset).to eq(7)
expect(service.col_offset).to eq(1)
end
it "parses csv correctly" do
expect(service.row_parsers[0].field_16).to eql(log.uprn)
end
end
context "when parsing csv without headers" do
let(:file) { Tempfile.new }
let(:path) { file.path }
let(:log) { build(:sales_log, :completed, :with_uprn) }
before do
file.write(BulkUpload::SalesLogToCsv.new(log:, col_offset: 0).to_year_csv_row(2025))
file.rewind
end
it "returns correct offsets" do
expect(service.row_offset).to eq(0)
expect(service.col_offset).to eq(0)
end
it "parses csv correctly" do
expect(service.row_parsers[0].field_16).to eql(log.uprn)
end
end
context "when parsing with BOM aka byte order mark" do
let(:file) { Tempfile.new }
let(:path) { file.path }
let(:log) { build(:sales_log, :completed, :with_uprn) }
let(:bom) { "\uFEFF" }
before do
file.write(bom)
file.write(BulkUpload::SalesLogToCsv.new(log:, col_offset: 0).to_year_csv_row(2025))
file.close
end
it "parses csv correctly" do
expect(service.row_parsers[0].field_16).to eql(log.uprn)
end
end
context "when an invalid byte sequence" do
let(:file) { Tempfile.new }
let(:path) { file.path }
let(:log) { build(:sales_log, :completed, :with_uprn) }
let(:invalid_sequence) { "\x81" }
before do
file.write(invalid_sequence)
file.write(BulkUpload::SalesLogToCsv.new(log:, col_offset: 0).to_year_csv_row(2025))
file.close
end
it "parses csv correctly" do
expect(service.row_parsers[0].field_16).to eql(log.uprn)
end
end
describe "#column_for_field", aggregate_failures: true do
context "when headers present" do
before do
file.write("Question\n")
file.write("Additional info\n")
file.write("Values\n")
file.write("Can be empty?\n")
file.write("Type of letting the question applies to\n")
file.write("Duplicate check field?\n")
file.write(BulkUpload::SalesLogToCsv.new(log:).default_field_numbers_row_for_year(2025))
file.write(BulkUpload::SalesLogToCsv.new(log:).to_year_csv_row(2025))
file.rewind
end
it "returns correct column" do
expect(service.column_for_field("field_1")).to eql("B")
expect(service.column_for_field("field_99")).to eql("CV")
end
end
end
context "when parsing csv with carriage returns" do
before do
file.write("Question\r\n")
file.write("Additional info\r")
file.write("Values\r\n")
file.write("Can be empty?\r")
file.write("Type of letting the question applies to\r\n")
file.write("Duplicate check field?\r")
file.write(BulkUpload::SalesLogToCsv.new(log:).default_field_numbers_row_for_year(2025))
file.write(BulkUpload::SalesLogToCsv.new(log:).to_year_csv_row(2025))
file.rewind
end
it "parses csv correctly" do
expect(service.row_parsers[0].field_16).to eql(log.uprn)
end
end
end

1915
spec/services/bulk_upload/sales/year2025/row_parser_spec.rb
View File

File diff suppressed because it is too large Load Diff
Write Preview
Loading…
Cancel
Save