From add39e9611a42aa35ef35ac9f38dd82d79845a90 Mon Sep 17 00:00:00 2001 From: Phil Lee Date: Thu, 8 Dec 2022 12:18:35 +0000 Subject: [PATCH] [1716] [1717] Bulk upload part four (#1043) * add start of bulk upload logs journey * split upload controller into 2 * add year page to bulk upload journey * bulk upload years now dynamic * add placeholder for upload your file page * handle bulk upload when not in crossover * add bulk upload csv user journey * bulk upload now persists and sends to S3 * swap ruby-filemagic for file command * match csv or text file for validation * in_crossover_period? now uses overlap of forms - also moved from Form to FormHandler * add production env var for CSV bucket * stub kernel call methods * remove duplicate env var * hardcode env var for review apps * move feature toggle to FeatureToggle * crossover period checks now specific to the form * fix typo in bulk upload journey --- .github/workflows/review_pipeline.yml | 2 +- app/controllers/bulk_upload_controller.rb | 4 +- .../bulk_upload_lettings_logs_controller.rb | 10 +- .../bulk_upload_sales_logs_controller.rb | 10 +- app/models/bulk_upload.rb | 198 +----------------- app/models/form.rb | 4 - app/models/form_handler.rb | 14 ++ .../bulk_upload_lettings/checking_file.rb | 27 +++ .../bulk_upload_lettings/prepare_your_file.rb | 6 +- .../bulk_upload_lettings/upload_your_file.rb | 50 +++++ app/models/forms/bulk_upload_lettings/year.rb | 4 + .../forms/bulk_upload_sales/checking_file.rb | 27 +++ .../bulk_upload_sales/prepare_your_file.rb | 6 +- .../bulk_upload_sales/upload_your_file.rb | 50 +++++ app/models/forms/bulk_upload_sales/year.rb | 4 + app/models/legacy_bulk_upload.rb | 197 +++++++++++++++++ .../forms/checking_file.html.erb | 28 +++ .../forms/upload_your_file.html.erb | 28 ++- .../forms/checking_file.html.erb | 28 +++ .../forms/upload_your_file.html.erb | 28 ++- app/views/logs/bulk_upload.html.erb | 2 +- config/initializers/feature_toggle.rb | 4 + config/locales/en.yml | 10 + .../20221128130843_create_bulk_uploads.rb | 14 ++ db/schema.rb | 11 + .../bulk_upload_lettings_logs_spec.rb | 36 +++- spec/features/bulk_upload_sales_logs_spec.rb | 36 +++- .../files/blank_bulk_upload_sales.csv | 118 +++++++++++ spec/fixtures/files/excel_as_csv.csv | Bin 0 -> 24693 bytes spec/models/form_handler_spec.rb | 16 ++ spec/models/form_spec.rb | 34 --- .../upload_your_file_spec.rb | 47 +++++ .../upload_your_file_spec.rb | 47 +++++ spec/requests/bulk_upload_controller_spec.rb | 2 +- ...lk_upload_lettings_logs_controller_spec.rb | 2 +- 35 files changed, 837 insertions(+), 267 deletions(-) create mode 100644 app/models/forms/bulk_upload_lettings/checking_file.rb create mode 100644 app/models/forms/bulk_upload_sales/checking_file.rb create mode 100644 app/models/legacy_bulk_upload.rb create mode 100644 app/views/bulk_upload_lettings_logs/forms/checking_file.html.erb create mode 100644 app/views/bulk_upload_sales_logs/forms/checking_file.html.erb create mode 100644 db/migrate/20221128130843_create_bulk_uploads.rb create mode 100644 spec/fixtures/files/blank_bulk_upload_sales.csv create mode 100644 spec/fixtures/files/excel_as_csv.csv create mode 100644 spec/models/forms/bulk_upload_lettings/upload_your_file_spec.rb create mode 100644 spec/models/forms/bulk_upload_sales/upload_your_file_spec.rb diff --git a/.github/workflows/review_pipeline.yml b/.github/workflows/review_pipeline.yml index 603a8315c..52c055360 100644 --- a/.github/workflows/review_pipeline.yml +++ b/.github/workflows/review_pipeline.yml @@ -120,7 +120,7 @@ jobs: cf set-env $APP_NAME IMPORT_PAAS_INSTANCE $IMPORT_PAAS_INSTANCE cf set-env $APP_NAME EXPORT_PAAS_INSTANCE $EXPORT_PAAS_INSTANCE cf set-env $APP_NAME S3_CONFIG $S3_CONFIG - cf set-env $APP_NAME CSV_DOWNLOAD_PAAS_INSTANCE $CSV_DOWNLOAD_PAAS_INSTANCE + cf set-env $APP_NAME CSV_DOWNLOAD_PAAS_INSTANCE "dluhc-core-review-csv-bucket" cf set-env $APP_NAME SENTRY_DSN $SENTRY_DSN cf set-env $APP_NAME APP_HOST "https://dluhc-core-review-${{ github.event.pull_request.number }}.london.cloudapps.digital" diff --git a/app/controllers/bulk_upload_controller.rb b/app/controllers/bulk_upload_controller.rb index ed2ac2627..c045dc066 100644 --- a/app/controllers/bulk_upload_controller.rb +++ b/app/controllers/bulk_upload_controller.rb @@ -2,14 +2,14 @@ class BulkUploadController < ApplicationController before_action :authenticate_user! def show - @bulk_upload = BulkUpload.new(nil, nil) + @bulk_upload = LegacyBulkUpload.new(nil, nil) render "logs/bulk_upload" end def bulk_upload file = upload_params.tempfile content_type = upload_params.content_type - @bulk_upload = BulkUpload.new(file, content_type) + @bulk_upload = LegacyBulkUpload.new(file, content_type) @bulk_upload.process(current_user) if @bulk_upload.errors.present? render "logs/bulk_upload", status: :unprocessable_entity diff --git a/app/controllers/bulk_upload_lettings_logs_controller.rb b/app/controllers/bulk_upload_lettings_logs_controller.rb index 108c25fef..104c31c05 100644 --- a/app/controllers/bulk_upload_lettings_logs_controller.rb +++ b/app/controllers/bulk_upload_lettings_logs_controller.rb @@ -14,7 +14,7 @@ class BulkUploadLettingsLogsController < ApplicationController end def update - if form.valid? + if form.valid? && form.save! redirect_to form.next_path else render form.view_path @@ -28,7 +28,7 @@ private end def in_crossover_period? - FormHandler.instance.forms.values.any?(&:in_crossover_period?) + FormHandler.instance.lettings_in_crossover_period? end def form @@ -38,13 +38,15 @@ private when "prepare-your-file" Forms::BulkUploadLettings::PrepareYourFile.new(form_params) when "upload-your-file" - Forms::BulkUploadLettings::UploadYourFile.new(form_params) + Forms::BulkUploadLettings::UploadYourFile.new(form_params.merge(current_user:)) + when "checking-file" + Forms::BulkUploadLettings::CheckingFile.new(form_params) else raise "Page not found for path #{params[:id]}" end end def form_params - params.fetch(:form, {}).permit(:year) + params.fetch(:form, {}).permit(:year, :file) end end diff --git a/app/controllers/bulk_upload_sales_logs_controller.rb b/app/controllers/bulk_upload_sales_logs_controller.rb index 81d018d4c..5f9b8d9a6 100644 --- a/app/controllers/bulk_upload_sales_logs_controller.rb +++ b/app/controllers/bulk_upload_sales_logs_controller.rb @@ -14,7 +14,7 @@ class BulkUploadSalesLogsController < ApplicationController end def update - if form.valid? + if form.valid? && form.save! redirect_to form.next_path else render form.view_path @@ -28,7 +28,7 @@ private end def in_crossover_period? - FormHandler.instance.forms.values.any?(&:in_crossover_period?) + FormHandler.instance.sales_in_crossover_period? end def form @@ -38,13 +38,15 @@ private when "prepare-your-file" Forms::BulkUploadSales::PrepareYourFile.new(form_params) when "upload-your-file" - Forms::BulkUploadSales::UploadYourFile.new(form_params) + Forms::BulkUploadSales::UploadYourFile.new(form_params.merge(current_user:)) + when "checking-file" + Forms::BulkUploadSales::CheckingFile.new(form_params) else raise "Page not found for path #{params[:id]}" end end def form_params - params.fetch(:form, {}).permit(:year) + params.fetch(:form, {}).permit(:year, :file) end end diff --git a/app/models/bulk_upload.rb b/app/models/bulk_upload.rb index fa2a7a0e9..8b007f450 100644 --- a/app/models/bulk_upload.rb +++ b/app/models/bulk_upload.rb @@ -1,197 +1,13 @@ -class BulkUpload - include ActiveModel::Model - include ActiveModel::Validations - include ActiveModel::Conversion +class BulkUpload < ApplicationRecord + enum log_type: { lettings: "lettings", sales: "sales" } - SPREADSHEET_CONTENT_TYPES = %w[ - application/vnd.ms-excel - application/vnd.openxmlformats-officedocument.spreadsheetml.sheet - ].freeze + belongs_to :user - FIRST_DATA_ROW = 7 + after_initialize :generate_identifier, unless: :identifier - def initialize(file, content_type) - @file = file - @content_type = content_type - end - - def process(current_user) - return unless valid_content_type? - - xlsx = Roo::Spreadsheet.open(@file, extension: :xlsx) - sheet = xlsx.sheet(0) - last_row = sheet.last_row - if last_row < FIRST_DATA_ROW - errors.add(:lettings_log_bulk_upload, "No data found") - else - data_range = FIRST_DATA_ROW..last_row - data_range.map do |row_num| - row = sheet.row(row_num) - # owning_organisation = Organisation.find(row[111]) - # managing_organisation = Organisation.find(row[113]) - lettings_log = LettingsLog.create!( - owning_organisation: current_user.organisation, - managing_organisation: current_user.organisation, - created_by: current_user, - ) - map_row(row).each do |attr_key, attr_val| - _update = lettings_log.update(attr_key => attr_val) - rescue ArgumentError - # TODO: determine what we want to do when bulk upload contains totally invalid data for a field. - end - end - end - end - - def valid_content_type? - if SPREADSHEET_CONTENT_TYPES.include?(@content_type) - true - else - errors.add(:lettings_log_bulk_upload, "Invalid file type") - false - end - end - - def map_row(row) - { - lettype: row[1], - # reg_num_la_core_code: row[3], - # managementgroup: row[4], - # schemecode: row[5], - # firstletting: row[6], - tenancycode: row[7], - startertenancy: row[8], - tenancy: row[9], - tenancyother: row[10], - # tenancyduration: row[11], - hhmemb: hhmemb(row), - age1: row[12], - age2: row[13], - age3: row[14], - age4: row[15], - age5: row[16], - age6: row[17], - age7: row[18], - age8: row[19], - sex1: row[20], - sex2: row[21], - sex3: row[22], - sex4: row[23], - sex5: row[24], - sex6: row[25], - sex7: row[26], - sex8: row[27], - relat2: row[28], - relat3: row[29], - relat4: row[30], - relat5: row[31], - relat6: row[32], - relat7: row[33], - relat8: row[34], - ecstat1: row[35], - ecstat2: row[36], - ecstat3: row[37], - ecstat4: row[38], - ecstat5: row[39], - ecstat6: row[40], - ecstat7: row[41], - ecstat8: row[42], - ethnic: row[43], - national: row[44], - armedforces: row[45], - reservist: row[46], - preg_occ: row[47], - hb: row[48], - benefits: row[49], - net_income_known: row[50].present? ? 1 : nil, - earnings: row[50], - # increfused: row[51], - reason: row[52], - reasonother: row[53], - underoccupation_benefitcap: row[54], - housingneeds_a: row[55], - housingneeds_b: row[56], - housingneeds_c: row[57], - housingneeds_f: row[58], - housingneeds_g: row[59], - housingneeds_h: row[60], - prevten: row[61], - prevloc: row[62], - # prevpco_unknown: row[65], - layear: row[66], - waityear: row[67], - homeless: row[68], - reasonpref: row[69], - rp_homeless: row[70], - rp_insan_unsat: row[71], - rp_medwel: row[72], - rp_hardship: row[73], - rp_dontknow: row[74], - cbl: row[75], - chr: row[76], - cap: row[77], - # referral_source: row[78], - period: row[79], - brent: row[80], - scharge: row[81], - pscharge: row[82], - supcharg: row[83], - tcharge: row[84], - # tcharge_care_homes: row[85], - # no_rent_or_charge: row[86], - hbrentshortfall: row[87], - tshortfall: row[88], - voiddate: row[89].to_s + row[90].to_s + row[91].to_s, - majorrepairs: row[92].present? ? "1" : nil, - mrcdate: row[92].to_s + row[93].to_s + row[94].to_s, - # supported_scheme: row[95], - startdate: date_time(row[98], row[97], row[96]), - # startdate_day: row[96], - # startdate_month: row[97], - # startdate_year: row[98], - offered: row[99], - # property_reference: row[100], - beds: row[101], - unittype_gn: row[102], - builtype: row[103], - wchair: row[104], - property_relet: row[105], - rsnvac: row[106], - la: row[107], - # row[110] removed - # row[111] is owning organisation used above - # username: row[112], - # row[113] is managing organisation used above - leftreg: row[114], - # uprn: row[115], - incfreq: row[116], - # sheltered_accom: row[117], - illness: row[118], - illness_type_1: row[119], - illness_type_2: row[120], - illness_type_3: row[121], - illness_type_4: row[122], - illness_type_8: row[123], - illness_type_5: row[124], - illness_type_6: row[125], - illness_type_7: row[126], - illness_type_9: row[127], - illness_type_10: row[128], - # london_affordable: row[129], - rent_type: row[130], - irproduct_other: row[131], - # data_protection: row[132], - declaration: 1, - } - end - - def date_time(year, month, day) - return unless year && month && day - - Time.zone.local("20#{year}", month.to_s, day.to_s) - end +private - def hhmemb(row) - [14, 15, 16, 17, 18, 19, 20].count { |idx| row[idx].present? } + def generate_identifier + self.identifier ||= SecureRandom.uuid end end diff --git a/app/models/form.rb b/app/models/form.rb index 022b9c261..fceb48b65 100644 --- a/app/models/form.rb +++ b/app/models/form.rb @@ -231,10 +231,6 @@ class Form end end - def in_crossover_period?(now: Time.zone.now) - ((end_date - 3.months) < now) && (now < end_date) - end - def inspect "#<#{self.class} @type=#{type} @name=#{name}>" end diff --git a/app/models/form_handler.rb b/app/models/form_handler.rb index 03de5e290..69b05859e 100644 --- a/app/models/form_handler.rb +++ b/app/models/form_handler.rb @@ -63,6 +63,20 @@ class FormHandler form_mappings[current_collection_start_year - year] end + def in_crossover_period?(now: Time.zone.now) + lettings_in_crossover_period?(now:) || sales_in_crossover_period?(now:) + end + + def lettings_in_crossover_period?(now: Time.zone.now) + forms = lettings_forms.values + forms.count { |form| form.start_date < now && now < form.end_date } > 1 + end + + def sales_in_crossover_period?(now: Time.zone.now) + forms = sales_forms.values + forms.count { |form| form.start_date < now && now < form.end_date } > 1 + end + private def get_all_forms diff --git a/app/models/forms/bulk_upload_lettings/checking_file.rb b/app/models/forms/bulk_upload_lettings/checking_file.rb new file mode 100644 index 000000000..8cd9ee696 --- /dev/null +++ b/app/models/forms/bulk_upload_lettings/checking_file.rb @@ -0,0 +1,27 @@ +module Forms + module BulkUploadLettings + class CheckingFile + include ActiveModel::Model + include ActiveModel::Attributes + include Rails.application.routes.url_helpers + + attribute :year, :integer + + def view_path + "bulk_upload_lettings_logs/forms/checking_file" + end + + def back_path + bulk_upload_lettings_log_path(id: "start") + end + + def year_combo + "#{year}/#{year + 1 - 2000}" + end + + def save! + true + end + end + end +end diff --git a/app/models/forms/bulk_upload_lettings/prepare_your_file.rb b/app/models/forms/bulk_upload_lettings/prepare_your_file.rb index 684ba1437..4e2c9b9a6 100644 --- a/app/models/forms/bulk_upload_lettings/prepare_your_file.rb +++ b/app/models/forms/bulk_upload_lettings/prepare_your_file.rb @@ -31,10 +31,14 @@ module Forms "#{year}/#{year + 1 - 2000}" end + def save! + true + end + private def in_crossover_period? - FormHandler.instance.forms.values.any?(&:in_crossover_period?) + FormHandler.instance.lettings_in_crossover_period? end end end diff --git a/app/models/forms/bulk_upload_lettings/upload_your_file.rb b/app/models/forms/bulk_upload_lettings/upload_your_file.rb index 1415ffe19..0f6325b75 100644 --- a/app/models/forms/bulk_upload_lettings/upload_your_file.rb +++ b/app/models/forms/bulk_upload_lettings/upload_your_file.rb @@ -1,3 +1,5 @@ +require "shellwords" + module Forms module BulkUploadLettings class UploadYourFile @@ -6,6 +8,11 @@ module Forms include Rails.application.routes.url_helpers attribute :year, :integer + attribute :file + attribute :current_user + + validates :file, presence: true + validate :validate_file_is_csv def view_path "bulk_upload_lettings_logs/forms/upload_your_file" @@ -14,6 +21,49 @@ module Forms def back_path bulk_upload_lettings_log_path(id: "prepare-your-file", form: { year: }) end + + def year_combo + "#{year}/#{year + 1 - 2000}" + end + + def next_path + bulk_upload_lettings_log_path(id: "checking-file", form: { year: }) + end + + def save! + bulk_upload = BulkUpload.create!( + user: current_user, + log_type: BulkUpload.log_types[:lettings], + year:, + ) + + if upload_enabled? + storage_service.write_file(bulk_upload.identifier, File.read(file.path)) + end + + true + end + + private + + def upload_enabled? + FeatureToggle.upload_enabled? + end + + def storage_service + @storage_service ||= Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["CSV_DOWNLOAD_PAAS_INSTANCE"]) + end + + def validate_file_is_csv + return unless file + + argv = %W[file --brief --mime-type -- #{file.path}] + output = `#{argv.shelljoin}` + + unless output.match?(/text\/csv|text\/plain/) + errors.add(:file, :not_csv) + end + end end end end diff --git a/app/models/forms/bulk_upload_lettings/year.rb b/app/models/forms/bulk_upload_lettings/year.rb index 9fa17b19e..68db63978 100644 --- a/app/models/forms/bulk_upload_lettings/year.rb +++ b/app/models/forms/bulk_upload_lettings/year.rb @@ -27,6 +27,10 @@ module Forms bulk_upload_lettings_log_path(id: "prepare-your-file", form: { year: }) end + def save! + true + end + private def possible_years diff --git a/app/models/forms/bulk_upload_sales/checking_file.rb b/app/models/forms/bulk_upload_sales/checking_file.rb new file mode 100644 index 000000000..a37be3ccb --- /dev/null +++ b/app/models/forms/bulk_upload_sales/checking_file.rb @@ -0,0 +1,27 @@ +module Forms + module BulkUploadSales + class CheckingFile + include ActiveModel::Model + include ActiveModel::Attributes + include Rails.application.routes.url_helpers + + attribute :year, :integer + + def view_path + "bulk_upload_sales_logs/forms/checking_file" + end + + def back_path + bulk_upload_sales_log_path(id: "start") + end + + def year_combo + "#{year}/#{year + 1 - 2000}" + end + + def save! + true + end + end + end +end diff --git a/app/models/forms/bulk_upload_sales/prepare_your_file.rb b/app/models/forms/bulk_upload_sales/prepare_your_file.rb index da017dbbd..5c7e775ef 100644 --- a/app/models/forms/bulk_upload_sales/prepare_your_file.rb +++ b/app/models/forms/bulk_upload_sales/prepare_your_file.rb @@ -31,10 +31,14 @@ module Forms "#{year}/#{year + 1 - 2000}" end + def save! + true + end + private def in_crossover_period? - FormHandler.instance.forms.values.any?(&:in_crossover_period?) + FormHandler.instance.sales_in_crossover_period? end end end diff --git a/app/models/forms/bulk_upload_sales/upload_your_file.rb b/app/models/forms/bulk_upload_sales/upload_your_file.rb index 3d421e9f1..7211d67a7 100644 --- a/app/models/forms/bulk_upload_sales/upload_your_file.rb +++ b/app/models/forms/bulk_upload_sales/upload_your_file.rb @@ -1,3 +1,5 @@ +require "shellwords" + module Forms module BulkUploadSales class UploadYourFile @@ -6,6 +8,11 @@ module Forms include Rails.application.routes.url_helpers attribute :year, :integer + attribute :file + attribute :current_user + + validates :file, presence: true + validate :validate_file_is_csv def view_path "bulk_upload_sales_logs/forms/upload_your_file" @@ -14,6 +21,49 @@ module Forms def back_path bulk_upload_sales_log_path(id: "prepare-your-file", form: { year: }) end + + def year_combo + "#{year}/#{year + 1 - 2000}" + end + + def next_path + bulk_upload_sales_log_path(id: "checking-file", form: { year: }) + end + + def save! + bulk_upload = BulkUpload.create!( + user: current_user, + log_type: BulkUpload.log_types[:sales], + year:, + ) + + if upload_enabled? + storage_service.write_file(bulk_upload.identifier, File.read(file.path)) + end + + true + end + + private + + def upload_enabled? + !Rails.env.development? + end + + def storage_service + @storage_service ||= Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["CSV_DOWNLOAD_PAAS_INSTANCE"]) + end + + def validate_file_is_csv + return unless file + + argv = %W[file --brief --mime-type -- #{file.path}] + output = `#{argv.shelljoin}` + + unless output.match?(/text\/csv|text\/plain/) + errors.add(:file, :not_csv) + end + end end end end diff --git a/app/models/forms/bulk_upload_sales/year.rb b/app/models/forms/bulk_upload_sales/year.rb index 361061990..2d4b48e4a 100644 --- a/app/models/forms/bulk_upload_sales/year.rb +++ b/app/models/forms/bulk_upload_sales/year.rb @@ -27,6 +27,10 @@ module Forms bulk_upload_sales_log_path(id: "prepare-your-file", form: { year: }) end + def save! + true + end + private def possible_years diff --git a/app/models/legacy_bulk_upload.rb b/app/models/legacy_bulk_upload.rb new file mode 100644 index 000000000..e80321d04 --- /dev/null +++ b/app/models/legacy_bulk_upload.rb @@ -0,0 +1,197 @@ +class LegacyBulkUpload + include ActiveModel::Model + include ActiveModel::Validations + include ActiveModel::Conversion + + SPREADSHEET_CONTENT_TYPES = %w[ + application/vnd.ms-excel + application/vnd.openxmlformats-officedocument.spreadsheetml.sheet + ].freeze + + FIRST_DATA_ROW = 7 + + def initialize(file, content_type) + @file = file + @content_type = content_type + end + + def process(current_user) + return unless valid_content_type? + + xlsx = Roo::Spreadsheet.open(@file, extension: :xlsx) + sheet = xlsx.sheet(0) + last_row = sheet.last_row + if last_row < FIRST_DATA_ROW + errors.add(:lettings_log_bulk_upload, "No data found") + else + data_range = FIRST_DATA_ROW..last_row + data_range.map do |row_num| + row = sheet.row(row_num) + # owning_organisation = Organisation.find(row[111]) + # managing_organisation = Organisation.find(row[113]) + lettings_log = LettingsLog.create!( + owning_organisation: current_user.organisation, + managing_organisation: current_user.organisation, + created_by: current_user, + ) + map_row(row).each do |attr_key, attr_val| + _update = lettings_log.update(attr_key => attr_val) + rescue ArgumentError + # TODO: determine what we want to do when bulk upload contains totally invalid data for a field. + end + end + end + end + + def valid_content_type? + if SPREADSHEET_CONTENT_TYPES.include?(@content_type) + true + else + errors.add(:lettings_log_bulk_upload, "Invalid file type") + false + end + end + + def map_row(row) + { + lettype: row[1], + # reg_num_la_core_code: row[3], + # managementgroup: row[4], + # schemecode: row[5], + # firstletting: row[6], + tenancycode: row[7], + startertenancy: row[8], + tenancy: row[9], + tenancyother: row[10], + # tenancyduration: row[11], + hhmemb: hhmemb(row), + age1: row[12], + age2: row[13], + age3: row[14], + age4: row[15], + age5: row[16], + age6: row[17], + age7: row[18], + age8: row[19], + sex1: row[20], + sex2: row[21], + sex3: row[22], + sex4: row[23], + sex5: row[24], + sex6: row[25], + sex7: row[26], + sex8: row[27], + relat2: row[28], + relat3: row[29], + relat4: row[30], + relat5: row[31], + relat6: row[32], + relat7: row[33], + relat8: row[34], + ecstat1: row[35], + ecstat2: row[36], + ecstat3: row[37], + ecstat4: row[38], + ecstat5: row[39], + ecstat6: row[40], + ecstat7: row[41], + ecstat8: row[42], + ethnic: row[43], + national: row[44], + armedforces: row[45], + reservist: row[46], + preg_occ: row[47], + hb: row[48], + benefits: row[49], + net_income_known: row[50].present? ? 1 : nil, + earnings: row[50], + # increfused: row[51], + reason: row[52], + reasonother: row[53], + underoccupation_benefitcap: row[54], + housingneeds_a: row[55], + housingneeds_b: row[56], + housingneeds_c: row[57], + housingneeds_f: row[58], + housingneeds_g: row[59], + housingneeds_h: row[60], + prevten: row[61], + prevloc: row[62], + # prevpco_unknown: row[65], + layear: row[66], + waityear: row[67], + homeless: row[68], + reasonpref: row[69], + rp_homeless: row[70], + rp_insan_unsat: row[71], + rp_medwel: row[72], + rp_hardship: row[73], + rp_dontknow: row[74], + cbl: row[75], + chr: row[76], + cap: row[77], + # referral_source: row[78], + period: row[79], + brent: row[80], + scharge: row[81], + pscharge: row[82], + supcharg: row[83], + tcharge: row[84], + # tcharge_care_homes: row[85], + # no_rent_or_charge: row[86], + hbrentshortfall: row[87], + tshortfall: row[88], + voiddate: row[89].to_s + row[90].to_s + row[91].to_s, + majorrepairs: row[92].present? ? "1" : nil, + mrcdate: row[92].to_s + row[93].to_s + row[94].to_s, + # supported_scheme: row[95], + startdate: date_time(row[98], row[97], row[96]), + # startdate_day: row[96], + # startdate_month: row[97], + # startdate_year: row[98], + offered: row[99], + # property_reference: row[100], + beds: row[101], + unittype_gn: row[102], + builtype: row[103], + wchair: row[104], + property_relet: row[105], + rsnvac: row[106], + la: row[107], + # row[110] removed + # row[111] is owning organisation used above + # username: row[112], + # row[113] is managing organisation used above + leftreg: row[114], + # uprn: row[115], + incfreq: row[116], + # sheltered_accom: row[117], + illness: row[118], + illness_type_1: row[119], + illness_type_2: row[120], + illness_type_3: row[121], + illness_type_4: row[122], + illness_type_8: row[123], + illness_type_5: row[124], + illness_type_6: row[125], + illness_type_7: row[126], + illness_type_9: row[127], + illness_type_10: row[128], + # london_affordable: row[129], + rent_type: row[130], + irproduct_other: row[131], + # data_protection: row[132], + declaration: 1, + } + end + + def date_time(year, month, day) + return unless year && month && day + + Time.zone.local("20#{year}", month.to_s, day.to_s) + end + + def hhmemb(row) + [14, 15, 16, 17, 18, 19, 20].count { |idx| row[idx].present? } + end +end diff --git a/app/views/bulk_upload_lettings_logs/forms/checking_file.html.erb b/app/views/bulk_upload_lettings_logs/forms/checking_file.html.erb new file mode 100644 index 000000000..f38178b7e --- /dev/null +++ b/app/views/bulk_upload_lettings_logs/forms/checking_file.html.erb @@ -0,0 +1,28 @@ +<% content_for :before_content do %> + <%= govuk_back_link href: @form.back_path %> +<% end %> + +
+
+ <%= form_with model: @form, scope: :form, url: bulk_upload_lettings_log_path(id: "prepare-your-file"), method: :patch do |f| %> + <%= f.hidden_field :year %> + + Upload lettings logs in bulk (<%= @form.year_combo %>) +

We’re checking the file

+ +

Once this is done, we’ll email you the outcome.

+ +

+ We’re checking for any missing data, issues or errors. Depending on the file size, this could take a few minutes. +

+ +

+ If there are errors we will email you a link to view all the errors. +

+ +

+ If there are no errors we will email you to let you know the upload is complete and that your logs have been created. +

+ <% end %> +
+
diff --git a/app/views/bulk_upload_lettings_logs/forms/upload_your_file.html.erb b/app/views/bulk_upload_lettings_logs/forms/upload_your_file.html.erb index 86dde8ae2..8964438de 100644 --- a/app/views/bulk_upload_lettings_logs/forms/upload_your_file.html.erb +++ b/app/views/bulk_upload_lettings_logs/forms/upload_your_file.html.erb @@ -2,16 +2,24 @@ <%= govuk_back_link href: @form.back_path %> <% end %> -<%= form_with model: @form, scope: :form, url: bulk_upload_lettings_log_path(id: "upload-your-file"), method: :patch do |f| %> - <%= f.govuk_error_summary %> +
+
+ <%= form_with model: @form, scope: :form, url: bulk_upload_lettings_log_path(id: "upload-your-file"), method: :patch do |f| %> + <%= f.hidden_field :year %> -
- Upload your file goes here -
+ <%= f.govuk_error_summary %> -
- year selected <%= @form.year %> -
+ Upload lettings logs in bulk (<%= @form.year_combo %>) +

Upload your file

- <%= f.govuk_submit %> -<% end %> +

+ All data must be complete and formatted correctly (CSV only) - if not, we will reject the entire file. +

+ + <%= f.govuk_file_field :file, + label: { text: "Upload file" } %> + + <%= f.govuk_submit "Upload" %> + <% end %> +
+
diff --git a/app/views/bulk_upload_sales_logs/forms/checking_file.html.erb b/app/views/bulk_upload_sales_logs/forms/checking_file.html.erb new file mode 100644 index 000000000..b729abc09 --- /dev/null +++ b/app/views/bulk_upload_sales_logs/forms/checking_file.html.erb @@ -0,0 +1,28 @@ +<% content_for :before_content do %> + <%= govuk_back_link href: @form.back_path %> +<% end %> + +
+
+ <%= form_with model: @form, scope: :form, url: bulk_upload_sales_log_path(id: "prepare-your-file"), method: :patch do |f| %> + <%= f.hidden_field :year %> + + Upload sales logs in bulk (<%= @form.year_combo %>) +

We’re checking the file

+ +

Once this is done, we’ll email you the outcome.

+ +

+ We’re checking for any missing data, issues or errors. Depending on the file size, this could take a few minutes. +

+ +

+ If there are errors we will email you a link to view all the errors. +

+ +

+ If there are no errors we will email you to let you know the upload is complete and that your logs have been created. +

+ <% end %> +
+
diff --git a/app/views/bulk_upload_sales_logs/forms/upload_your_file.html.erb b/app/views/bulk_upload_sales_logs/forms/upload_your_file.html.erb index a178339e8..36e7240dc 100644 --- a/app/views/bulk_upload_sales_logs/forms/upload_your_file.html.erb +++ b/app/views/bulk_upload_sales_logs/forms/upload_your_file.html.erb @@ -2,16 +2,24 @@ <%= govuk_back_link href: @form.back_path %> <% end %> -<%= form_with model: @form, scope: :form, url: bulk_upload_sales_log_path(id: "upload-your-file"), method: :patch do |f| %> - <%= f.govuk_error_summary %> +
+
+ <%= form_with model: @form, scope: :form, url: bulk_upload_sales_log_path(id: "upload-your-file"), method: :patch do |f| %> + <%= f.hidden_field :year %> -
- Upload your file goes here -
+ <%= f.govuk_error_summary %> -
- year selected <%= @form.year %> -
+ Upload sales logs in bulk (<%= @form.year_combo %>) +

Upload your file

- <%= f.govuk_submit %> -<% end %> +

+ All data must be complete and formatted correctly (CSV only) - if not, we will reject the entire file. +

+ + <%= f.govuk_file_field :file, + label: { text: "Upload file" } %> + + <%= f.govuk_submit "Upload" %> + <% end %> +
+
diff --git a/app/views/logs/bulk_upload.html.erb b/app/views/logs/bulk_upload.html.erb index 0f1b544f7..6d06af7a3 100644 --- a/app/views/logs/bulk_upload.html.erb +++ b/app/views/logs/bulk_upload.html.erb @@ -1,6 +1,6 @@ <% content_for :title, "Bulk upload" %> -<%= form_for @bulk_upload, url: bulk_upload_lettings_logs_path, method: "post" do |f| %> +<%= form_for @bulk_upload, scope: :bulk_upload, url: bulk_upload_lettings_logs_path, method: "post" do |f| %> <%= f.govuk_error_summary %> <%= f.govuk_file_field :lettings_log_bulk_upload, diff --git a/config/initializers/feature_toggle.rb b/config/initializers/feature_toggle.rb index 8fc2cd7d4..b315ff723 100644 --- a/config/initializers/feature_toggle.rb +++ b/config/initializers/feature_toggle.rb @@ -26,4 +26,8 @@ class FeatureToggle def self.bulk_upload_logs? !Rails.env.production? end + + def self.upload_enabled? + !Rails.env.development? + end end diff --git a/config/locales/en.yml b/config/locales/en.yml index 6c2d7db46..4f6f6c5b5 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -49,6 +49,16 @@ en: attributes: year: blank: You must select a collection period to upload for + forms/bulk_upload_lettings/upload_your_file: + attributes: + file: + blank: Select which file to upload + not_csv: Your file must be in CSV format + forms/bulk_upload_sales/upload_your_file: + attributes: + file: + blank: Select which file to upload + not_csv: Your file must be in CSV format activerecord: errors: diff --git a/db/migrate/20221128130843_create_bulk_uploads.rb b/db/migrate/20221128130843_create_bulk_uploads.rb new file mode 100644 index 000000000..adb3385b5 --- /dev/null +++ b/db/migrate/20221128130843_create_bulk_uploads.rb @@ -0,0 +1,14 @@ +class CreateBulkUploads < ActiveRecord::Migration[7.0] + def change + create_table :bulk_uploads do |t| + t.references :user + t.text :log_type, null: false + t.integer :year, null: false + t.uuid :identifier, null: false + + t.timestamps + end + + add_index :bulk_uploads, :identifier, unique: true + end +end diff --git a/db/schema.rb b/db/schema.rb index 70daf01b5..4f7d267ce 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -14,6 +14,17 @@ ActiveRecord::Schema[7.0].define(version: 2022_12_06_081127) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" + create_table "bulk_uploads", force: :cascade do |t| + t.bigint "user_id" + t.text "log_type", null: false + t.integer "year", null: false + t.uuid "identifier", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["identifier"], name: "index_bulk_uploads_on_identifier", unique: true + t.index ["user_id"], name: "index_bulk_uploads_on_user_id" + end + create_table "data_protection_confirmations", force: :cascade do |t| t.bigint "organisation_id" t.bigint "data_protection_officer_id" diff --git a/spec/features/bulk_upload_lettings_logs_spec.rb b/spec/features/bulk_upload_lettings_logs_spec.rb index e9a05b07d..3df49fe44 100644 --- a/spec/features/bulk_upload_lettings_logs_spec.rb +++ b/spec/features/bulk_upload_lettings_logs_spec.rb @@ -3,13 +3,26 @@ require "rails_helper" RSpec.describe "Bulk upload lettings log" do let(:user) { create(:user) } + let(:stub_file_upload) do + vcap_services = { "aws-s3-bucket" => {} } + mock_storage_service = instance_double("S3Service") + + allow(ENV).to receive(:[]) + allow(ENV).to receive(:[]).with("VCAP_SERVICES").and_return(vcap_services.to_json) + + allow(Storage::S3Service).to receive(:new).and_return(mock_storage_service) + allow(mock_storage_service).to receive(:write_file) + end + before do + stub_file_upload sign_in user end + # rubocop:disable RSpec/AnyInstance context "when during crossover period" do it "shows journey with year option" do - Timecop.freeze(2023, 6, 1) do + Timecop.freeze(2022, 6, 1) do visit("/lettings-logs") expect(page).to have_link("Upload lettings logs in bulk") click_link("Upload lettings logs in bulk") @@ -30,9 +43,30 @@ RSpec.describe "Bulk upload lettings log" do click_button("Continue") expect(page).to have_content("Upload your file") + click_button("Upload") + + allow_any_instance_of(Forms::BulkUploadLettings::UploadYourFile).to receive(:`).and_return("not a csv") + + expect(page).to have_content("Select which file to upload") + attach_file "file", file_fixture("2021_22_lettings_bulk_upload.xlsx") + click_button("Upload") + + allow_any_instance_of(Forms::BulkUploadLettings::UploadYourFile).to receive(:`).and_return("text/csv") + + expect(page).to have_content("Your file must be in CSV format") + attach_file "file", file_fixture("blank_bulk_upload_sales.csv") + expect { + click_button("Upload") + }.to change(BulkUpload, :count).by(1) + + expect(page).to have_content("Once this is done") + click_link("Back") + + expect(page).to have_content("Upload lettings logs in bulk") end end end + # rubocop:enable RSpec/AnyInstance context "when not it crossover period" do it "shows journey with year option" do diff --git a/spec/features/bulk_upload_sales_logs_spec.rb b/spec/features/bulk_upload_sales_logs_spec.rb index 67187ff78..b69b6e237 100644 --- a/spec/features/bulk_upload_sales_logs_spec.rb +++ b/spec/features/bulk_upload_sales_logs_spec.rb @@ -3,13 +3,26 @@ require "rails_helper" RSpec.describe "Bulk upload sales log" do let(:user) { create(:user) } + let(:stub_file_upload) do + vcap_services = { "aws-s3-bucket" => {} } + mock_storage_service = instance_double("S3Service") + + allow(ENV).to receive(:[]) + allow(ENV).to receive(:[]).with("VCAP_SERVICES").and_return(vcap_services.to_json) + + allow(Storage::S3Service).to receive(:new).and_return(mock_storage_service) + allow(mock_storage_service).to receive(:write_file) + end + before do + stub_file_upload sign_in user end + # rubocop:disable RSpec/AnyInstance context "when during crossover period" do it "shows journey with year option" do - Timecop.freeze(2023, 6, 1) do + Timecop.freeze(2023, 5, 1) do visit("/sales-logs") expect(page).to have_link("Upload sales logs in bulk") click_link("Upload sales logs in bulk") @@ -30,9 +43,30 @@ RSpec.describe "Bulk upload sales log" do click_button("Continue") expect(page).to have_content("Upload your file") + click_button("Upload") + + allow_any_instance_of(Forms::BulkUploadSales::UploadYourFile).to receive(:`).and_return("not a csv") + + expect(page).to have_content("Select which file to upload") + attach_file "file", file_fixture("2021_22_lettings_bulk_upload.xlsx") + click_button("Upload") + + allow_any_instance_of(Forms::BulkUploadSales::UploadYourFile).to receive(:`).and_return("text/csv") + + expect(page).to have_content("Your file must be in CSV format") + attach_file "file", file_fixture("blank_bulk_upload_sales.csv") + expect { + click_button("Upload") + }.to change(BulkUpload, :count).by(1) + + expect(page).to have_content("Once this is done") + click_link("Back") + + expect(page).to have_content("Upload sales logs in bulk") end end end + # rubocop:enable RSpec/AnyInstance context "when not it crossover period" do it "shows journey with year option" do diff --git a/spec/fixtures/files/blank_bulk_upload_sales.csv b/spec/fixtures/files/blank_bulk_upload_sales.csv new file mode 100644 index 000000000..4771d9fc9 --- /dev/null +++ b/spec/fixtures/files/blank_bulk_upload_sales.csv @@ -0,0 +1,118 @@ +Question,What is the purchaser code?,What is the day of the sale completion date? - DD,What is the month of the sale completion date? - MM,What is the year of the sale completion date? - YY,[BLANK],Was the buyer interviewed for any of the answers you will provide on this log?,Age of Buyer 1,Age of Buyer 2 or Person 2,Age of Person 3,Age of Person 4,Age of Person 5,Age of Person 6,Gender identity of Buyer 1,Gender identity of Buyer 2 or Person 2,Gender identity of Person 3,Gender identity of Person 4,Gender identity of Person 5,Gender identity of Person 6,Person 2's relationship to lead tenant,Person 3's relationship to lead tenant,Person 4's relationship to lead tenant,Person 5's relationship to lead tenant,Person 6's relationship to lead tenant,Working situation of Buyer 1,Working situation of Buyer 2 or Person 2,Working situation of Person 3,Working situation of Person 4,Working situation of Person 5,Working situation of Person 6,What is the buyer 1's ethnic group?,What is buyer 1's nationality?,What is buyer 1's gross annual income?,What is buyer 2's gross annual income?,Was buyer 1's income used for a mortgage application?,Was buyer 2's income used for a mortgage application?,"What is the total amount the buyers had in savings before they paid any deposit for the property? + +To the nearest £10",Have any of the buyers previously owned a property?,[BLANK],What was buyer 1's previous tenure?,What is the local authority of buyer 1's last settled home,Part 1 of postcode of buyer 1's last settled home,Part 2 of postcode of buyer 1's last settled home,Do you know the postcode of buyer 1's last settled home?,Was the buyer registered with their PRP (HA)?,Was the buyer registered with the local authority?,Was the buyer registered with a Help to Buy agent?,Was the buyer registered with another PRP (HA)?,Does anyone in the household consider themselves to have a disability?,Does anyone in the household use a wheelchair?,How many bedrooms does the property have?,What type of unit is the property?,Which type of building is the property?,What is the local authority of the property?,Part 1 of postcode of property,Part 2 of postcode of property,Is the property built or adapted to wheelchair-user standards?,What is the type of shared ownership sale?,"Is this a resale? + +Shared ownership","What is the day of the practical completion or handover date? - DD + +Shared ownership","What is the month of the practical completion or handover date? - MM + +Shared ownership","What is the year of the practical completion or handover date? - YY + +Shared ownership","What is the day of the exchange of contracts date? - DD + +Shared ownership","What is the month of the exchange of contracts date? - MM + +Shared ownership","What is the year of the exchange of contracts date? - YY + +Shared ownership","Was the household re-housed under a local authority nominations agreement? + +Shared ownership","How many bedrooms did the buyer's previous property have? + +Shared ownership","What was the type of the buyer's previous property? + +Shared ownership","What was the full purchase price? + +Shared ownership","What was the initial percentage equity stake purchased? + +Shared ownership","What is the mortgage amount? + +Shared ownership","Does this include any extra borrowing? + +Shared ownership","How much was the cash deposit paid on the property? + +Shared ownership","How much cash discount was given through Social Homebuy? + +Shared ownership","What is the basic monthly rent? + +Shared ownership","What are the total monthly leasehold charges for the property? + +Shared ownership",What is the type of discounted ownership sale?,"What was the full purchase price? + +Discounted ownership","What was the amount of any loan, grant, discount or subsidy given? + +Discounted ownership","What was the percentage discount? + +Discounted ownership","What is the mortgage amount? + +Discounted ownership","Does this include any extra borrowing? + +Discounted ownership","How much was the cash deposit paid on the property? + +Discounted ownership","What are the total monthly leasehold charges for the property? + +Discounted ownership",What is the type of outright sale?,"What is the 'other' type of outright sale? + +Outright sale",[BLANK],"What is the full purchase price? + +Outright sale","What is the mortgage amount? + +Outright sale","Does this include any extra borrowing? + +Outright sale","How much was the cash deposit paid on the property? + +Outright sale","What are the total monthly leasehold charges for the property? + +Outright sale","Which organisation owned this property before the sale? + +Organisation's CORE ID",Username,BLANK,Has the buyer ever served in the UK Armed Forces and for how long?,[BLANK],Are any of the buyers a spouse or civil partner of a UK Armed Forces regular who died in service within the last 2 years?,"What is the name of the mortgage lender? + +Shared ownership","What is the name of the 'other' mortgage lender? + +Shared ownership","What is the name of the mortgage lender? + +Discounted ownership","What is the name of the 'other' mortgage lender? + +Discounted ownership","What is the name of the mortgage lender? + +Outright sale","What is the name of the 'other' mortgage lender? + +Outright sale",Were the buyers receiving any of these housing-related benefits immediately before buying this property?,"What is the length of the mortgage in years? + +Shared ownership","What is the length of the mortgage in years? + +Discounted ownership","What is the length of the mortgage in years? + +Outright sale","How long have the buyers been living in the property before the purchase? + +Discounted ownership",Are there more than two joint purchasers of this property?,"How long have the buyers been living in the property before the purchase? + +Shared ownership",Is this a staircasing transaction?,Data Protection question,Was this purchase made through an ownership scheme?,"Is the buyer a company? + +Outright sale",Will the buyers live in the property?,Is this a joint purchase?,Will buyer 1 live in the property?,Will buyer 2 live in the property?,"Besides the buyers, how many people live in the property?","What percentage of the property has been bought in this staircasing transaction? + +Shared ownership","What percentage of the property does the buyer now own in total? + +Shared ownership","What was the rent type of the buyer's previous property? + +Shared ownership","Was a mortgage used for the purchase of this property? + +Shared ownership","Was a mortgage used for the purchase of this property? + +Discounted ownership","Was a mortgage used for the purchase of this property? + +Outright sale" +Values,Max 9 digits,1 - 31,1 - 12,19 - 23,,1 or null,"15 - 110 +or R",1 - 110 or R,,,,,"M, F, X or R",,,,,,"P, C, X or R",,,,,0 - 10,,,,,,1 - 19,"12 -13, 17 -19",0 - 99999,,1 or 2,1 or 2,0 - 999990,1 - 3,,1 - 7 or 9,ONS CODE - E + 9 digits,XXX(X),XXX,1 or null,,,,,1 - 3,1 - 3,1 - 9,1 - 4 or 9,1 or 2,ONS CODE E + 9 digits,XXX(X),XXX,1 - 3,"2, 16, 18, 24, 28 or 30-31",1 or 2,1 - 31,1 - 12,19 - 23,1 - 31,1 - 12,19 - 23,1 - 3,1 - 9,1 - 4 or 9,0 - 999999,0 - 100,0 - 999999,1 - 3,0 - 999999,,0 - 999.99,,"8, 9, 14, 21, 22, 27 or 29",0 - 999999,,0 - 100,0 - 999999,1 - 3,0 - 999999,0 - 999.99,10 or 12,,,0 - 999999,,1-3,0 - 999999,0-999.99,Up to 7 digits,Username of CORE account this sales log should be assigned to,,3 - 8,,4 - 7,1 - 40,,1 - 40,,1 - 40,,1 - 4, Integer <=60, Integer <=60, Integer <=60, Integer <=80,1 - 3, Integer <=80,1 - 3,1,1 - 3,1 - 2,1 - 2,1 - 2,1 - 2,1 - 2,0 - 5,1 - 100,1 - 100,1-3 or 9-10,1 - 2,1 - 2,1 - 2 +Can be Null?,No,,,,,No,No,"If fields 14, 19 and 25 are all also null","If fields 15, 20 and 26 are all also null","If fields 16, 21 and 27 are all also null","If fields 17, 22 and 28 are all also null","If fields 18, 23 and 29 are all also null",No,"If fields 8, 19 and 25 are all also null","If fields 9, 20 and 26 are also null","If fields 10, 21 and 27 are all also null","If fields 11, 22 and 28 are all also null","If fields 12, 23 and 29 are all also null","If fields 8, 14 and 25 are all also null","If fields 9, 15 and 26 are all also null","If fields 10, 16 and 27 are all also null","If fields 11, 17 and 28 are all also null","If fields 12, 18 and 29 are all also null",If field 6 = 1,"If fields 8, 14 and 19 are all also null","If fields 9, 15 and 20 are all also null","If fields 10, 16 and 21 are all also null","If fields 11, 17 and 22 are all also null","If fields 12, 18 and 23 are all also null",If field 6 = 1,,,If field 116 = 2,If field 32 is null,If field 116 = 2,If field 6 = 1,,,If field 6 = 1,No,If field 43 = 1,,If fields 41 and 42 BOTH have valid entries,Yes,,,,If field 6 = 1,,No,,,,,,,If field 113 = 2 or 3,,,,,,,,,"If field 113 = 2 or 3 +OR +field 39 = 3 - 7 or 9",,If field 113 = 2 or 3,,,,,"If field 57 is null, 2, 16, 24 or 28",If field 113 = 2 or 3,,If field 113 = 1 or 3,If field 76 is null,"If field 76 is null, 9 or 14","If field 76 is null, 8, 21 or 22",If field 113 = 1 or 3,,,,If field 113 = 1 or 2,If field 84 is null or 10,,If field 113 = 1 or 2,,,,,No,Yes,,No,,No,If field 113 = 2 or 3,"If field 113 = 2 or 3 +OR +If field 98 is not 40",If field 113 = 1 or 3,"If field 113 = 1 or 3 +OR +If field 100 is not 40",If field 113 = 1 or 2,"If field 113 = 1 or 2 +OR +If field 102 is not 40",No,If field 113 = 2 or 3,If field 113 = 1 or 3,If field 113 = 1 or 2,If field 113 = 1 or 3,If field 116 = 2,If field 113 = 2 or 3,If field 113 = 2 or 3,No,No,If field 113 = 1 or 2,If field 113 = 1 or 2,No,No,If field 116 = 2,No,If field 113 = 2 or 3,If field 113 = 2 or 3,"If field 113 = 1 or 2 +OR +If field 39 = 3 - 9",If field 113 = 2 or 3,If field 113 = 1 or 3,If field 113 = 1 or 2 +Bulk upload format and duplicate check,Yes,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +Field number,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125 diff --git a/spec/fixtures/files/excel_as_csv.csv b/spec/fixtures/files/excel_as_csv.csv new file mode 100644 index 0000000000000000000000000000000000000000..f1b7de0bbd3a9f1cce894328a3c5e2df7c2760af GIT binary patch literal 24693 zcmeEuWq2G-lBU37ijMw@+w{y3*JGV3U z^H$H(74b$!WMxN2WmJW-T3!nL0~!c4G&G2ylerSe|MG_j4H$v6qQj7NQ+9gvSMR0(JM2w`DOtgv4qm4!W`0Li=`}0xoCuy{x*~w zF0{=!=c?pur-K3>$N9xZNhHUS^ktugh@r^O)Wij}d!|A%vQoK>2dkyG^QASSQ&7MeMM>0ayA`_M4xZWN}%Zc+{8&mr(-p&a6DlGv~ zd4Lp@ftqyHl7Os$`!b|{F5O-ey+K2G^~(Af`SY;YvxQfw1k7ucj2`T2QiQF2rYM*K zHFh{VD3!>Qb3Dd-KN3=Y0fe=(K=jlv+1PLnDR2Z&icimYu(>{s`0f_5c?w?-go`DA zakix>HsYMp5N9H6Z>jz;x3vnei)P8-lbpqFpo3aX5uPh1R4+d%_C`ANTfH-GH(gHG zy$ybDp5lbeu2ei_tJ4#W(kp;}y@n7N4=Ijk5>?;qa!r6A6Q-ve2~x&-(|ka|a)3 zW?1kk(qvmaf`y$9ZxCcs`?RL2=r_Guvwj`v4(`$SM4T-fSsySDY$VsDiEf(CpjU|OVHQN!x-2w4V$p7SRY(>W+aVZ2`Hr@5&SNk*q`~Kx zq!|Zu(WJ;Ucx91fj1Kv#c6cw_4AExBC1;56~-8A52UK{oPwS=~lu;Qwt0fhpt?pL<>MEKR}K7H&lYG`O$X2vi_XF$%`QJ?TYs4!H-bGd}`^s5hJb*20qNMaVn&W}*&r!=TwGByoo zRYHIg(qKUw#*rem(2k8TBRvpKEa;JN^PdrqRV=6=Hk!elu%yz_5{d4gtH{e%ohY5{ z`4t%}3|7O)R*c^;7v30#6OF?dG$E`N2Nob=a+||*!X{UHI0K2PWbs*}&aqB-2-{Fv z$-B2-Oy4Opr7#4vS6ifvtlGCUre1_iesGZ(m&$Ms%3BHjHk18Rg<%{(Ej z#+od-x9n_dd}+b4$ro~IS|wESeWT00>K7=Mhh(Yl<3uXMX=_8co@2imX&Df!Vrea8 z|L#r_&-*j?BQfNshuIYBcyz+l=g?z(lGFm%WBkyewEo-Ijkxmu!qL|GzUcX{kh`!F z$MjNUrJ?ZX)k(dgVOR-qrITwo3;sgnSC&H$4aJm$LC-qbPbROM{mdY#Af=sXqx~Sv zg8Hc;x;;sRz%lMh%1Lvkrf@=@7wRL&hVC0p2Dm2Q#5LuP#u{tGZW;2U)vO9*2S`Vb zNO=(wl_tZWzU&CjHYN*pYN_;+zC0`T>S@RsqSub3q_UqqJGB0yTpzRyE$d*J?XynB zn}nQH6%`r7(QY14XGpr9g%EX3Pg*RO`>EB?5mjm=9DSAU_^- zpo58!E zTYSO>@GQTi6+<1u$LYNYy4`;^IQNjY|2R9zP%)ylnFzxfdqjW|JYqLkawXi-BzNd+2i~hJ{2oR7KvwxH( zu>O84V{d9?e>q)yd$_E-vEuOZ#N+K;qwl1$vVqU$ zk2)pH=5Jr?=vaJFvv5Chw7c{0zGHPG=vaHTxNmtmLwvhAeKLLCJKEc@tx3#Sf4F(M z-5c6A+B;dAi_Q+#J^i{C@ik~-C^mTNeq@F8;piT)G?#pFe(I&VKLS{iES+(A+Bv;& zb@vL&yddAh;^pD-!h77`nq%nM8+s<)+5oJOHb2izL^D(#EWFxxPCc9sIIqQP$y8F< zhw|~)@aQ%}?mb=Jf9G+-=apXd!1L75##uFbvT{G+>-1RO?(SHQe0zSptlaKp7#e%I zxWDFjuyBf8B3vDl_2OG~bMZ*~I6#sA)SDdKT0Ey`cf4~7sN&Ah?i`Jhs}S7_jUn)Q z+E`f<*+4uyC5%UtwOYeLBwV$TaxFI4*=04xu)E@TeocG0nmyW^x?9A9z+8Wr?%aiTb#oXN4Z;i~ljaMeMG(!9sBr&e2_k4d~Zx=bx6FhX85Z}t*(o+n~ z+vj{*r^iRzRHX$^bIJUoawDkAh=ivdd@@DJG0N;X#gC2KVQV)0a;T398Ftsq<=sJY zasb?w)kls=H?scQ-}a>AO2~v+5c%>O`N}R%d>6cJv?S^;4HtYg2ENTZB_v zlnoE3yIswxY3M3zn5ZPJ)g5+r4EN^^LzdQPh)0fzM9R3ucXo8l=5=6*6^Ce9GW@W- zq7FCxw4=BPmmpqPE~F{$x^Q(~QS5`EDBGIFeB=&$2s3PUrewS#_Y)~#*z8WJ=tZtP z1y^RtIQ3TGH~SdaqKT0Ny;W`^d!j06V$r@TR!g6{rUYHNyc688HCK^xNnxL1V@t-T zD!sxS?((98@@WYnLFcrh#z$N&>CgCeKWW?hy0o;$yY`!OMk8mF>=|ca8ep--ElTr0 z@0zxzY!?O1nz~aN*Jwy_KjGNdEz%aW&rC0^h~=;O&08+>LXr0Jh><0|>R;83cbrcz9mVZ;n@&m9@m|{}owut@P)kL`$t+o0XOWkCKb~Cj!)(H-NmW~y+gz*Fg zr%b8@t+s4Z{3!r3^R*rhdmP6sdooAP>_#6+k~Zdmd^HIc5VOluBf`jRZ!l zjMWL7*C3Rp=+&j(ipeYzqSx}wN2!%Vs^zooJOcUhxCC7&EZ)=%FPj1T5i6G5LatC1OiqiK^)CrrubOa}FrW2gi zEFBu7PXmBHcdR8qA2R9o5}#dXfN~S!?hu~+2$5Q(Er5=0%{F;{xnOx7Z!xezuYF{0 znQRulKD_Doj@0Qi;%YYfWq)&>$AnHEO9JtaGKQbqs0qa6%{z()8Z<{?)I;$$g(2q2 zmDf`I>Kb&r)>Tg<_te~Mt%l*{HT7s-EH*psC|44;%7&X+xQK|!T$(W z98;ODn!P@0V@n-4(|nwE&(6`u1D`xHavJI#2%v;6HhNPoik9yyvrSOT#!J+AQ%381 zZ*x!{#c^wHJI)&Zd0P7DtvUHB1_<4mliiUFozd!b$g6pjuk5y|M>3!p@yb)&`HdlT zaS5nc$evg1>gRc3F-R^C_F7d5sU9g(%9Yt8!SrZQz_OEsxcDhL49r41d0pV~YD`Sb3IPzE!~+>E zpgH#^BBEAeQt|>dah9@xXIP66qF+L3`ecUc;s#|wXmDmxzhev*NSKI*gwhN@8)}H> zmj$uHnI-*BF_~uk$uv=5q-u~Q0VwO|iEq$|bS00h#ki8zG@;u~Wu4F*y&?+@bv=zs zN!b&DvobQ!&yqkb@8gMQ(+me{*1c<*y=wy8Mp?{?Z(>ZyPU;9Q%L+1Qg|iat0B(b| zaZZ|uuBy|rM?@3DL1X`3K{P`q#%VFcmHea*`AT}xjAS>p4j}s8S0r$FvbezEWwHTJ z%5Bxl6aP0w>?;ou{l8?uAv-Nbyc=V`8?(F{8^0R^&qD$fURnaJg|i~-0IGwvu}+I& zR|eCvT0|3~i)J+23*y{Xfjqco#pN+32Jy>(S)jjnL-XE^>U%dx!i!YB!0|QAb^~Vbwx>h7@Kl0N885^MiMiDM!w>J3^K_&M;$SNkFqi6qX4p? zgaWf$VpU221UDs7Kuz#8pCl9|{`658;WT~bx^Ib43vp-_H%6OH*MMS7WK3^sGiB{p?8%b(eAxEf`$1{$)h6hIy1mE| z0m;^{>d=R@`O|2J^pftBC;1LOim(rh+sLYErj;-jGE{dKQioNG4np~xR}WO>OnkZ7 zOn`4=FiSCCCjmWRFpk`-PRU=tv?(8F-r$f~GKRHR-^dSgB{lp)#9@%^fB~E%F-7&G^mo zqEJcqNTfY13rM839UQs3dyUiv`uFDVGh}y3Kx9p>z9Y{es@rWcR{+Ri_wr;|>F?YY zzswQ9Z~0XM$Z@zKie=F7B-2fwiubro6k>jd8_i5dNhP#U>f|~^PQy#`COlHE{>vbyNA<_$Tu3^W7aWIA6&3f}>B%*>7~8&-p4HUN(3;SSG3S`JF^r zDQ^i?n3JO_;kk~bFWP@ue`dRHZ&uXE=jo7~U)xqiSYHm#!J7*IIFAR&nY7BFt#~R< zb{O{CYz=3@L0jDAgolmNoy@V7NoJ+wSTz#72>gZqE~+)`FMP$tL9;xV%K0Xh4H+1* zHk}i50SX^>b`>bvCjzmEww2ad)u0|4{<_IZ&8Jg4s~$S?S|gRe3AiDM42zyFi&Whl z?!=ocxu^7OlGs9-O`*Af_pO@M4y}aV4iFUymeo-9koaW6O3mdUo1Z6)`OY*;yiw}x zBkaJE&3>b~DDD|bscdH0CaO4LKT4lroAXM zc!N`jn}DQSxCOU7Nr!>wW9S!`SC>`wk|GZC1!~_1~bMKrR>)rRU zyv_(sj5ZN7kT{0(@)>WZ8`;*yCg?3udh`hqqG({2^I1! zq6d(L1d^T!)(SXPxM4O<+(P;5xS$-K6emMV{DX&!H9}SxT2DhYLO)+elF@e)E9hvH zH=j&1p;YGl^|PBkYLa6nAW*jE98V1kg|7wcL*?N4*?kyhqNK1lYM1CUc@`=CJyO9U zKQ7n|vV)R9PpOupZDSkAF8>2ck}yF_9x^8+r+^Y$GD;DuP$%J5eo^cIq_78`nT!f; zq45~5G|6=&YMM&YgfcVV!FQVM{}{AS0nq=3l$?_I2~G-&!e)ep;0Xu{TtYA_st#IH zRw9k){~06P2*1MSurz5U+(_rxn&pbNkTZqT;N5d;6HJMp>ku&wBRRi^>+~4Z-l5lM zGk>O)yyED=?~({vo&_uLD{IXy-OxdB@R63uYtNt&=urbOar$fR?_YnU&+Io$HGAaS zJDw{#D@eAc_fL?2>gW_yeDV3Jn&HO4IL2p+pPQiHaw`sE^P*_SqO`1JS9tN0QnxCD z$_Zf@f-*VBL9iKY2PXkn-Yw@^uo=QAhI&COQuhEB&b+tTgU6ZTWNVk-Yhrxqs1`ht zP;QClguU}oY1K8ezTQ0~JT*L}KGfm)coai0u|DU1vgMs3=WxGW_j8hsh{)FZm0qM#m0QvER0->jYghF$Bn3`|-!&ig0b1GTiUUQ0*YLR)bBC+ms&&c%Nc!ZeYD zsYhl>-JV}I5n<(uEDkz<5AQDv4iPF5cPsM`{OWwg3Ks?|;hzmK8?odZC2W}Jh%tz~@;o#qGCA&Kf`h@6$E+VIWAN2?%eAMT(q;U$G=9I*w)XAGMN=7F{sK(l~ zDjSHdw55aaI#D3;x;HHY1gix#S`=%35NU;^rR2-4aza_d!nXrAsJW1kv*(sk@C;7*|1f-ilngyi$KpG3Ar$Cwt zq}L-?2ZR?O&f4z~#g?WY2n9j!Oy41f?bSQ<1$B0Ohd8#a13>r-^v?Gk z64?3=0wE}vbJRN|wv`%sw*|XPdxzwL|hk=lu<5xd} zYaj~=IIqEG(_1wf0;<)w8;O;K&GZd%p$)l* znFP|3KAl}(-MH6B$K*=%<46Qn6~5JKh+Zl($*c9sAEV&G~JbW>YMV(980v|=nZ5Y2QY3Xf|jHGrAr+7l$zz-qY_ucldBxOLRs=G#N3 zh2s^KthV@$``U7QxEH1`>J3)4U?(E*n-*+V*EigOl0ePX{f(!E>2qw`uLu5?X7go~ zl@7P{5SEl}SOSNIgo$6>?U6M`a0PjuF2TI$LWBp|YoNY!sg`-te@I8l29yjALWKKS zg)o>YU$;r+=QRRvx!qi3^7GW#fy{Ki0@Ix*4!XU!RKETx6f3oLEh#Om6ND-by4{+_ z_4k`a%^B)D3!9Ymi%uHE{HR7*0IV}Ct47Y`KrOb=Eg z5H#cjG&12Kg~Ob>@M!|cCrbPrhY*xSn*GY0Uw#O1nzZ}XIYWN*aErA2wK-FMvT&Mo z`}H{se%3TC<@*hJOJV;4le-jTjoV_i-|$Z}+$|^j4f#t^*8eo)WHNSAU#k0LqSX0V znRQ zFGbj2fyFHFFF0&PV#Qjzupavx_AP+U0hXhjzAH2?e@4&@v+(^ng-hT7sgNWgN`Xu8 z0IiTEVO*g_-~gqNUQ&%xC*Q$e5wVa@vXK&A9)$?!LMvoL2oF=g8EGOgg}&F3r~&Rm zC*(pX15>{p=`1jXq1Th>7Vbh98bk;h$ad00pKR6pxwTx2!HVucpMvD zFi^4_w*LiDrX#{EMefcTHw+zk)BpKUaPVHVkX7Ik1{SH|66h5U^Be-FKTEPx-pC*3 zJA_Yz6dIz0{eSGZs&@VA_(@ow@P8rVB*}A2kUImh=zG5tCBa>2ha3oLW9qjetp#E+ z^tuww!d-j~c@SF0)bB)k3k3NT073)}ttSBeL5LE`&ll-minPCnC@|8F-eCT@Nc(#{ z{uNzp$e;e1Nc*>VoYdI+=82;D(jGra{ojuJC$8E_4LgAjnDUoZ2~+qkk$;y6r;#NG zC^K^$!ltn$_b7GV>o1svGzk?7ErO-NiiCw)36PXz@$xevYeF9}H=B?~0$u2PfyFzx zYMqcXp$N>)Hl(9K7lvLBqEonP-H7%9#TFW*e3armD%BeL>bA_pG}?GxzbTfUTrPoqozt=8# zKMrv-5<6J9rscGgjdqA9;dLv>#>1{6Nz$%A0g(6)N5OG#N{nZak4?mrG+v9_H6ZY6 z6}QxCkv)1j?!_F~X{^N3xNVbjPMe6${FGoji0#=PMCP^2z~jsn8;>JNz7}WaoX!_} z{Ee^V)Ej5Pk_IYO(ItBjyA2!h)t5}|8~e3511Ix0F`j0yVzu?Xp;#@a0+QdE26X|k&EopKYlY*{K^{Ex35H?H>{Nd@GozoH1^N0(&)Z(dIarw@Fb zds1S{EO^bjsD;_i1(dfBo}?1y<{LD$7rDcV8bxZpPgX3c1D)=|J&Af z`gcirtc*nuT`l__kB4%*}zsE%UfMHC5x^iMETipO1PXT-kox} zn?D+kxdv_{s+ks^slbTjMWEq^tZx;vVy&!dYgyK*%n43a~4*SsKcepYNtjbBWm7@xzEhOU_?2%7s(cwm_P+I|{l_9;c(|8gQyvD@nxQ7xWPSmNA z$;+!&PU{s*YvGYoiR~HKxpYzr2d~cp7jNXcy==CA!~N84&oRqd3XS;Yv~Hl{Rl)Z4 zMVazQU>B3_?ob2{>UC&68W4(NOo_6CM;jq>H8Kxv6^Kg{+jWT=KTxR=P;6dMK0+o* z!3^M9L~bRNUF+5~=+i&{YHU}an8?XAEW@OSoSA!z?j7fu=?@_lc#d(aV1#0*FZX@d z@KkyAapO$zX`p9xhn$|+{J4$l^M@os5)z~Fw^HipEU~da1~mw7*qrAXOP*^ zmuMlA&0^b0r1&oqKPLw=)5eJKMu+BLkdDz;*J<-Ik?Hn7D== z7r7pSdnS5kZ)oynK=0~sUH*@l{OM`~wzhiwO2hkhsc-CYm z;EtCLT#>!Fc^PJGiud}SPp;e6J2SJAA*_q{3jG_cGeAGG^%S>{Xm-oJ_e<$GA0(Vw`gV*JH;s=Dwq6I zA$AbYb%TNQque@g{&8H!H)|*rP+LDelJ2yx?L1MzS=m>V==WW530afC+vG3a3RQ06 z#NhSUYNWVnq)eDqReL<1>2R?aQ&>ZwW`5MUro_*g-7P`Q64hD1S|rJd!5l5-D4EC; z#MCV{(9Cm~ZE;LQWOPI&d{vS|`f(C`HN}3f$J;3jdn{)1E5WEh!{dY)DHot_Y!_Db z8S2{wFJGwfJ9|n5ay~nQz0G@v*{2Gu%&nUK%J;1D@ved+;2ZqRhd-?Bzq$WC1Tn9A zw453W^1eW1^tI0N{`N+I$*60BDP@o@`RMK#01k%+z@Na{`ypVUi2Apn&T;Cf zdjzU3v;8lNlBh~BfVtW|+ha@vtraGZPBGQCzAbOnw7@Tr9WmvDi_Su2w>p0UPo`Or zJxpR4yVsXdJ1`DR>Mvy&k&g3x#o6|bQRvud(*>oW4=)D#SgRyG!Uz;R@KnsKhNAWP zh&@{UibZ}AmUgKuFA&r}dVPKmTo!f9n*=iGO10kiCZ|AyNnn7O*p&?uk%>oWFvt7J zTl_wn>Upt_di&?}t<&h+w4Ic1e56^32#hzkp~5)L6?df0)fct)8SP0`@) zh*o1Uj2Xudyd)>?L<1gS4jGN|dho*H#{?l}~+62_|?QQu`! z4J*1w@}6jqHN#a3SIIUObv-qwT7guzP@9ZN<}t5*R;ooY3bUl(m`AFNnj4)E`S!zy z-^VjT{Z*ABR(E5_HZOOqE58o>WgG3g(@#gyT))7iZqh*^rk(jGB3R3*EPTHZ#}d4S zUe@!Y6C|MbBq@&)_-HJA_0c^bmfB?V#BAokpKRRJ`Jv9lXyj+N17T#3d8XVt2K3-Z zWKpIfqOYu(E>*`4*M-YmzIY!~sboY2SHea|%alYb7!{PItMqp1Dre`Hol8RNl#g?` z>oDwYgjfuk7tv8ARL~`lEcDTP5d7N>I>SJ3Lm;+ITPTF0vN!1v;#?W;@D$)4Tlmsl z$b%)Xp_y)$X}%|XJyolyWj#B#r`32AZ^oi2acdf}>!?Lqg{9Q-XKH2__F~zmMx~^p zy-mygK+@E=E1%Z@*1%gw;kYFAyGvC6g$zZTlGaO7jNYXn1`GsO2^GI#+eY`z^irs( z4Oj1iU=W^|&#>6%@g}9Y4tZnnz6IIioH=%KTzPqf3-gDQN!(c zbspyGttshEAzj~Mb&fJNoi*$TmC{Ks6Nh@W23Z(_WiE7ZFe#=oFVJT!_C0~GV%L=5 z>SW5t&k?m%Pid-&-X)S<^8R_1&RtBQW>4F_v2lEi7%{T68@&gIQkSE8(Wjm-jH(8!+i4T&LS~Ys zT=t>tl=gev?T2)kBO#uymaFJFit}*ai-kI=!*L~_U}+IBVedInzj&+Lf90iwQ=kD* zH=-CQ_&*Z16(dA3`W+5Ex@B8)5S&(Xc>0}TOVixeOS;_0lMj$;FU3dYJ!ub+_66e> zAbG4QDG@%%^HK@$I<&xtYx68y3us^f8%6QjIP0okZyRka88G7az%6f?}>yxc8*H#n$GPqm4x9i+?CW>cR<$zYHw)!A3f_bWt z2LOE;Bf8(JUQ-#Ep!BU>!;I}m z;9eDWAscp)RU7Wevh6BO6)nS5yY4mBX{4c#T^s_eUDZkxnmzPoB&>0@88M^1ViucE zt#IDhf$v;(9fnxWmA*SHuI^${cb>2T45BK;F_fV066ey-WUD=tW%X+CtsJ+(aycpZ zMQhM1LxNr5LS#^V{2tY;WM{jgaJ)N&+e4n(DTyx4RN*iZXx-W{eVQB6q^8R&tRt5n zypvLUN~qpX8JKKM0|E_BVHMgTYd_m8&TFM%jpaTeO?)b?4!#tOAkXc5GiW2Jsz6Yc_G-Dy zv;<`C$kHL#CmUh;ei(sD!Qu_{mEgBV!HHh_%o84@F z?+VXdP-!ep8Ny{WJYvDSAen2yT3v9DSE}aEvjc*Gyol{Vh_r5bN!S!E@Pg;e&@OyP z=egr#9-kQ)7A(2joL&6b$VgY`Wdp013ItczZ~0w#dT@m=D%(nJE_iWJgnLZtR!R1P zG!m|}?2?}Y5Mr31+;+=A-Ko$(JLk%l@LA0ePX65(NiD4feJ*?T-DNPob;*j;vMpKEfFPbY@PvQk#UTDb;b#}%67bsv+Bjx>S189z?1*H% zGylf_G_#!W`qc8}+tK*!fZJZw?0}t_%$6>Txvg`4kTaY4d~>Zr&lorE=i}9d61}r= zQTWEwk&=y;!!~B~rV1|1h?c({DkBy()H2mX`+VAg|F!-`Q!5S^-q_-i3~_JeguE-y za47c3m^aG0JK!Mnv0@vfnnW`aT&6p&youQHOg2PuF>YRd zOPM)_e;vU-bG7CPYo3CC%IDyJM^*N+pzC}Pz4HQp_PYO)5P6sF$9#A*@E{+g^5!1V zIBikyDc{9%|9Vj7DFh{b@im(~n`1BK30 zbHb#xs&nDRw)w%)Vy@fqWd{9TXnpXLL9~Xk1Hk~Rt+FUD`PVZzjOjB)wVHw1uHe3? zD0I)`!aLrsxgpD34xa!dXBr`P{dOhL<@l$I%xR9HV;l zgo55p!K&+@3DgF%TiyaqRaJ)w}R~ zQ|5oE(_n$K{;+weGnPLrsqIQ-3Qu64c<+AJrlc^0%eD?5?$YDJhkt1Z=UNj8PR_lG z!&bN(xh2s!hH9MiYVaGB%fvkzF>Fj4#(j4v%LvOKH^I&Xc7F|Y=3#n1f9 zkKySoR(`(yRgUm>nB|c{0s$%c_K(UD$iQ-hp^bsOosF$MgMp2m(fhl4o{GO@f%Td8 zD|D4pooR$gG^x%ZYC*q-ZB3`eK`P#k)E&B_@pw|MLGy%b~oU|H+Ui*-5k^ zBuVLT2czyhzl$-qsWbxvOHoapW;>YDrzk>437*o<*RR2^8Xs*)fjQZ~V5PYRb&xC& zg%#_X6`SljJ$${6-#zzd24j79f47FYF5Z0hSNPtR3bC7}{GbQa$C?)Jcj2|#sj7ntKrQ9NNE?biViSry;5Haf?+RI!p0i3XD4t@Ax!eXeDA zuOBT;dY;&Uf`IJ)!}<|$M*luxdbYOj^OYI%w|d$W`mgIW4mu09u&o+INdNBs!C~t@ z`ANww^pe)eF=(mG*{!)6w!8`JYKzqox`hEMMrgDqx`;^%{l zX2Jm;?)c|h%ShDuD>>*-ZI-p{5$hNCnx>)?T={BG4}ZO|8dS}Cga~wI5AgG!7voU> zS7#bH+B?`-z5BC1UdLwX6I#&O9SyBDg?U6IKQV=hK6hNJvCbUpqD~;Zbz9#z(1+=| zH@>{_NW1EEndUE0S$CYa;x>pY{R`g}z#bpfIZ0j-2A8`lCdv-TYdJI?))$sGFQ$=eQ`Q%>{ zTJ}^95vJa#HV{!#(fZlk4$BNNhxUDF!HHOFDLf7Pc3p6wzWTxPcX5Vx`{k5~S8QT_ z^!k0bL6_yyu2Rmq3J9~Im*$r;5&VtFt*7>#$RwGQ(_i^b036b6pF|u6hMNC}IQ~{= zYV%$;$HQMgSz$x_<01_>t>fVTXlmhrQ~Rg<{8!1jeTG-AhY<;Ky}gt8a+O0us#HFX z9=a4)NJRhhEm<{+_h5(6{hc~`9vV6zci8mlw0(d;+gkK9sm*#DYYVY!TI>f_cA>)t z;)&MahcwGWjfW55J@T8}W}YZ+b(P}5re~BlZQy+#=Cu2^(^$n=L4_8jMr|f^lf)yc zGcTi`ZX}Fr(9yqZ#l{=n>21VW0$AgE+Be)Ga3&(|RgmLSEkJ3Q5w2I7UVo?X!RH2K zHKHYJy>?Kj_((GvCg4KZ9|<)I!@Le62k{44d6$IvV$6BqVIkLnAX6||8077I7V$Wn zD)G+Rmq#U8(ayd`De_dBzHX8t^BXou`#qCJT-2PYSsP+Dt7MU+o2DTl_gedi`o8F3 zc41WlEBPRhz?ZxK@pba&@=&+3WH56uvij?t?h{Qd`=tgHud~Yaa~`R-(73|V#qYGO zW2~~w)o$aiM*<+ExkNBQAXW!!&aR!3#8i!2vPV1|q<31UUhd!m-fvsK91~1%p!?ek z%_AIDatU?=Tw4ntj~?|L@?Yvd^sN5T=AdQGGy~#O+_FOQW&-doIyq6sKqb#!-W)u`yB^SA)%D600JWYxkJ5d zboRnS%Ed`m6DC&=#^KeaYnHo7p7}yE2!6C*I8|*)_U?CfjGpx)U8};eKG(hRdx>xU z*zI01dN~qGF1-k7R;J4BKkhFgvG6}P_1k@fma#rJHPj(jS(*hx|d4xwbL z9m`13a+flhsdCK30hAgoCRm4JUa$P&Hl<=?t7?F#YLATSyoi>~!G|2#eX8arofSs!WnaK&7`I z3ORukGr)=_z;VT=hX)x`;MU`-dJK+oiSukl`G9lMi*#Jv%CPcncGD~xO!9SL`&A8e zXO*x4j4+jDa5TF_3>?j|__O^168zo7`MwG1t#igl#VWs~CQ;o|cGfSQlUpX^Je(mu z_TS0}mC93lH6@&#G=4-?gfwBl6qctq<%uy^&eoCt%53w0YD#C`wQ7#u=9}gM@Z}2# z?XLWqfpbY{i8NlZqAalFJMYq&qJzcXpI8I_ywU#HwnKFL>JtF|-i>*&^(wR?1Y?xs zx=PvRTLa^OAZ~DyF`8bt^W?>rE4sTV^o=BA2{ew&V6%_^&Bupr&H=R4LZTj_@`E2=?F(KrEG#G@<1tOU$0`-!da-!v9djX?Q$kXxw{{+Ze86>4NT~4YXuhc}~ zp6?#?-)fw#MNkhoEL<^e3#*~x!Z?{{r)p8p*h~xzzv%ZwW`h5bAo^myh@TI%KWiX50d#i zIm6qz%5>jV2tHQqXHj0rfl=Qca|4GvwcYGtVy&c#i2G@h(Q6cM=M+zq9Is1F;B8Pr zh&vZ+YUN}Y&YBDumpMVkT?Wx{&3TBc5V3fsM;hA88X`DF5K>toJ=_=;t|k%OcN&>& z=1*R6Qg)_on3`ruA8^)UMh5PCQhE6G(Gom0en1C&l`!10eg5oyO}Nd0Ndds7MvdmP zjfk5Nq4EpB4R(#aVWPj$0?!V-g|G8TPAZaPUHOn{;PSA8Jd2Sv=i*-WQ(vgw=8HLB zGI7lHuhtuzdMM4>qhg%2n!Qd6q_(apHFkoGR5sBkjqitYc|L55UYHPYkVmfAMpB)3 zCJu=msab-JWASbHM`NeyY~Ez59>aD+1@;^DeNyCf#o|n_2~Nn?Ud8GNn+a%Z%1|+n zi3GYX0*6qq$ymHb5&msD&|}8pE<`__csRbAN#xPKC_YoSfIW^vQjSnhT{V3aqn(-}n_-J=SWXxVJe=4Li)8wX@Y{G-Sc`W}$D zD<#{8ctf8`U2Uu;%ERk6Xt{hn z-g}B(YKB**_@;fXNZ+UKP}hJW%gh)>y)VA?3>O8{;&GXUPOVUFC_)sSC%#`%kc?g? zxBXnsSGDG_F!Yl17LCJtgtl2qn(1N^q1YsR2SA&bRX3*d@e`Na5|`5vNk$0=eTLrj zZK3__@PR9nD?H9SiCNo;wbGpxc^o~@%)_%YDCtev**8y7h}U|ncdXl`^;gVXigoB+ zv7sH=8Qqo2AKtyrpoWdz=i@h=m0q9n-UiCT1&bY;+#+V+9ny-+nqPcS+hpR0|rHK^^)C zZb8$*K(G32NW+l|-y1mBXt6OaAy6RO1DT}2lUgtiNF`&|O&a9qvc1bL{8i!RA&RMK zj0{yif=3?W}1{Bfb(=||}saN)A7`m&x zkFB6SVs7l2_x00+HRrqTs3_;TEYvi))7^k83q3EC|WBkiCN2j~-n(b`1JMquN zU%I|9bOMNVC+hk}Mq?FmT-vTr}D{a6Fm5}p$DX3;9^G(2;}yBKQV^;%u8 zvJGE-b@gDsQ^Is+Qiv%)kYrP?>NlWQ{U7#}3Da1~w;e)(?|ln2k2bkooZt+o!o+DG|(X2?kK*O{a4wI7*lEh}8o z{Oh!9{czet|FGrSeU%*Lfs5!F3eo;Lc~M{*o-rbA^J#b{AR*sDJ&z=OK1u^uB^S|) zP(od%8BDpjX}vLUxzl<`M$PRBgQeXQ+w41?ftQqyYWY)_@lr?+ZC9t<%yv^y0Ha9ld)FK^?d zQnEmk8OvgvQ?A`^vlTCCl`+;n9Yoqtcxa_@J$?Lclz?fFxwu$b|M&Fn^5m_$zERDD zxd28#?i}VAl9y(Fdr9|8=y0c!|7M%i)D)2<;c_DIw5ecbocpzi*&%`_rHy-7|sDkK|(4WhX!Eo^^d#yQv`EWtK@zocP#j zz};fwYqBozU~T05Oj8)Q;ysfd2t%BcxSRiL**9A;cj90oY=|Bx@o4ClN*m}fWTM`# zq%PqJM0_@YDq{XI|CR_5{=1RzbjiRef2zT#29|7{3+_Y(rr|NGJng$|T8&6A76C zn!;uiXVQ1~h;+AzkVG>yv?%lbCkGoTIXm66bh>A%@8x3YYK)A(?N_^;5Tc`Diw;t5 zU~W=c%ImLmwu8Pz|5MrB1mKh}+(|fSVi+zNz=LrzriL zf1YF}&LtU zR38rMylTZfTg|r%Za(lka4(|fc^v7XeUL`jRF#;~hkded3+1J)(_D|pEO!lBmXsZQ zjHrZI0O+CAn4^G^L#dA&)`)8C#h3#kk<)_F zmX9cBAQq!%2V*bA>?lUwn`rAI{%bV$5A!6kQ82r2k=@5=8$ORh`AIJ_HV$Ul4B4xT zwspZjiC}l`Vk2PIaggOTwE4+n)NcM$Y{OoTS-C(Kd(bARh_U?Nr626om@mhX)eW@y zC}FHd*E+BPFkc8G-=U-Jn)0zX>evf0Q`h6)oUekNuA&qjHUMTuc>ElQYGVVS^IdEd z%*5*WHkfs=QBe8{8vrvuLFQj*W75S0I4K*$#==zm$Fb-kCu5=W8*D604SgJoO7CPW zbjE;tkXe^H!AGPJHT^vg)|oR(?Q1JtORGU@Dg5 z+ZZ-DaT_P1V8;LB+kjogMnQ#p>}_Di)#KZEaSa;)^)yaI!MvuAZzKIW0B}M?*3}>& TL>^Rn#9ud}l#_4+`QN_)vami( literal 0 HcmV?d00001 diff --git a/spec/models/form_handler_spec.rb b/spec/models/form_handler_spec.rb index f9d03aa36..fe979ac27 100644 --- a/spec/models/form_handler_spec.rb +++ b/spec/models/form_handler_spec.rb @@ -153,4 +153,20 @@ RSpec.describe FormHandler do expect(form.type).to eq("lettings") expect(form.start_date.year).to eq(2022) end + + # rubocop:disable RSpec/PredicateMatcher + describe "#in_crossover_period?" do + context "when not in overlapping period" do + it "returns false" do + expect(form_handler.in_crossover_period?(now: Date.new(2022, 1, 1))).to be_falsey + end + end + + context "when in overlapping period" do + it "returns true" do + expect(form_handler.in_crossover_period?(now: Date.new(2022, 6, 1))).to be_truthy + end + end + end + # rubocop:enable RSpec/PredicateMatcher end diff --git a/spec/models/form_spec.rb b/spec/models/form_spec.rb index a9a0c9b40..110a521e2 100644 --- a/spec/models/form_spec.rb +++ b/spec/models/form_spec.rb @@ -237,40 +237,6 @@ RSpec.describe Form, type: :model do end end - describe "#in_crossover_period?" do - context "when now not specified" do - context "when after end period" do - subject(:form) { described_class.new(nil, 2022, [], "sales") } - - it "returns false" do - Timecop.freeze(2023, 8, 1) do - expect(form).not_to be_in_crossover_period - end - end - end - - context "when during crossover" do - subject(:form) { described_class.new(nil, 2022, [], "sales") } - - it "returns true" do - Timecop.freeze(2023, 6, 1) do - expect(form).to be_in_crossover_period - end - end - end - - context "when before crossover" do - subject(:form) { described_class.new(nil, 2022, [], "sales") } - - it "returns false" do - Timecop.freeze(2023, 1, 1) do - expect(form).not_to be_in_crossover_period - end - end - end - end - end - describe "when creating a lettings log", :aggregate_failures do it "creates a valid lettings form" do form = described_class.new("spec/fixtures/forms/2021_2022.json") diff --git a/spec/models/forms/bulk_upload_lettings/upload_your_file_spec.rb b/spec/models/forms/bulk_upload_lettings/upload_your_file_spec.rb new file mode 100644 index 000000000..1199201fa --- /dev/null +++ b/spec/models/forms/bulk_upload_lettings/upload_your_file_spec.rb @@ -0,0 +1,47 @@ +require "rails_helper" + +RSpec.describe Forms::BulkUploadLettings::UploadYourFile do + subject(:form) { described_class.new(year:, file:, current_user:) } + + let(:year) { 2022 } + let(:actual_file) { File.open(file_fixture("blank_bulk_upload_sales.csv")) } + let(:file) { ActionDispatch::Http::UploadedFile.new(tempfile: actual_file) } + let(:current_user) { create(:user) } + let(:mock_storage_service) { instance_double("S3Service") } + + before do + vcap_services = { "aws-s3-bucket" => {} } + + allow(ENV).to receive(:[]) + allow(ENV).to receive(:[]).with("VCAP_SERVICES").and_return(vcap_services.to_json) + + allow(Storage::S3Service).to receive(:new).and_return(mock_storage_service) + allow(mock_storage_service).to receive(:write_file) + end + + describe "#save" do + it "persists a BulkUpload" do + expect { form.save! }.to change(BulkUpload, :count).by(1) + end + + it "persists a BulkUpload correctly" do + form.save! + + bulk_upload = BulkUpload.last + + expect(bulk_upload.user).to eql(current_user) + expect(bulk_upload.log_type).to eql("lettings") + expect(bulk_upload.year).to eql(year) + expect(bulk_upload.identifier).to be_present + end + + it "uploads file via storage service" do + form.save! + + bulk_upload = BulkUpload.last + + expect(Storage::S3Service).to have_received(:new) + expect(mock_storage_service).to have_received(:write_file).with(bulk_upload.identifier, actual_file.read) + end + end +end diff --git a/spec/models/forms/bulk_upload_sales/upload_your_file_spec.rb b/spec/models/forms/bulk_upload_sales/upload_your_file_spec.rb new file mode 100644 index 000000000..1b11b0f82 --- /dev/null +++ b/spec/models/forms/bulk_upload_sales/upload_your_file_spec.rb @@ -0,0 +1,47 @@ +require "rails_helper" + +RSpec.describe Forms::BulkUploadSales::UploadYourFile do + subject(:form) { described_class.new(year:, file:, current_user:) } + + let(:year) { 2022 } + let(:actual_file) { File.open(file_fixture("blank_bulk_upload_sales.csv")) } + let(:file) { ActionDispatch::Http::UploadedFile.new(tempfile: actual_file) } + let(:current_user) { create(:user) } + let(:mock_storage_service) { instance_double("S3Service") } + + before do + vcap_services = { "aws-s3-bucket" => {} } + + allow(ENV).to receive(:[]) + allow(ENV).to receive(:[]).with("VCAP_SERVICES").and_return(vcap_services.to_json) + + allow(Storage::S3Service).to receive(:new).and_return(mock_storage_service) + allow(mock_storage_service).to receive(:write_file) + end + + describe "#save" do + it "persists a BulkUpload" do + expect { form.save! }.to change(BulkUpload, :count).by(1) + end + + it "persists a BulkUpload correctly" do + form.save! + + bulk_upload = BulkUpload.last + + expect(bulk_upload.user).to eql(current_user) + expect(bulk_upload.log_type).to eql("sales") + expect(bulk_upload.year).to eql(year) + expect(bulk_upload.identifier).to be_present + end + + it "uploads file via storage service" do + form.save! + + bulk_upload = BulkUpload.last + + expect(Storage::S3Service).to have_received(:new) + expect(mock_storage_service).to have_received(:write_file).with(bulk_upload.identifier, actual_file.read) + end + end +end diff --git a/spec/requests/bulk_upload_controller_spec.rb b/spec/requests/bulk_upload_controller_spec.rb index bc75dbe84..fc3afd21c 100644 --- a/spec/requests/bulk_upload_controller_spec.rb +++ b/spec/requests/bulk_upload_controller_spec.rb @@ -45,7 +45,7 @@ RSpec.describe BulkUploadController, type: :request do end it "returns a page with a file upload form" do - expect(response.body).to match(/