Browse Source
* add class to create logs from bulk upload * create logs when processing bulk uploads * remove bulk_upload_id from csv output * create bulk upload logs only if all valid - this will be changed later to allow for partial logs - and only to create logs when a threshold has been met * add method to blank invalid non setup fields * bulk upload log creation blanks invalid fields * fix incorrect logic for bulk upload renewal * fix linting * bulk upload log creation fail logs to sentry * fix bulk upload line ending parsing * extract bulk uploading csv parsing to class * use csv parser in log creator * change handle line endings mechanism - we now strip all windows line endings for unix based line endings - this normalises things making it simplerpull/1221/head
Phil Lee
2 years ago
committed by
GitHub
18 changed files with 638 additions and 47 deletions
@ -0,0 +1,54 @@
|
||||
require "csv" |
||||
|
||||
class BulkUpload::Lettings::CsvParser |
||||
attr_reader :path |
||||
|
||||
def initialize(path:) |
||||
@path = path |
||||
end |
||||
|
||||
def row_offset |
||||
5 |
||||
end |
||||
|
||||
def col_offset |
||||
1 |
||||
end |
||||
|
||||
def cols |
||||
@cols ||= ("A".."EE").to_a |
||||
end |
||||
|
||||
def row_parsers |
||||
@row_parsers ||= body_rows.map do |row| |
||||
stripped_row = row[1..] |
||||
headers = ("field_1".."field_134").to_a |
||||
hash = Hash[headers.zip(stripped_row)] |
||||
|
||||
BulkUpload::Lettings::RowParser.new(hash) |
||||
end |
||||
end |
||||
|
||||
def body_rows |
||||
rows[row_offset..] |
||||
end |
||||
|
||||
def rows |
||||
@rows ||= CSV.parse(normalised_string, row_sep:) |
||||
end |
||||
|
||||
private |
||||
|
||||
def row_sep |
||||
"\n" |
||||
end |
||||
|
||||
def normalised_string |
||||
return @normalised_string if @normalised_string |
||||
|
||||
@normalised_string = File.read(path) |
||||
@normalised_string.gsub!("\r\n", "\n") |
||||
|
||||
@normalised_string |
||||
end |
||||
end |
||||
@ -0,0 +1,57 @@
|
||||
class BulkUpload::Lettings::LogCreator |
||||
attr_reader :bulk_upload, :path |
||||
|
||||
def initialize(bulk_upload:, path:) |
||||
@bulk_upload = bulk_upload |
||||
@path = path |
||||
end |
||||
|
||||
def call |
||||
row_parsers.each do |row_parser| |
||||
row_parser.valid? |
||||
|
||||
row_parser.log.blank_invalid_non_setup_fields! |
||||
row_parser.log.bulk_upload = bulk_upload |
||||
|
||||
begin |
||||
row_parser.log.save! |
||||
rescue StandardError => e |
||||
Sentry.capture_exception(e) |
||||
end |
||||
end |
||||
end |
||||
|
||||
private |
||||
|
||||
def csv_parser |
||||
@csv_parser ||= BulkUpload::Lettings::CsvParser.new(path:) |
||||
end |
||||
|
||||
def row_offset |
||||
csv_parser.row_offset |
||||
end |
||||
|
||||
def col_offset |
||||
csv_parser.col_offset |
||||
end |
||||
|
||||
def row_parsers |
||||
return @row_parsers if @row_parsers |
||||
|
||||
@row_parsers = csv_parser.row_parsers |
||||
|
||||
@row_parsers.each do |row_parser| |
||||
row_parser.bulk_upload = bulk_upload |
||||
end |
||||
|
||||
@row_parsers |
||||
end |
||||
|
||||
def body_rows |
||||
csv_parser.body_rows |
||||
end |
||||
|
||||
def rows |
||||
csv_parser.rows |
||||
end |
||||
end |
||||
@ -0,0 +1,6 @@
|
||||
class AddBulkUploadToLogs < ActiveRecord::Migration[7.0] |
||||
def change |
||||
add_reference :lettings_logs, :bulk_upload |
||||
add_reference :sales_logs, :bulk_upload |
||||
end |
||||
end |
||||
|
Can't render this file because it has a wrong number of fields in line 72.
|
@ -0,0 +1,68 @@
|
||||
require "rails_helper" |
||||
|
||||
RSpec.describe BulkUpload::Lettings::LogCreator do |
||||
subject(:service) { described_class.new(bulk_upload:, path:) } |
||||
|
||||
let(:owning_org) { create(:organisation, old_visible_id: 123) } |
||||
let(:user) { create(:user, organisation: owning_org) } |
||||
|
||||
let(:bulk_upload) { create(:bulk_upload, :lettings, user:) } |
||||
let(:path) { file_fixture("2022_23_lettings_bulk_upload.csv") } |
||||
|
||||
describe "#call" do |
||||
context "when a valid csv with new log" do |
||||
it "creates a new log" do |
||||
expect { service.call }.to change(LettingsLog, :count) |
||||
end |
||||
|
||||
it "associates log with bulk upload" do |
||||
service.call |
||||
|
||||
log = LettingsLog.last |
||||
expect(log.bulk_upload).to eql(bulk_upload) |
||||
expect(bulk_upload.lettings_logs).to include(log) |
||||
end |
||||
end |
||||
|
||||
context "when a valid csv with row with one invalid non setup field" do |
||||
let(:file) { Tempfile.new } |
||||
let(:path) { file.path } |
||||
let(:log) do |
||||
build( |
||||
:lettings_log, |
||||
:completed, |
||||
renttype: 3, |
||||
age1: 5, |
||||
owning_organisation: owning_org, |
||||
managing_organisation: owning_org, |
||||
national: 18, |
||||
waityear: 9, |
||||
joint: 2, |
||||
tenancy: 9, |
||||
ppcodenk: 0, |
||||
) |
||||
end |
||||
|
||||
before do |
||||
5.times { file.write "\n" } |
||||
file.write(BulkUpload::LogToCsv.new(log:).to_csv_row) |
||||
file.rewind |
||||
end |
||||
|
||||
it "creates the log" do |
||||
expect { service.call }.to change(LettingsLog, :count).by(1) |
||||
end |
||||
|
||||
it "blanks invalid field" do |
||||
service.call |
||||
|
||||
record = LettingsLog.last |
||||
expect(record.age1).to be_blank |
||||
end |
||||
end |
||||
|
||||
context "when valid csv with existing log" do |
||||
xit "what should happen?" |
||||
end |
||||
end |
||||
end |
||||
@ -0,0 +1,255 @@
|
||||
class BulkUpload::LogToCsv |
||||
attr_reader :log, :line_ending |
||||
|
||||
def initialize(log:, line_ending: "\n") |
||||
@log = log |
||||
@line_ending = line_ending |
||||
end |
||||
|
||||
def to_csv_row |
||||
[ |
||||
nil, # 0 |
||||
log.renttype, # 1 |
||||
nil, |
||||
nil, |
||||
log.scheme&.old_visible_id, |
||||
log.location&.old_visible_id, |
||||
nil, |
||||
log.tenancycode, |
||||
log.startertenancy, |
||||
log.tenancy, |
||||
log.tenancyother, # 10 |
||||
log.tenancylength, |
||||
log.age1, |
||||
log.age2, |
||||
log.age3, |
||||
log.age4, |
||||
log.age5, |
||||
log.age6, |
||||
log.age7, |
||||
log.age8, |
||||
|
||||
log.sex1, # 20 |
||||
log.sex2, |
||||
log.sex3, |
||||
log.sex4, |
||||
log.sex5, |
||||
log.sex6, |
||||
log.sex7, |
||||
log.sex8, |
||||
|
||||
log.relat2, |
||||
log.relat3, |
||||
log.relat4, # 30 |
||||
log.relat5, |
||||
log.relat6, |
||||
log.relat7, |
||||
log.relat8, |
||||
|
||||
log.ecstat1, |
||||
log.ecstat2, |
||||
log.ecstat3, |
||||
log.ecstat4, |
||||
log.ecstat5, |
||||
log.ecstat6, # 40 |
||||
log.ecstat7, |
||||
log.ecstat8, |
||||
|
||||
log.ethnic, |
||||
log.national, |
||||
log.armedforces, |
||||
log.reservist, |
||||
log.preg_occ, |
||||
log.hb, |
||||
log.benefits, |
||||
log.earnings, # 50 |
||||
net_income_known, |
||||
nil, |
||||
log.reasonother, |
||||
nil, |
||||
nil, |
||||
nil, |
||||
nil, |
||||
nil, |
||||
nil, |
||||
nil, # 60 |
||||
log.prevten, |
||||
log.prevloc, |
||||
((log.ppostcode_full || "").split(" ") || [""]).first, |
||||
((log.ppostcode_full || "").split(" ") || [""]).last, |
||||
previous_postcode_known, |
||||
log.layear, |
||||
log.waityear, |
||||
homeless, |
||||
log.reasonpref, |
||||
log.rp_homeless, # 70 |
||||
log.rp_insan_unsat, |
||||
log.rp_medwel, |
||||
log.rp_hardship, |
||||
log.rp_dontknow, |
||||
cbl, |
||||
chr, |
||||
cap, |
||||
log.referral, |
||||
log.period, |
||||
|
||||
log.brent, # 80 |
||||
log.scharge, |
||||
log.pscharge, |
||||
log.supcharg, |
||||
log.tcharge, |
||||
log.chcharge, |
||||
log.household_charge, |
||||
log.hbrentshortfall, |
||||
log.tshortfall, |
||||
log.voiddate&.day, |
||||
|
||||
log.voiddate&.month, # 90 |
||||
log.voiddate&.strftime("%y"), |
||||
log.mrcdate&.day, |
||||
log.mrcdate&.month, |
||||
log.mrcdate&.strftime("%y"), |
||||
nil, |
||||
log.startdate&.day, |
||||
log.startdate&.month, |
||||
log.startdate&.strftime("%y"), |
||||
log.offered, |
||||
|
||||
log.propcode, # 100 |
||||
log.beds, |
||||
log.unittype_gn, |
||||
log.builtype, |
||||
log.wchair, |
||||
log.unitletas, |
||||
log.rsnvac, |
||||
log.la, |
||||
((log.postcode_full || "").split(" ") || [""]).first, |
||||
((log.postcode_full || "").split(" ") || [""]).last, |
||||
|
||||
nil, # 110 |
||||
log.owning_organisation&.old_visible_id, |
||||
nil, |
||||
log.managing_organisation&.old_visible_id, |
||||
leftreg, |
||||
nil, |
||||
log.incfreq, |
||||
log.sheltered, |
||||
log.illness, |
||||
log.illness_type_1, |
||||
|
||||
log.illness_type_2, # 120 |
||||
log.illness_type_3, |
||||
log.illness_type_4, |
||||
log.illness_type_5, |
||||
log.illness_type_6, |
||||
log.illness_type_7, |
||||
log.illness_type_8, |
||||
log.illness_type_9, |
||||
log.illness_type_10, |
||||
london_affordable_rent, |
||||
|
||||
intermediate_rent_type, # 130 |
||||
log.irproduct_other, |
||||
log.declaration, |
||||
log.joint, |
||||
renewal, |
||||
line_ending, |
||||
].join(",") |
||||
end |
||||
|
||||
def renewal |
||||
case log.renewal |
||||
when 1 |
||||
1 |
||||
when 0 |
||||
2 |
||||
end |
||||
end |
||||
|
||||
def london_affordable_rent |
||||
case log.renttype |
||||
when Imports::LettingsLogsImportService::RENT_TYPE[:london_affordable_rent] |
||||
1 |
||||
end |
||||
end |
||||
|
||||
def intermediate_rent_type |
||||
case log.renttype |
||||
when Imports::LettingsLogsImportService::RENT_TYPE[:rent_to_buy] |
||||
1 |
||||
when Imports::LettingsLogsImportService::RENT_TYPE[:london_living_rent] |
||||
2 |
||||
when Imports::LettingsLogsImportService::RENT_TYPE[:other_intermediate_rent_product] |
||||
3 |
||||
end |
||||
end |
||||
|
||||
def leftreg |
||||
case log.leftreg |
||||
when 3 |
||||
3 |
||||
when 1 |
||||
4 |
||||
when 2 |
||||
5 |
||||
when 0 |
||||
6 |
||||
end |
||||
end |
||||
|
||||
def net_income_known |
||||
case log.net_income_known |
||||
when 0 |
||||
1 |
||||
when 1 |
||||
2 |
||||
when 2 |
||||
4 |
||||
end |
||||
end |
||||
|
||||
def previous_postcode_known |
||||
case log.ppcodenk |
||||
when 1 |
||||
1 |
||||
when 0 |
||||
2 |
||||
end |
||||
end |
||||
|
||||
def homeless |
||||
case log.homeless |
||||
when 1 |
||||
1 |
||||
when 11 |
||||
12 |
||||
end |
||||
end |
||||
|
||||
def cbl |
||||
case log.cbl |
||||
when 0 |
||||
2 |
||||
when 1 |
||||
1 |
||||
end |
||||
end |
||||
|
||||
def chr |
||||
case log.chr |
||||
when 0 |
||||
2 |
||||
when 1 |
||||
1 |
||||
end |
||||
end |
||||
|
||||
def cap |
||||
case log.cap |
||||
when 0 |
||||
2 |
||||
when 1 |
||||
1 |
||||
end |
||||
end |
||||
end |
||||
Loading…
Reference in new issue