|
|
|
@ -1,9 +1,14 @@ |
|
|
|
|
|
|
|
require 'securerandom' |
|
|
|
|
|
|
|
require 'json' |
|
|
|
|
|
|
|
|
|
|
|
module Imports |
|
|
|
module Imports |
|
|
|
class LettingsLogsImportService < ImportService |
|
|
|
class LettingsLogsImportService < ImportService |
|
|
|
|
|
|
|
include Wisper::Publisher |
|
|
|
|
|
|
|
|
|
|
|
def initialize(storage_service, logger = Rails.logger) |
|
|
|
def initialize(storage_service, logger = Rails.logger) |
|
|
|
@logs_with_discrepancies = Set.new |
|
|
|
|
|
|
|
@logs_overridden = Set.new |
|
|
|
|
|
|
|
super |
|
|
|
super |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
Wisper.subscribe(LettingsLogImportListener.new, prefix: :on) |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
def create_logs(folder) |
|
|
|
def create_logs(folder) |
|
|
|
@ -13,8 +18,58 @@ module Imports |
|
|
|
end |
|
|
|
end |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
private |
|
|
|
def import_from(folder, create_method) |
|
|
|
|
|
|
|
filenames = @storage_service.list_files(folder) |
|
|
|
|
|
|
|
filenames.each do |filename| |
|
|
|
|
|
|
|
file_io = @storage_service.get_file_io(filename) |
|
|
|
|
|
|
|
xml_document = Nokogiri::XML(file_io) |
|
|
|
|
|
|
|
send(create_method, xml_document) |
|
|
|
|
|
|
|
rescue StandardError => e |
|
|
|
|
|
|
|
@logger.error "#{e.class} in #{filename}: #{e.message}. Caller: #{e.backtrace.first}" |
|
|
|
|
|
|
|
end |
|
|
|
|
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def local_load(folder) |
|
|
|
|
|
|
|
filenames = Dir["#{folder}/**/*.xml"] |
|
|
|
|
|
|
|
puts "FILENAMES (#{filenames.size}): #{filenames}" |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@run_id = SecureRandom.uuid.to_s |
|
|
|
|
|
|
|
logs_import = LogsImport.create!( |
|
|
|
|
|
|
|
run_id: @run_id, |
|
|
|
|
|
|
|
started_at: Time.zone.now, |
|
|
|
|
|
|
|
total: filenames.size, |
|
|
|
|
|
|
|
discrepancies: [], |
|
|
|
|
|
|
|
filenames: filenames |
|
|
|
|
|
|
|
) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
redis = Redis.new |
|
|
|
|
|
|
|
redis.set(@run_id, Marshal.dump(logs_import)) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
broadcast(::Import::STARTED, @run_id) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
filenames.each do |filename| |
|
|
|
|
|
|
|
puts "Loading filename: #{filename}" |
|
|
|
|
|
|
|
Rack::MiniProfiler.step("Start Processing file #{filename}") do |
|
|
|
|
|
|
|
# Generate background job to process file completely |
|
|
|
|
|
|
|
xml_document = Nokogiri::XML(File.open(filename)) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
LettingsLogImportJob.perform_now(@run_id, xml_document.to_s) |
|
|
|
|
|
|
|
#send(:create_log, xml_document) |
|
|
|
|
|
|
|
end |
|
|
|
|
|
|
|
rescue StandardError => e |
|
|
|
|
|
|
|
@logger.error "#{e.class} in #{filename}: #{e.message}. Caller: #{e.backtrace.first}" |
|
|
|
|
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if @logs_with_discrepancies.count.positive? |
|
|
|
|
|
|
|
@logger.warn("The following lettings logs had status discrepancies: [#{@logs_with_discrepancies.join(', ')}]") |
|
|
|
|
|
|
|
end |
|
|
|
|
|
|
|
end |
|
|
|
|
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class LettingsLogsImportProcessor |
|
|
|
FORM_NAME_INDEX = { |
|
|
|
FORM_NAME_INDEX = { |
|
|
|
start_year: 0, |
|
|
|
start_year: 0, |
|
|
|
rent_type: 2, |
|
|
|
rent_type: 2, |
|
|
|
@ -54,11 +109,24 @@ module Imports |
|
|
|
other_intermediate_rent_product: 5, |
|
|
|
other_intermediate_rent_product: 5, |
|
|
|
}.freeze |
|
|
|
}.freeze |
|
|
|
|
|
|
|
|
|
|
|
def create_log(xml_doc) |
|
|
|
attr_reader :xml_doc, :logs_with_discrepancies, :logs_overridden, :discrepancy, :old_id |
|
|
|
attributes = {} |
|
|
|
|
|
|
|
|
|
|
|
def initialize(xml_doc) |
|
|
|
|
|
|
|
@xml_doc = xml_doc |
|
|
|
|
|
|
|
@discrepancy = false |
|
|
|
|
|
|
|
@old_id = '' |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@logs_with_discrepancies = Set.new |
|
|
|
|
|
|
|
@logs_overridden = Set.new |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def create_log |
|
|
|
|
|
|
|
attributes = {} |
|
|
|
|
|
|
|
lettings_log = nil |
|
|
|
previous_status = field_value(xml_doc, "meta", "status") |
|
|
|
previous_status = field_value(xml_doc, "meta", "status") |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
Rack::MiniProfiler.step("Loading attributes") do |
|
|
|
# Required fields for status complete or logic to work |
|
|
|
# Required fields for status complete or logic to work |
|
|
|
# Note: order matters when we derive from previous values (attributes parameter) |
|
|
|
# Note: order matters when we derive from previous values (attributes parameter) |
|
|
|
attributes["startdate"] = compose_date(xml_doc, "DAY", "MONTH", "YEAR") |
|
|
|
attributes["startdate"] = compose_date(xml_doc, "DAY", "MONTH", "YEAR") |
|
|
|
@ -182,6 +250,7 @@ module Imports |
|
|
|
attributes["created_at"] = Time.zone.parse(field_value(xml_doc, "meta", "created-date")) |
|
|
|
attributes["created_at"] = Time.zone.parse(field_value(xml_doc, "meta", "created-date")) |
|
|
|
attributes["updated_at"] = Time.zone.parse(field_value(xml_doc, "meta", "modified-date")) |
|
|
|
attributes["updated_at"] = Time.zone.parse(field_value(xml_doc, "meta", "modified-date")) |
|
|
|
attributes["old_id"] = field_value(xml_doc, "meta", "document-id") |
|
|
|
attributes["old_id"] = field_value(xml_doc, "meta", "document-id") |
|
|
|
|
|
|
|
@old_id = attributes["old_id"] |
|
|
|
|
|
|
|
|
|
|
|
# Required for our form invalidated questions (not present in import) |
|
|
|
# Required for our form invalidated questions (not present in import) |
|
|
|
attributes["previous_la_known"] = attributes["prevloc"].nil? ? 0 : 1 |
|
|
|
attributes["previous_la_known"] = attributes["prevloc"].nil? ? 0 : 1 |
|
|
|
@ -231,16 +300,22 @@ module Imports |
|
|
|
|
|
|
|
|
|
|
|
apply_date_consistency!(attributes) |
|
|
|
apply_date_consistency!(attributes) |
|
|
|
apply_household_consistency!(attributes) |
|
|
|
apply_household_consistency!(attributes) |
|
|
|
|
|
|
|
end # ENDPROFILER |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lettings_log = save_lettings_log(attributes) |
|
|
|
lettings_log = save_lettings_log(attributes) |
|
|
|
|
|
|
|
|
|
|
|
compute_differences(lettings_log, attributes) |
|
|
|
compute_differences(lettings_log, attributes) |
|
|
|
check_status_completed(lettings_log, previous_status) unless @logs_overridden.include?(lettings_log.old_id) |
|
|
|
check_status_completed(lettings_log, previous_status) unless @logs_overridden.include?(lettings_log.old_id) |
|
|
|
|
|
|
|
|
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
def save_lettings_log(attributes) |
|
|
|
def save_lettings_log(attributes) |
|
|
|
lettings_log = LettingsLog.new(attributes) |
|
|
|
lettings_log = LettingsLog.new(attributes) |
|
|
|
begin |
|
|
|
begin |
|
|
|
lettings_log.save! |
|
|
|
Rack::MiniProfiler.step("Saving...") do |
|
|
|
|
|
|
|
lettings_log.save!(validate: true) |
|
|
|
|
|
|
|
end |
|
|
|
lettings_log |
|
|
|
lettings_log |
|
|
|
rescue ActiveRecord::RecordNotUnique |
|
|
|
rescue ActiveRecord::RecordNotUnique |
|
|
|
legacy_id = attributes["old_id"] |
|
|
|
legacy_id = attributes["old_id"] |
|
|
|
@ -291,6 +366,7 @@ module Imports |
|
|
|
@logger.warn "lettings log #{lettings_log.id} is not completed" |
|
|
|
@logger.warn "lettings log #{lettings_log.id} is not completed" |
|
|
|
@logger.warn "lettings log with old id:#{lettings_log.old_id} is incomplete but status should be complete" |
|
|
|
@logger.warn "lettings log with old id:#{lettings_log.old_id} is incomplete but status should be complete" |
|
|
|
@logs_with_discrepancies << lettings_log.old_id |
|
|
|
@logs_with_discrepancies << lettings_log.old_id |
|
|
|
|
|
|
|
@discrepancy = true |
|
|
|
end |
|
|
|
end |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
@ -657,5 +733,12 @@ module Imports |
|
|
|
0 |
|
|
|
0 |
|
|
|
end |
|
|
|
end |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def discrepancy? |
|
|
|
|
|
|
|
!!@discrepancy |
|
|
|
|
|
|
|
end |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
end |
|
|
|
end |
|
|
|
|
Mo Seedat
2 years ago