baarkerlounger
/
submit-social-housing-lettings-and-sales-data
mirror of https://github.com/communitiesuk/submit-social-housing-lettings-and-sales-data.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Count and report invalid logs efficiency improvements (#3066) 

* Add tasks to just count and surface invalid LettingsLog and SalesLog

* Refactor InvalidLogsHelper module structure

* Refactor InvalidLogsHelper for improved logging and module structure

* Update logging level in InvalidLogsHelper from debug to info for better visibility

* Reorganise includes in invalid_logs.rake

* Refactor InvalidLogsHelper to improve invalid log counting and display

* Enhance logging in InvalidLogsHelper for better tracking of invalid log processing

* Filter logs by status to only process completed entries in InvalidLogsHelper

* Enhance invalid log detection in InvalidLogsHelper to include incomplete subsections and questions

* Refactor InvalidLogsHelper to filter logs by status and enhance invalid log reporting

* Refactor InvalidLogsHelper to optimize invalid log detection and streamline batch processing

* Lint
main v0.5.14
Manny Dinssa 2 days ago committed by GitHub
parent
197d4bab36
commit
aef90cf004
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 112 additions and 25 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 137
      app/helpers/invalid_logs_helper.rb

137
app/helpers/invalid_logs_helper.rb
Unescape View File

@ -2,24 +2,18 @@ module InvalidLogsHelper
def count_and_display_invalid_logs(model, log_type, year)
Rails.logger.info "Starting to count invalid #{log_type} for year #{year}..."
invalid_ids = []
total_logs_seen = 0
total_logs_checked = 0
model.filter_by_year(year).find_in_batches(batch_size: 1000).with_index(1) do |batch, batch_index|
model.filter_by_year(year).where(status: "completed").find_in_batches(batch_size: 1000).with_index(1) do |batch, batch_index|
Rails.logger.info "Processing batch #{batch_index} with #{batch.size} logs..."
batch.each do |log|
next unless log.status == "completed"
invalid_logs = batch.select { |log| log_invalid?(log) }
invalid_ids.concat(invalid_logs.map(&:id))
total_logs_checked += batch.size
total_logs_seen += 1
next unless !log.valid? || log.incomplete_subsections.any? || log.incomplete_questions.any?
invalid_ids << log.id
end
Rails.logger.info "Batch #{batch_index} complete. Progress: #{invalid_ids.size} invalid logs found out of #{total_logs_seen} logs seen so far."
Rails.logger.info "Batch #{batch_index} complete. Progress: #{invalid_ids.size} invalid logs found out of #{total_logs_checked} completed logs checked so far."
end
Rails.logger.info "Counting complete for #{log_type}. Total invalid logs: #{invalid_ids.size}, Total logs seen: #{total_logs_seen}."
Rails.logger.info "Counting complete for #{log_type}. Total invalid logs: #{invalid_ids.size}, Total completed logs checked: #{total_logs_checked}."
Rails.logger.info "Invalid #{log_type} IDs: #{invalid_ids.join(', ')}" if invalid_ids.any?
Rails.logger.info "--------------------------------"
end
@ -27,26 +21,119 @@ module InvalidLogsHelper
def surface_invalid_logs(model, log_type, year)
Rails.logger.info "Surfacing invalid #{log_type} for year #{year}..."
invalid_ids = []
total_logs_seen = 0
total_logs_checked = 0
log_messages = [headers(log_type).join(", ")]
model.filter_by_year(year).find_in_batches(batch_size: 1000).with_index(1) do |batch, batch_index|
model.filter_by_year(year).where(status: "completed").find_in_batches(batch_size: 1000).with_index(1) do |batch, batch_index|
Rails.logger.info "Processing batch #{batch_index} with #{batch.size} logs..."
batch.each do |log|
next unless log.status == "completed"
total_logs_seen += 1
next unless !log.valid? || log.incomplete_subsections.any? || log.incomplete_questions.any?
invalid_ids << log.id
error_messages = log.errors.full_messages.join(";\n")
Rails.logger.info "#{log_type} ID: #{log.id} \n Errors----\n #{error_messages}"
invalid_logs = batch.select { |log| log_invalid?(log) }
invalid_ids.concat(invalid_logs.map(&:id))
invalid_logs.each do |log|
log_row_data = log_row(log, log_type).join(", ")
log_messages << log_row_data
Rails.logger.info log_row_data
end
Rails.logger.info "Batch #{batch_index} complete. Processed #{batch.size} logs."
end
Rails.logger.info "Surfacing complete for #{log_type}. Total invalid logs: #{invalid_ids.size}, Total logs seen: #{total_logs_seen}."
Rails.logger.info "Surfacing complete for #{log_type}. Total invalid logs: #{invalid_ids.size}, Total completed logs checked: #{total_logs_checked}."
Rails.logger.info "Invalid #{log_type} IDs: #{invalid_ids.join(', ')}" if invalid_ids.any?
Rails.logger.info log_messages.join("\n")
Rails.logger.info "--------------------------------"
end
private
def log_invalid?(log)
!log.valid? || log.incomplete_subsections.any? || log.incomplete_questions.any?
end
def log_row(log, log_type)
incomplete_subsections = log.incomplete_subsections.map(&:label).join("; ")
incomplete_question_ids = log.incomplete_questions.map(&:id).join("; ")
created_at = log.created_at&.strftime("%d/%m/%Y")
updated_at = log.updated_at&.strftime("%d/%m/%Y")
if log_type == "LettingsLog"
[
log.id,
log.tenancycode,
log.propcode,
log.managing_organisation_id,
log.managing_organisation&.name,
log.assigned_to&.email,
log.address_line1,
log.address_line2,
log.town_or_city,
log.county,
log.postcode_full,
incomplete_subsections,
incomplete_question_ids,
log.status,
created_at,
updated_at,
]
else
[
log.id,
log.purchid,
log.managing_organisation_id,
log.managing_organisation&.name,
log.assigned_to&.email,
log.address_line1,
log.address_line2,
log.town_or_city,
log.county,
log.postcode_full,
incomplete_subsections,
incomplete_question_ids,
log.status,
created_at,
updated_at,
]
end
end
def headers(log_type)
if log_type == "LettingsLog"
[
"Log ID",
"Tenant Code",
"Property Reference",
"Managing Organisation ID",
"Managing Organisation Name",
"Assigned To (User Email)",
"Address Line 1",
"Address Line 2",
"Town/City",
"County",
"Postcode",
"Incomplete Subsections",
"Incomplete Question IDs",
"Status",
"Created At",
"Updated At",
]
else
[
"Log ID",
"Purchaser Code",
"Managing Organisation ID",
"Managing Organisation Name",
"Assigned To (User Email)",
"Address Line 1",
"Address Line 2",
"Town/City",
"County",
"Postcode",
"Incomplete Subsections",
"Incomplete Question IDs",
"Status",
"Created At",
"Updated At",
]
end
end
end

Write Preview
Loading…
Cancel
Save