diff --git a/app/helpers/invalid_logs_helper.rb b/app/helpers/invalid_logs_helper.rb index 00e8cb786..4cd0e2976 100644 --- a/app/helpers/invalid_logs_helper.rb +++ b/app/helpers/invalid_logs_helper.rb @@ -1,23 +1,29 @@ module InvalidLogsHelper def count_and_display_invalid_logs(model, log_type, year) - invalid_logs = fetch_invalid_logs(model, year) - Rails.logger.info "Number of invalid #{log_type} for year #{year}: #{invalid_logs.size}" - Rails.logger.info "Invalid #{log_type} IDs: #{invalid_logs.map(&:id).join(', ')}" - end + invalid_ids = [] + total_logs_seen = 0 - def surface_invalid_logs(model, log_type, year) - invalid_logs = fetch_invalid_logs(model, year) - if invalid_logs.any? - invalid_logs.each do |log| - Rails.logger.info "#{log_type} ID: #{log.id}" - log.errors.full_messages.each { |message| Rails.logger.info " - #{message}" } + model.filter_by_year(year).find_in_batches(batch_size: 1000) do |batch| + batch.each do |log| + total_logs_seen += 1 + invalid_ids << log.id unless log.valid? end - else - Rails.logger.info "No invalid #{log_type} found for year #{year}." + + Rails.logger.debug "Progress: #{invalid_ids.size} invalid logs found out of #{total_logs_seen} logs seen so far." end + + Rails.logger.info "Number of invalid #{log_type} for year #{year}: #{invalid_ids.size}" + Rails.logger.info "Invalid #{log_type} IDs: #{invalid_ids.join(', ')}" end - def fetch_invalid_logs(model, year) - model.filter_by_year(year).reject(&:valid?) + def surface_invalid_logs(model, log_type, year) + model.filter_by_year(year).find_in_batches(batch_size: 1000) do |batch| + batch.each do |log| + next if log.valid? + + error_messages = log.errors.full_messages.join(";\n") + Rails.logger.info "#{log_type} ID: #{log.id} \n Errors----\n #{error_messages}" + end + end end end