Refactor InvalidLogsHelper to improve invalid log counting and display

app/helpers/invalid_logs_helper.rb

@@ -1,23 +1,29 @@
 module InvalidLogsHelper
   def count_and_display_invalid_logs(model, log_type, year)
-    invalid_logs = fetch_invalid_logs(model, year)
+    invalid_ids = []
-    Rails.logger.info "Number of invalid #{log_type} for year #{year}: #{invalid_logs.size}"
+    total_logs_seen = 0
-    Rails.logger.info "Invalid #{log_type} IDs: #{invalid_logs.map(&:id).join(', ')}"
-  end
 
-  def surface_invalid_logs(model, log_type, year)
+    model.filter_by_year(year).find_in_batches(batch_size: 1000) do |batch|
-    invalid_logs = fetch_invalid_logs(model, year)
+      batch.each do |log|
-    if invalid_logs.any?
+        total_logs_seen += 1
-      invalid_logs.each do |log|
+        invalid_ids << log.id unless log.valid?
-        Rails.logger.info "#{log_type} ID: #{log.id}"
-        log.errors.full_messages.each { |message| Rails.logger.info "  - #{message}" }
       end
-    else
+
-      Rails.logger.info "No invalid #{log_type} found for year #{year}."
+      Rails.logger.debug "Progress: #{invalid_ids.size} invalid logs found out of #{total_logs_seen} logs seen so far."
     end
+
+    Rails.logger.info "Number of invalid #{log_type} for year #{year}: #{invalid_ids.size}"
+    Rails.logger.info "Invalid #{log_type} IDs: #{invalid_ids.join(', ')}"
   end
 
-  def fetch_invalid_logs(model, year)
+  def surface_invalid_logs(model, log_type, year)
-    model.filter_by_year(year).reject(&:valid?)
+    model.filter_by_year(year).find_in_batches(batch_size: 1000) do |batch|
+      batch.each do |log|
+        next if log.valid?
+
+        error_messages = log.errors.full_messages.join(";\n")
+        Rails.logger.info "#{log_type} ID: #{log.id} \n Errors----\n #{error_messages}"
+      end
+    end
   end
 end