From 5f4295d249340b01102807ffd88c3feceb09731e Mon Sep 17 00:00:00 2001
From: Manny Dinssa <44172848+Dinssa@users.noreply.github.com>
Date: Wed, 23 Apr 2025 20:42:19 +0100
Subject: [PATCH] Refactor InvalidLogsHelper to improve invalid log counting
 and display (#3064)

* Add tasks to just count and surface invalid LettingsLog and SalesLog

* Refactor InvalidLogsHelper module structure

* Refactor InvalidLogsHelper for improved logging and module structure

* Update logging level in InvalidLogsHelper from debug to info for better visibility

* Reorganise includes in invalid_logs.rake

* Refactor InvalidLogsHelper to improve invalid log counting and display
---
 app/helpers/invalid_logs_helper.rb | 34 ++++++++++++++++++------------
 1 file changed, 20 insertions(+), 14 deletions(-)

diff --git a/app/helpers/invalid_logs_helper.rb b/app/helpers/invalid_logs_helper.rb
index 00e8cb786..4cd0e2976 100644
--- a/app/helpers/invalid_logs_helper.rb
+++ b/app/helpers/invalid_logs_helper.rb
@@ -1,23 +1,29 @@
 module InvalidLogsHelper
   def count_and_display_invalid_logs(model, log_type, year)
-    invalid_logs = fetch_invalid_logs(model, year)
-    Rails.logger.info "Number of invalid #{log_type} for year #{year}: #{invalid_logs.size}"
-    Rails.logger.info "Invalid #{log_type} IDs: #{invalid_logs.map(&:id).join(', ')}"
-  end
+    invalid_ids = []
+    total_logs_seen = 0
 
-  def surface_invalid_logs(model, log_type, year)
-    invalid_logs = fetch_invalid_logs(model, year)
-    if invalid_logs.any?
-      invalid_logs.each do |log|
-        Rails.logger.info "#{log_type} ID: #{log.id}"
-        log.errors.full_messages.each { |message| Rails.logger.info "  - #{message}" }
+    model.filter_by_year(year).find_in_batches(batch_size: 1000) do |batch|
+      batch.each do |log|
+        total_logs_seen += 1
+        invalid_ids << log.id unless log.valid?
       end
-    else
-      Rails.logger.info "No invalid #{log_type} found for year #{year}."
+
+      Rails.logger.debug "Progress: #{invalid_ids.size} invalid logs found out of #{total_logs_seen} logs seen so far."
     end
+
+    Rails.logger.info "Number of invalid #{log_type} for year #{year}: #{invalid_ids.size}"
+    Rails.logger.info "Invalid #{log_type} IDs: #{invalid_ids.join(', ')}"
   end
 
-  def fetch_invalid_logs(model, year)
-    model.filter_by_year(year).reject(&:valid?)
+  def surface_invalid_logs(model, log_type, year)
+    model.filter_by_year(year).find_in_batches(batch_size: 1000) do |batch|
+      batch.each do |log|
+        next if log.valid?
+
+        error_messages = log.errors.full_messages.join(";\n")
+        Rails.logger.info "#{log_type} ID: #{log.id} \n Errors----\n #{error_messages}"
+      end
+    end
   end
 end