Count and report invalid logs efficiency improvements (#3066)

* Add tasks to just count and surface invalid LettingsLog and SalesLog

* Refactor InvalidLogsHelper module structure

* Refactor InvalidLogsHelper for improved logging and module structure

* Update logging level in InvalidLogsHelper from debug to info for better visibility

* Reorganise includes in invalid_logs.rake

* Refactor InvalidLogsHelper to improve invalid log counting and display

* Enhance logging in InvalidLogsHelper for better tracking of invalid log processing

* Filter logs by status to only process completed entries in InvalidLogsHelper

* Enhance invalid log detection in InvalidLogsHelper to include incomplete subsections and questions

* Refactor InvalidLogsHelper to filter logs by status and enhance invalid log reporting

* Refactor InvalidLogsHelper to optimize invalid log detection and streamline batch processing

* Lint

app/helpers/invalid_logs_helper.rb

@@ -2,24 +2,18 @@ module InvalidLogsHelper
   def count_and_display_invalid_logs(model, log_type, year)
     Rails.logger.info "Starting to count invalid #{log_type} for year #{year}..."
     invalid_ids = []
-    total_logs_seen = 0
+    total_logs_checked = 0
 
-    model.filter_by_year(year).find_in_batches(batch_size: 1000).with_index(1) do |batch, batch_index|
+    model.filter_by_year(year).where(status: "completed").find_in_batches(batch_size: 1000).with_index(1) do |batch, batch_index|
       Rails.logger.info "Processing batch #{batch_index} with #{batch.size} logs..."
-      batch.each do |log|
+      invalid_logs = batch.select { |log| log_invalid?(log) }
-        next unless log.status == "completed"
+      invalid_ids.concat(invalid_logs.map(&:id))
+      total_logs_checked += batch.size
 
-        total_logs_seen += 1
+      Rails.logger.info "Batch #{batch_index} complete. Progress: #{invalid_ids.size} invalid logs found out of #{total_logs_checked} completed logs checked so far."
-
-        next unless !log.valid? || log.incomplete_subsections.any? || log.incomplete_questions.any?
-
-        invalid_ids << log.id
-      end
-
-      Rails.logger.info "Batch #{batch_index} complete. Progress: #{invalid_ids.size} invalid logs found out of #{total_logs_seen} logs seen so far."
     end
 
-    Rails.logger.info "Counting complete for #{log_type}. Total invalid logs: #{invalid_ids.size}, Total logs seen: #{total_logs_seen}."
+    Rails.logger.info "Counting complete for #{log_type}. Total invalid logs: #{invalid_ids.size}, Total completed logs checked: #{total_logs_checked}."
     Rails.logger.info "Invalid #{log_type} IDs: #{invalid_ids.join(', ')}" if invalid_ids.any?
     Rails.logger.info "--------------------------------"
   end
@@ -27,26 +21,119 @@ module InvalidLogsHelper
   def surface_invalid_logs(model, log_type, year)
     Rails.logger.info "Surfacing invalid #{log_type} for year #{year}..."
     invalid_ids = []
-    total_logs_seen = 0
+    total_logs_checked = 0
+    log_messages = [headers(log_type).join(", ")]
 
-    model.filter_by_year(year).find_in_batches(batch_size: 1000).with_index(1) do |batch, batch_index|
+    model.filter_by_year(year).where(status: "completed").find_in_batches(batch_size: 1000).with_index(1) do |batch, batch_index|
       Rails.logger.info "Processing batch #{batch_index} with #{batch.size} logs..."
-      batch.each do |log|
-        next unless log.status == "completed"
-
-        total_logs_seen += 1
 
-        next unless !log.valid? || log.incomplete_subsections.any? || log.incomplete_questions.any?
+      invalid_logs = batch.select { |log| log_invalid?(log) }
-
+      invalid_ids.concat(invalid_logs.map(&:id))
-        invalid_ids << log.id
+      invalid_logs.each do |log|
-        error_messages = log.errors.full_messages.join(";\n")
+        log_row_data = log_row(log, log_type).join(", ")
-        Rails.logger.info "#{log_type} ID: #{log.id} \n Errors----\n #{error_messages}"
+        log_messages << log_row_data
+        Rails.logger.info log_row_data
       end
       Rails.logger.info "Batch #{batch_index} complete. Processed #{batch.size} logs."
     end
 
-    Rails.logger.info "Surfacing complete for #{log_type}. Total invalid logs: #{invalid_ids.size}, Total logs seen: #{total_logs_seen}."
+    Rails.logger.info "Surfacing complete for #{log_type}. Total invalid logs: #{invalid_ids.size}, Total completed logs checked: #{total_logs_checked}."
     Rails.logger.info "Invalid #{log_type} IDs: #{invalid_ids.join(', ')}" if invalid_ids.any?
+    Rails.logger.info log_messages.join("\n")
     Rails.logger.info "--------------------------------"
   end
+
+private
+
+  def log_invalid?(log)
+    !log.valid? || log.incomplete_subsections.any? || log.incomplete_questions.any?
+  end
+
+  def log_row(log, log_type)
+    incomplete_subsections = log.incomplete_subsections.map(&:label).join("; ")
+    incomplete_question_ids = log.incomplete_questions.map(&:id).join("; ")
+
+    created_at = log.created_at&.strftime("%d/%m/%Y")
+    updated_at = log.updated_at&.strftime("%d/%m/%Y")
+
+    if log_type == "LettingsLog"
+      [
+        log.id,
+        log.tenancycode,
+        log.propcode,
+        log.managing_organisation_id,
+        log.managing_organisation&.name,
+        log.assigned_to&.email,
+        log.address_line1,
+        log.address_line2,
+        log.town_or_city,
+        log.county,
+        log.postcode_full,
+        incomplete_subsections,
+        incomplete_question_ids,
+        log.status,
+        created_at,
+        updated_at,
+      ]
+    else
+      [
+        log.id,
+        log.purchid,
+        log.managing_organisation_id,
+        log.managing_organisation&.name,
+        log.assigned_to&.email,
+        log.address_line1,
+        log.address_line2,
+        log.town_or_city,
+        log.county,
+        log.postcode_full,
+        incomplete_subsections,
+        incomplete_question_ids,
+        log.status,
+        created_at,
+        updated_at,
+      ]
+    end
+  end
+
+  def headers(log_type)
+    if log_type == "LettingsLog"
+      [
+        "Log ID",
+        "Tenant Code",
+        "Property Reference",
+        "Managing Organisation ID",
+        "Managing Organisation Name",
+        "Assigned To (User Email)",
+        "Address Line 1",
+        "Address Line 2",
+        "Town/City",
+        "County",
+        "Postcode",
+        "Incomplete Subsections",
+        "Incomplete Question IDs",
+        "Status",
+        "Created At",
+        "Updated At",
+      ]
+    else
+      [
+        "Log ID",
+        "Purchaser Code",
+        "Managing Organisation ID",
+        "Managing Organisation Name",
+        "Assigned To (User Email)",
+        "Address Line 1",
+        "Address Line 2",
+        "Town/City",
+        "County",
+        "Postcode",
+        "Incomplete Subsections",
+        "Incomplete Question IDs",
+        "Status",
+        "Created At",
+        "Updated At",
+      ]
+    end
+  end
 end