CLDC-2684 Retrieve recent export before creating new one (#1860)

* Retrieve recent export before creating new ones

* remove comment

app/services/exports/lettings_log_export_service.rb

@@ -13,9 +13,10 @@ module Exports
       daily_run_number = get_daily_run_number
       archives_for_manifest = {}
       base_number = LogsExport.where(empty_export: false).maximum(:base_number) || 1
+      recent_export = LogsExport.order("started_at").last
       available_collection_years.each do |collection|
         export = build_export_run(collection, start_time, base_number, full_update)
-        archives = write_export_archive(export, collection, start_time, full_update)
+        archives = write_export_archive(export, collection, start_time, recent_export, full_update)
 
         archives_for_manifest.merge!(archives)
 
@@ -71,13 +72,11 @@ module Exports
       "core_#{collection}_#{collection + 1}_apr_mar_#{base_number_str}_#{increment_str}".downcase
     end
 
-    def write_export_archive(export, collection, start_time, full_update)
+    def write_export_archive(export, collection, start_time, recent_export, full_update)
-      @logger.info("Writing export archives")
       archive = get_archive_name(collection, export.base_number, export.increment_number) # archive name would be the same for all logs because they're already filtered by year (?)
 
-      # Write archive
+      logs_count = retrieve_lettings_logs(start_time, recent_export, full_update).filter_by_year(collection).count
-      logs_count = retrieve_lettings_logs(start_time, full_update).filter_by_year(collection).count
+      @logger.info("Creating #{archive} - #{logs_count} logs")
-      @logger.info("Writing #{archive} - #{logs_count} logs")
       manifest_xml = build_manifest_xml(logs_count)
       return {} if logs_count.zero?
 
@@ -89,12 +88,12 @@ module Exports
 
       loop do
         lettings_logs_slice = if last_processed_marker.present?
-                                retrieve_lettings_logs(start_time, full_update).filter_by_year(collection)
+                                retrieve_lettings_logs(start_time, recent_export, full_update).filter_by_year(collection)
                                       .where("created_at > ?", last_processed_marker)
                                       .order(:created_at)
                                       .limit(MAX_XML_RECORDS)
                               else
-                                retrieve_lettings_logs(start_time, full_update).filter_by_year(collection)
+                                retrieve_lettings_logs(start_time, recent_export, full_update).filter_by_year(collection)
                                 .order(:created_at)
                                 .limit(MAX_XML_RECORDS)
                               end
@@ -103,7 +102,6 @@ module Exports
 
         data_xml = build_export_xml(lettings_logs_slice)
         part_number_str = "pt#{part_number.to_s.rjust(3, '0')}"
-        @logger.info("Adding #{archive}_#{part_number_str}.xml")
         zip_file.add("#{archive}_#{part_number_str}.xml", data_xml)
         part_number += 1
         last_processed_marker = lettings_logs_slice.last.created_at
@@ -112,15 +110,12 @@ module Exports
       # Required by S3 to avoid Aws::S3::Errors::BadDigest
       zip_io = zip_file.write_buffer
       zip_io.rewind
-      @logger.info("Writting #{archive}.zip")
+      @logger.info("Writing #{archive}.zip")
       @storage_service.write_file("#{archive}.zip", zip_io)
       { archive => Time.zone.now }
     end
 
-    def retrieve_lettings_logs(start_time, full_update)
+    def retrieve_lettings_logs(start_time, recent_export, full_update)
-      @logger.info("Retrieving lettings logs")
-      recent_export = LogsExport.order("started_at").last
-
       if !full_update && recent_export
         params = { from: recent_export.started_at, to: start_time }
         LettingsLog.exportable.where("updated_at >= :from and updated_at <= :to", params)

spec/services/exports/lettings_log_export_service_spec.rb

@@ -33,7 +33,7 @@ RSpec.describe Exports::LettingsLogExportService do
   end
 
   before do
-    Timecop.freeze(start_time)
+    Timecop.travel(start_time)
     allow(storage_service).to receive(:write_file)
 
     # Stub the form handler to use the real form
@@ -43,7 +43,7 @@ RSpec.describe Exports::LettingsLogExportService do
   end
 
   after do
-    Timecop.unfreeze
+    Timecop.return
   end
 
   context "when exporting daily lettings logs in XML" do
@@ -206,6 +206,14 @@ RSpec.describe Exports::LettingsLogExportService do
         it "generates multiple ZIP export files with the expected filenames" do
           expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args)
           expect(storage_service).to receive(:write_file).with(expected_zip_filename2, any_args)
+          expect(Rails.logger).to receive(:info).with("Building export run for 2021")
+          expect(Rails.logger).to receive(:info).with("Creating core_2021_2022_apr_mar_f0001_inc0001 - 1 logs")
+          expect(Rails.logger).to receive(:info).with("Writing core_2021_2022_apr_mar_f0001_inc0001.zip")
+          expect(Rails.logger).to receive(:info).with("Building export run for 2022")
+          expect(Rails.logger).to receive(:info).with("Creating core_2022_2023_apr_mar_f0001_inc0001 - 1 logs")
+          expect(Rails.logger).to receive(:info).with("Writing core_2022_2023_apr_mar_f0001_inc0001.zip")
+          expect(Rails.logger).to receive(:info).with("Building export run for 2023")
+          expect(Rails.logger).to receive(:info).with("Creating core_2023_2024_apr_mar_f0001_inc0001 - 0 logs")
 
           export_service.export_xml_lettings_logs
         end
@@ -232,7 +240,7 @@ RSpec.describe Exports::LettingsLogExportService do
       it "creates a logs export record in a database with correct time" do
         expect { export_service.export_xml_lettings_logs }
           .to change(LogsExport, :count).by(3)
-        expect(LogsExport.last.started_at).to eq(start_time)
+        expect(LogsExport.last.started_at).to be_within(2.seconds).of(start_time)
       end
 
       context "when this is the first export (full)" do