From e88e1bae8a10591ae2b7a0ca2bfad0179081482a Mon Sep 17 00:00:00 2001 From: kosiakkatrina <54268893+kosiakkatrina@users.noreply.github.com> Date: Wed, 23 Aug 2023 12:17:54 +0100 Subject: [PATCH] CLDC-2684 Retrieve recent export before creating new one (#1860) * Retrieve recent export before creating new ones * remove comment --- .../exports/lettings_log_export_service.rb | 23 ++++++++----------- .../lettings_log_export_service_spec.rb | 14 ++++++++--- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/app/services/exports/lettings_log_export_service.rb b/app/services/exports/lettings_log_export_service.rb index 316ea8bc5..d7af5867c 100644 --- a/app/services/exports/lettings_log_export_service.rb +++ b/app/services/exports/lettings_log_export_service.rb @@ -13,9 +13,10 @@ module Exports daily_run_number = get_daily_run_number archives_for_manifest = {} base_number = LogsExport.where(empty_export: false).maximum(:base_number) || 1 + recent_export = LogsExport.order("started_at").last available_collection_years.each do |collection| export = build_export_run(collection, start_time, base_number, full_update) - archives = write_export_archive(export, collection, start_time, full_update) + archives = write_export_archive(export, collection, start_time, recent_export, full_update) archives_for_manifest.merge!(archives) @@ -71,13 +72,11 @@ module Exports "core_#{collection}_#{collection + 1}_apr_mar_#{base_number_str}_#{increment_str}".downcase end - def write_export_archive(export, collection, start_time, full_update) - @logger.info("Writing export archives") + def write_export_archive(export, collection, start_time, recent_export, full_update) archive = get_archive_name(collection, export.base_number, export.increment_number) # archive name would be the same for all logs because they're already filtered by year (?) - # Write archive - logs_count = retrieve_lettings_logs(start_time, full_update).filter_by_year(collection).count - @logger.info("Writing #{archive} - #{logs_count} logs") + logs_count = retrieve_lettings_logs(start_time, recent_export, full_update).filter_by_year(collection).count + @logger.info("Creating #{archive} - #{logs_count} logs") manifest_xml = build_manifest_xml(logs_count) return {} if logs_count.zero? @@ -89,12 +88,12 @@ module Exports loop do lettings_logs_slice = if last_processed_marker.present? - retrieve_lettings_logs(start_time, full_update).filter_by_year(collection) + retrieve_lettings_logs(start_time, recent_export, full_update).filter_by_year(collection) .where("created_at > ?", last_processed_marker) .order(:created_at) .limit(MAX_XML_RECORDS) else - retrieve_lettings_logs(start_time, full_update).filter_by_year(collection) + retrieve_lettings_logs(start_time, recent_export, full_update).filter_by_year(collection) .order(:created_at) .limit(MAX_XML_RECORDS) end @@ -103,7 +102,6 @@ module Exports data_xml = build_export_xml(lettings_logs_slice) part_number_str = "pt#{part_number.to_s.rjust(3, '0')}" - @logger.info("Adding #{archive}_#{part_number_str}.xml") zip_file.add("#{archive}_#{part_number_str}.xml", data_xml) part_number += 1 last_processed_marker = lettings_logs_slice.last.created_at @@ -112,15 +110,12 @@ module Exports # Required by S3 to avoid Aws::S3::Errors::BadDigest zip_io = zip_file.write_buffer zip_io.rewind - @logger.info("Writting #{archive}.zip") + @logger.info("Writing #{archive}.zip") @storage_service.write_file("#{archive}.zip", zip_io) { archive => Time.zone.now } end - def retrieve_lettings_logs(start_time, full_update) - @logger.info("Retrieving lettings logs") - recent_export = LogsExport.order("started_at").last - + def retrieve_lettings_logs(start_time, recent_export, full_update) if !full_update && recent_export params = { from: recent_export.started_at, to: start_time } LettingsLog.exportable.where("updated_at >= :from and updated_at <= :to", params) diff --git a/spec/services/exports/lettings_log_export_service_spec.rb b/spec/services/exports/lettings_log_export_service_spec.rb index b4adb9bd7..2731aba87 100644 --- a/spec/services/exports/lettings_log_export_service_spec.rb +++ b/spec/services/exports/lettings_log_export_service_spec.rb @@ -33,7 +33,7 @@ RSpec.describe Exports::LettingsLogExportService do end before do - Timecop.freeze(start_time) + Timecop.travel(start_time) allow(storage_service).to receive(:write_file) # Stub the form handler to use the real form @@ -43,7 +43,7 @@ RSpec.describe Exports::LettingsLogExportService do end after do - Timecop.unfreeze + Timecop.return end context "when exporting daily lettings logs in XML" do @@ -206,6 +206,14 @@ RSpec.describe Exports::LettingsLogExportService do it "generates multiple ZIP export files with the expected filenames" do expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) expect(storage_service).to receive(:write_file).with(expected_zip_filename2, any_args) + expect(Rails.logger).to receive(:info).with("Building export run for 2021") + expect(Rails.logger).to receive(:info).with("Creating core_2021_2022_apr_mar_f0001_inc0001 - 1 logs") + expect(Rails.logger).to receive(:info).with("Writing core_2021_2022_apr_mar_f0001_inc0001.zip") + expect(Rails.logger).to receive(:info).with("Building export run for 2022") + expect(Rails.logger).to receive(:info).with("Creating core_2022_2023_apr_mar_f0001_inc0001 - 1 logs") + expect(Rails.logger).to receive(:info).with("Writing core_2022_2023_apr_mar_f0001_inc0001.zip") + expect(Rails.logger).to receive(:info).with("Building export run for 2023") + expect(Rails.logger).to receive(:info).with("Creating core_2023_2024_apr_mar_f0001_inc0001 - 0 logs") export_service.export_xml_lettings_logs end @@ -232,7 +240,7 @@ RSpec.describe Exports::LettingsLogExportService do it "creates a logs export record in a database with correct time" do expect { export_service.export_xml_lettings_logs } .to change(LogsExport, :count).by(3) - expect(LogsExport.last.started_at).to eq(start_time) + expect(LogsExport.last.started_at).to be_within(2.seconds).of(start_time) end context "when this is the first export (full)" do