Browse Source

CLDC-2684 Add some logging to export (#1856)

* Add some logging to export

* lint
pull/1859/head
kosiakkatrina 1 year ago committed by GitHub
parent
commit
91424c9049
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 11
      app/services/exports/lettings_log_export_service.rb

11
app/services/exports/lettings_log_export_service.rb

@ -36,6 +36,7 @@ module Exports
end end
def build_export_run(collection, current_time, base_number, full_update) def build_export_run(collection, current_time, base_number, full_update)
@logger.info("Building export run for #{collection}")
previous_exports_with_data = LogsExport.where(collection:, empty_export: false) previous_exports_with_data = LogsExport.where(collection:, empty_export: false)
increment_number = previous_exports_with_data.where(base_number:).maximum(:increment_number) || 1 increment_number = previous_exports_with_data.where(base_number:).maximum(:increment_number) || 1
@ -76,6 +77,7 @@ module Exports
end end
def write_export_archive(export, lettings_logs) def write_export_archive(export, lettings_logs)
@logger.info("Writing export archives")
# Order lettings logs per archive # Order lettings logs per archive
lettings_logs_per_archive = {} lettings_logs_per_archive = {}
lettings_logs.each do |lettings_log| lettings_logs.each do |lettings_log|
@ -91,6 +93,11 @@ module Exports
# Write all archives # Write all archives
archive_datetimes = {} archive_datetimes = {}
@logger.info("Following archives to write:")
# rubocop:disable Style/CombinableLoops
lettings_logs_per_archive.each do |archive, lettings_logs_to_export|
@logger.info("#{archive} - #{lettings_logs_to_export.count} logs")
end
lettings_logs_per_archive.each do |archive, lettings_logs_to_export| lettings_logs_per_archive.each do |archive, lettings_logs_to_export|
manifest_xml = build_manifest_xml(lettings_logs_to_export.count) manifest_xml = build_manifest_xml(lettings_logs_to_export.count)
zip_file = Zip::File.open_buffer(StringIO.new) zip_file = Zip::File.open_buffer(StringIO.new)
@ -100,6 +107,7 @@ module Exports
lettings_logs_to_export.each_slice(MAX_XML_RECORDS) do |lettings_logs_slice| lettings_logs_to_export.each_slice(MAX_XML_RECORDS) do |lettings_logs_slice|
data_xml = build_export_xml(lettings_logs_slice) data_xml = build_export_xml(lettings_logs_slice)
part_number_str = "pt#{part_number.to_s.rjust(3, '0')}" part_number_str = "pt#{part_number.to_s.rjust(3, '0')}"
@logger.info("Adding #{archive}_#{part_number_str}.xml")
zip_file.add("#{archive}_#{part_number_str}.xml", data_xml) zip_file.add("#{archive}_#{part_number_str}.xml", data_xml)
part_number += 1 part_number += 1
end end
@ -107,14 +115,17 @@ module Exports
# Required by S3 to avoid Aws::S3::Errors::BadDigest # Required by S3 to avoid Aws::S3::Errors::BadDigest
zip_io = zip_file.write_buffer zip_io = zip_file.write_buffer
zip_io.rewind zip_io.rewind
@logger.info("Writting #{archive}.zip")
@storage_service.write_file("#{archive}.zip", zip_io) @storage_service.write_file("#{archive}.zip", zip_io)
archive_datetimes[archive] = Time.zone.now archive_datetimes[archive] = Time.zone.now
end end
# rubocop:enable Style/CombinableLoops
archive_datetimes archive_datetimes
end end
def retrieve_lettings_logs(start_time, full_update) def retrieve_lettings_logs(start_time, full_update)
@logger.info("Retrieving lettings logs")
recent_export = LogsExport.order("started_at").last recent_export = LogsExport.order("started_at").last
if !full_update && recent_export if !full_update && recent_export

Loading…
Cancel
Save