From 3659b7bae6bd08fd3ddabd1996d176fbf5d8f53d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Meny?= Date: Tue, 17 May 2022 12:43:00 +0100 Subject: [PATCH] Start of Zip generation --- .../exports/case_log_export_service.rb | 59 +++++++++++++------ .../exports/case_log_export_service_spec.rb | 45 +++++++++----- 2 files changed, 73 insertions(+), 31 deletions(-) diff --git a/app/services/exports/case_log_export_service.rb b/app/services/exports/case_log_export_service.rb index 5b9ef48b1..2e72a43f1 100644 --- a/app/services/exports/case_log_export_service.rb +++ b/app/services/exports/case_log_export_service.rb @@ -1,15 +1,29 @@ module Exports class CaseLogExportService + QUARTERS = { + 0 => "jan_mar", + 1 => "apr_jun", + 2 => "jul_sep", + 3 => "oct_dec" + }.freeze + + LOG_ID_OFFSET = 300_000_000_000 + def initialize(storage_service, logger = Rails.logger) @storage_service = storage_service @logger = logger end def export_case_logs + # Case log data is already ordered by startdate case_logs = retrieve_case_logs - export = save_export_run - write_master_manifest(export) + daily_run_number = get_next_run_number + write_master_manifest(daily_run_number) write_export_data(case_logs) + + export = LogsExport.new(daily_run_number:) + export.save! + export end def is_omitted_field?(field_name) @@ -18,44 +32,55 @@ module Exports field_name.starts_with?("details_known_") || pattern_age.match(field_name) || omitted_attrs.include?(field_name) ? true : false end - LOG_ID_OFFSET = 300_000_000_000 - private - def save_export_run + def get_next_run_number today = Time.zone.today last_daily_run_number = LogsExport.where(created_at: today.beginning_of_day..today.end_of_day).maximum(:daily_run_number) - - export = LogsExport.new if last_daily_run_number.nil? - export.daily_run_number = 1 + 1 else - export.daily_run_number = last_daily_run_number + 1 + last_daily_run_number + 1 end - export.save! - export end - def write_master_manifest(export) + def write_master_manifest(daily_run_number) today = Time.zone.today - increment_number = export.daily_run_number.to_s.rjust(4, "0") + increment_number = daily_run_number.to_s.rjust(4, "0") month = today.month.to_s.rjust(2, "0") day = today.day.to_s.rjust(2, "0") - file_path = "Manifest_#{today.year}-#{month}-#{day}_#{increment_number}.csv" + file_path = "Manifest_#{today.year}_#{month}_#{day}_#{increment_number}.csv" string_io = build_manifest_csv_io @storage_service.write_file(file_path, string_io) end + def get_zip_naming(case_log, base_number, increment) + collection_start = case_log.collection_start_year + month = case_log.startdate.month + quarter = QUARTERS[(month - 1) / 3] + base_number_str = "f#{base_number.to_s.rjust(4, '0')}" + increment_str = "inc#{increment.to_s.rjust(3, '0')}" + "core_#{collection_start}_#{collection_start + 1}_#{quarter}_#{base_number_str}_#{increment_str}.zip" + end + def write_export_data(case_logs) + zip_filenames = [] + case_logs.each do |case_log| + zip_filename= get_zip_naming(case_log, 1, 1) + unless zip_filenames.include?(zip_filename) + zip_io = Zip::File.open_buffer(StringIO.new) + zip_io.add("my_file", Tempfile.new) + @storage_service.write_file(zip_filename, zip_io.write_buffer) + end + end + string_io = build_export_xml_io(case_logs) file_path = "#{get_folder_name}/#{get_file_name}.xml" @storage_service.write_file(file_path, string_io) end def retrieve_case_logs - # All logs from previous (successful) start time to current start time (not current) [ignore status] - params = { from: Time.current.beginning_of_day, to: Time.current, status: CaseLog.statuses[:completed] } - CaseLog.where("updated_at >= :from and updated_at <= :to and status = :status", params) + CaseLog.all end def build_manifest_csv_io diff --git a/spec/services/exports/case_log_export_service_spec.rb b/spec/services/exports/case_log_export_service_spec.rb index 4fb431df4..e938d3e35 100644 --- a/spec/services/exports/case_log_export_service_spec.rb +++ b/spec/services/exports/case_log_export_service_spec.rb @@ -2,14 +2,9 @@ require "rails_helper" RSpec.describe Exports::CaseLogExportService do let(:storage_service) { instance_double(StorageService) } - let(:export_filepath) { "spec/fixtures/exports/case_logs.xml" } let(:export_file) { File.open(export_filepath, "r:UTF-8") } - - let(:expected_data_filename) { "core_2022_02_08/dat_core_2022_02_08_0001.xml" } - let(:expected_master_manifest_filename) { "Manifest_2022_02_08_0001.csv" } - let(:expected_master_manifest_filename2) { "Manifest_2022_02_08_0002.csv" } - + let(:expected_master_manifest_filename) { "Manifest_2022_05_01_0001.csv" } let(:case_log) { FactoryBot.create(:case_log, :completed) } def replace_entity_ids(export_template) @@ -22,10 +17,10 @@ RSpec.describe Exports::CaseLogExportService do context "when exporting daily case logs" do subject(:export_service) { described_class.new(storage_service) } - let(:case_log) { FactoryBot.create(:case_log, :completed) } + let!(:case_log) { FactoryBot.create(:case_log, :completed) } before do - Timecop.freeze(case_log.updated_at) + Timecop.freeze(2022, 5, 1) allow(storage_service).to receive(:write_file) end @@ -45,13 +40,21 @@ RSpec.describe Exports::CaseLogExportService do end end - context "and case logs are available for export" do - before do - case_log + context "and one case log is available for export" do + let(:expected_zip_filename) { "core_2021_2022_jan_mar_f0001_inc001.zip" } + let(:expected_data_filename) { "core_2022_02_08/dat_core_2022_02_08_0001.xml" } + + it "generates a ZIP export file with the expected filename" do + expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) + + export_service.export_case_logs end - it "generates an XML export file with the expected filename" do - expect(storage_service).to receive(:write_file).with(expected_data_filename, any_args) + it "generates an XML export file with the expected filename within the ZIP file" do + allow(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) do |_, content| + pp Zip::File.open_buffer(content).entries + expect(content).to eq("Manifest_2022_05_01_0001.csv") + end export_service.export_case_logs end @@ -65,13 +68,27 @@ RSpec.describe Exports::CaseLogExportService do end end + context "and multiple case logs are available for export" do + let!(:case_log_2) { FactoryBot.create(:case_log, startdate: Time.zone.local(2022, 4, 1)) } + + context "when case logs are across multiple quarters" do + it "generates multiple ZIP export files with the expected filenames" do + expect(storage_service).to receive(:write_file).with("core_2021_2022_jan_mar_f0001_inc001.zip", any_args) + expect(storage_service).to receive(:write_file).with("core_2022_2023_apr_jun_f0001_inc001.zip", any_args) + + export_service.export_case_logs + end + end + end + context "and a previous export has run the same day" do + let(:expected_master_manifest_rerun) { "Manifest_2022_05_01_0002.csv" } before do export_service.export_case_logs end it "increments the master manifest number by 1" do - expect(storage_service).to receive(:write_file).with(expected_master_manifest_filename2, any_args) + expect(storage_service).to receive(:write_file).with(expected_master_manifest_rerun, any_args) export_service.export_case_logs end end