From c49a2ca17aede5b18a2bf3871ee5ea7898c871ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Meny?= Date: Wed, 18 May 2022 14:11:34 +0100 Subject: [PATCH] Add support for base and increment number --- .../exports/case_log_export_service.rb | 51 ++++++++++++++----- ...0220518115438_add_missing_export_fields.rb | 9 ++++ db/schema.rb | 5 +- .../exports/case_log_export_service_spec.rb | 31 ++++++----- 4 files changed, 64 insertions(+), 32 deletions(-) create mode 100644 db/migrate/20220518115438_add_missing_export_fields.rb diff --git a/app/services/exports/case_log_export_service.rb b/app/services/exports/case_log_export_service.rb index fcabc1e8d..3b355ba78 100644 --- a/app/services/exports/case_log_export_service.rb +++ b/app/services/exports/case_log_export_service.rb @@ -8,6 +8,7 @@ module Exports }.freeze LOG_ID_OFFSET = 300_000_000_000 + MAX_XML_RECORDS = 10_000 def initialize(storage_service, logger = Rails.logger) @storage_service = storage_service @@ -18,8 +19,9 @@ module Exports current_time = Time.zone.now case_logs = retrieve_case_logs(current_time) export = build_export_run(current_time) - write_master_manifest(export.daily_run_number) - write_export_archive(case_logs) + daily_run = get_daily_run_number + archive_list = write_export_archive(case_logs) + write_master_manifest(daily_run, archive_list) export.save! end @@ -31,20 +33,32 @@ module Exports private - def build_export_run(current_time) + def get_daily_run_number today = Time.zone.today - last_daily_run_number = LogsExport.where(created_at: today.beginning_of_day..today.end_of_day).maximum(:daily_run_number) - last_daily_run_number = 0 if last_daily_run_number.nil? + LogsExport.where(created_at: today.beginning_of_day..today.end_of_day).count + 1 + end + + def build_export_run(current_time, full_update = false) + if LogsExport.count == 0 + return LogsExport.new(started_at: current_time) + end - export = LogsExport.new - export.daily_run_number = last_daily_run_number + 1 - export.started_at = current_time - export + base_number = LogsExport.maximum(:base_number) + increment_number = LogsExport.where(base_number:).maximum(:increment_number) + + if full_update + base_number += 1 + increment_number = 1 + else + increment_number += 1 + end + + LogsExport.new(started_at: current_time, base_number:, increment_number:) end - def write_master_manifest(daily_run_number) + def write_master_manifest(daily_run, archive_list) today = Time.zone.today - increment_number = daily_run_number.to_s.rjust(4, "0") + increment_number = daily_run.to_s.rjust(4, "0") month = today.month.to_s.rjust(2, "0") day = today.day.to_s.rjust(2, "0") file_path = "Manifest_#{today.year}_#{month}_#{day}_#{increment_number}.csv" @@ -57,7 +71,7 @@ module Exports month = case_log.startdate.month quarter = QUARTERS[(month - 1) / 3] base_number_str = "f#{base_number.to_s.rjust(4, '0')}" - increment_str = "inc#{increment.to_s.rjust(3, '0')}" + increment_str = "inc#{increment.to_s.rjust(4, '0')}" "core_#{collection_start}_#{collection_start + 1}_#{quarter}_#{base_number_str}_#{increment_str}" end @@ -75,13 +89,22 @@ module Exports # Write all archives case_logs_per_archive.each do |archive, case_logs_to_export| - data_xml = build_export_xml(case_logs_to_export) manifest_xml = build_manifest_xml(case_logs_to_export.count) zip_io = Zip::File.open_buffer(StringIO.new) - zip_io.add("#{archive}.xml", data_xml) zip_io.add("manifest.xml", manifest_xml) + + part_number = 1 + case_logs_to_export.each_slice(MAX_XML_RECORDS) do |case_logs_slice| + data_xml = build_export_xml(case_logs_slice) + part_number_str = "pt#{part_number.to_s.rjust(3, '0')}" + zip_io.add("#{archive}_#{part_number_str}.xml", data_xml) + part_number += 1 + end + @storage_service.write_file("#{archive}.zip", zip_io.write_buffer) end + + case_logs_per_archive.keys end def retrieve_case_logs(current_time) diff --git a/db/migrate/20220518115438_add_missing_export_fields.rb b/db/migrate/20220518115438_add_missing_export_fields.rb new file mode 100644 index 000000000..971f61f5e --- /dev/null +++ b/db/migrate/20220518115438_add_missing_export_fields.rb @@ -0,0 +1,9 @@ +class AddMissingExportFields < ActiveRecord::Migration[7.0] + def change + change_table :logs_exports, bulk: true do |t| + t.column :base_number, :integer, default: 1, null: false + t.column :increment_number, :integer, default: 1, null: false + t.remove :daily_run_number, type: :integer + end + end +end diff --git a/db/schema.rb b/db/schema.rb index be67fb5f8..923f33204 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2022_05_16_111514) do +ActiveRecord::Schema[7.0].define(version: 2022_05_18_115438) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -259,9 +259,10 @@ ActiveRecord::Schema[7.0].define(version: 2022_05_16_111514) do end create_table "logs_exports", force: :cascade do |t| - t.integer "daily_run_number" t.datetime "created_at", default: -> { "CURRENT_TIMESTAMP" } t.datetime "started_at" + t.integer "base_number", default: 1, null: false + t.integer "increment_number", default: 1, null: false end create_table "organisation_las", force: :cascade do |t| diff --git a/spec/services/exports/case_log_export_service_spec.rb b/spec/services/exports/case_log_export_service_spec.rb index 37f1a9f38..30d9c58df 100644 --- a/spec/services/exports/case_log_export_service_spec.rb +++ b/spec/services/exports/case_log_export_service_spec.rb @@ -8,7 +8,7 @@ RSpec.describe Exports::CaseLogExportService do let(:expected_master_manifest_filename) { "Manifest_2022_05_01_0001.csv" } let(:expected_master_manifest_rerun) { "Manifest_2022_05_01_0002.csv" } - let(:expected_zip_filename) { "core_2021_2022_jan_mar_f0001_inc001.zip" } + let(:expected_zip_filename) { "core_2021_2022_jan_mar_f0001_inc0001.zip" } let(:expected_manifest_filename) { "manifest.xml" } let!(:case_log) { FactoryBot.create(:case_log, :completed) } @@ -51,7 +51,7 @@ RSpec.describe Exports::CaseLogExportService do end context "and one case log is available for export" do - let(:expected_data_filename) { "core_2021_2022_jan_mar_f0001_inc001.xml" } + let(:expected_data_filename) { "core_2021_2022_jan_mar_f0001_inc0001_pt001.xml" } it "generates a ZIP export file with the expected filename" do expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) @@ -100,12 +100,13 @@ RSpec.describe Exports::CaseLogExportService do end context "and multiple case logs are available for export on different periods" do + let(:expected_zip_filename2) { "core_2022_2023_apr_jun_f0001_inc0001.zip" } before { FactoryBot.create(:case_log, startdate: Time.zone.local(2022, 4, 1)) } context "when case logs are across multiple quarters" do it "generates multiple ZIP export files with the expected filenames" do - expect(storage_service).to receive(:write_file).with("core_2021_2022_jan_mar_f0001_inc001.zip", any_args) - expect(storage_service).to receive(:write_file).with("core_2022_2023_apr_jun_f0001_inc001.zip", any_args) + expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) + expect(storage_service).to receive(:write_file).with(expected_zip_filename2, any_args) export_service.export_case_logs end @@ -117,7 +118,7 @@ RSpec.describe Exports::CaseLogExportService do it "generates an XML manifest file with the expected content within the ZIP file" do expected_content = replace_record_number(local_manifest_file.read, 2) - allow(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) do |_, content| + expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) do |_, content| entry = Zip::File.open_buffer(content).find_entry(expected_manifest_filename) expect(entry).not_to be_nil expect(entry.get_input_stream.read).to eq(expected_content) @@ -136,17 +137,17 @@ RSpec.describe Exports::CaseLogExportService do it "records a ZIP archive in the master manifest (existing case logs)" do expect(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) do |_, csv_content| csv = CSV.parse(csv_content, headers: true) - expect(csv&.count).to eq(1) + expect(csv&.count).to be > 0 end export_service.export_case_logs end end - context "when this is an partial export" do + context "when this is a second export (partial)" do it "does not add any entry in the master manifest (no case logs)" do start_time = Time.zone.local(2022, 4, 1) - LogsExport.new(started_at: start_time, daily_run_number: 1).save! + LogsExport.new(started_at: start_time).save! expect(storage_service).to receive(:write_file).with(expected_master_manifest_rerun, any_args) do |_, csv_content| csv = CSV.parse(csv_content, headers: true) @@ -159,9 +160,7 @@ RSpec.describe Exports::CaseLogExportService do end context "and a previous export has run the same day" do - before do - export_service.export_case_logs - end + before { export_service.export_case_logs } it "increments the master manifest number by 1" do expect(storage_service).to receive(:write_file).with(expected_master_manifest_rerun, any_args) @@ -170,12 +169,12 @@ RSpec.describe Exports::CaseLogExportService do end context "when export has an error" do + before { allow(storage_service).to receive(:write_file).and_raise(StandardError.new("This is an exception")) } + it "does not save a record in the database" do - allow(storage_service).to receive(:write_file).and_raise(StandardError.new("This is an exception")) - export = LogsExport.new - allow(LogsExport).to receive(:new).and_return(export) - expect(export).not_to receive(:save!) - expect { export_service.export_case_logs }.to raise_error(StandardError) + expect { export_service.export_case_logs } + .to raise_error(StandardError) + .and(change(LogsExport, :count).by(0)) end end end