diff --git a/app/services/exports/case_log_export_service.rb b/app/services/exports/case_log_export_service.rb
index fcabc1e8d..3b355ba78 100644
--- a/app/services/exports/case_log_export_service.rb
+++ b/app/services/exports/case_log_export_service.rb
@@ -8,6 +8,7 @@ module Exports
}.freeze
LOG_ID_OFFSET = 300_000_000_000
+ MAX_XML_RECORDS = 10_000
def initialize(storage_service, logger = Rails.logger)
@storage_service = storage_service
@@ -18,8 +19,9 @@ module Exports
current_time = Time.zone.now
case_logs = retrieve_case_logs(current_time)
export = build_export_run(current_time)
- write_master_manifest(export.daily_run_number)
- write_export_archive(case_logs)
+ daily_run = get_daily_run_number
+ archive_list = write_export_archive(case_logs)
+ write_master_manifest(daily_run, archive_list)
export.save!
end
@@ -31,20 +33,32 @@ module Exports
private
- def build_export_run(current_time)
+ def get_daily_run_number
today = Time.zone.today
- last_daily_run_number = LogsExport.where(created_at: today.beginning_of_day..today.end_of_day).maximum(:daily_run_number)
- last_daily_run_number = 0 if last_daily_run_number.nil?
+ LogsExport.where(created_at: today.beginning_of_day..today.end_of_day).count + 1
+ end
+
+ def build_export_run(current_time, full_update = false)
+ if LogsExport.count == 0
+ return LogsExport.new(started_at: current_time)
+ end
- export = LogsExport.new
- export.daily_run_number = last_daily_run_number + 1
- export.started_at = current_time
- export
+ base_number = LogsExport.maximum(:base_number)
+ increment_number = LogsExport.where(base_number:).maximum(:increment_number)
+
+ if full_update
+ base_number += 1
+ increment_number = 1
+ else
+ increment_number += 1
+ end
+
+ LogsExport.new(started_at: current_time, base_number:, increment_number:)
end
- def write_master_manifest(daily_run_number)
+ def write_master_manifest(daily_run, archive_list)
today = Time.zone.today
- increment_number = daily_run_number.to_s.rjust(4, "0")
+ increment_number = daily_run.to_s.rjust(4, "0")
month = today.month.to_s.rjust(2, "0")
day = today.day.to_s.rjust(2, "0")
file_path = "Manifest_#{today.year}_#{month}_#{day}_#{increment_number}.csv"
@@ -57,7 +71,7 @@ module Exports
month = case_log.startdate.month
quarter = QUARTERS[(month - 1) / 3]
base_number_str = "f#{base_number.to_s.rjust(4, '0')}"
- increment_str = "inc#{increment.to_s.rjust(3, '0')}"
+ increment_str = "inc#{increment.to_s.rjust(4, '0')}"
"core_#{collection_start}_#{collection_start + 1}_#{quarter}_#{base_number_str}_#{increment_str}"
end
@@ -75,13 +89,22 @@ module Exports
# Write all archives
case_logs_per_archive.each do |archive, case_logs_to_export|
- data_xml = build_export_xml(case_logs_to_export)
manifest_xml = build_manifest_xml(case_logs_to_export.count)
zip_io = Zip::File.open_buffer(StringIO.new)
- zip_io.add("#{archive}.xml", data_xml)
zip_io.add("manifest.xml", manifest_xml)
+
+ part_number = 1
+ case_logs_to_export.each_slice(MAX_XML_RECORDS) do |case_logs_slice|
+ data_xml = build_export_xml(case_logs_slice)
+ part_number_str = "pt#{part_number.to_s.rjust(3, '0')}"
+ zip_io.add("#{archive}_#{part_number_str}.xml", data_xml)
+ part_number += 1
+ end
+
@storage_service.write_file("#{archive}.zip", zip_io.write_buffer)
end
+
+ case_logs_per_archive.keys
end
def retrieve_case_logs(current_time)
diff --git a/db/migrate/20220518115438_add_missing_export_fields.rb b/db/migrate/20220518115438_add_missing_export_fields.rb
new file mode 100644
index 000000000..971f61f5e
--- /dev/null
+++ b/db/migrate/20220518115438_add_missing_export_fields.rb
@@ -0,0 +1,9 @@
+class AddMissingExportFields < ActiveRecord::Migration[7.0]
+ def change
+ change_table :logs_exports, bulk: true do |t|
+ t.column :base_number, :integer, default: 1, null: false
+ t.column :increment_number, :integer, default: 1, null: false
+ t.remove :daily_run_number, type: :integer
+ end
+ end
+end
diff --git a/db/schema.rb b/db/schema.rb
index be67fb5f8..923f33204 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema[7.0].define(version: 2022_05_16_111514) do
+ActiveRecord::Schema[7.0].define(version: 2022_05_18_115438) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
@@ -259,9 +259,10 @@ ActiveRecord::Schema[7.0].define(version: 2022_05_16_111514) do
end
create_table "logs_exports", force: :cascade do |t|
- t.integer "daily_run_number"
t.datetime "created_at", default: -> { "CURRENT_TIMESTAMP" }
t.datetime "started_at"
+ t.integer "base_number", default: 1, null: false
+ t.integer "increment_number", default: 1, null: false
end
create_table "organisation_las", force: :cascade do |t|
diff --git a/spec/services/exports/case_log_export_service_spec.rb b/spec/services/exports/case_log_export_service_spec.rb
index 37f1a9f38..30d9c58df 100644
--- a/spec/services/exports/case_log_export_service_spec.rb
+++ b/spec/services/exports/case_log_export_service_spec.rb
@@ -8,7 +8,7 @@ RSpec.describe Exports::CaseLogExportService do
let(:expected_master_manifest_filename) { "Manifest_2022_05_01_0001.csv" }
let(:expected_master_manifest_rerun) { "Manifest_2022_05_01_0002.csv" }
- let(:expected_zip_filename) { "core_2021_2022_jan_mar_f0001_inc001.zip" }
+ let(:expected_zip_filename) { "core_2021_2022_jan_mar_f0001_inc0001.zip" }
let(:expected_manifest_filename) { "manifest.xml" }
let!(:case_log) { FactoryBot.create(:case_log, :completed) }
@@ -51,7 +51,7 @@ RSpec.describe Exports::CaseLogExportService do
end
context "and one case log is available for export" do
- let(:expected_data_filename) { "core_2021_2022_jan_mar_f0001_inc001.xml" }
+ let(:expected_data_filename) { "core_2021_2022_jan_mar_f0001_inc0001_pt001.xml" }
it "generates a ZIP export file with the expected filename" do
expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args)
@@ -100,12 +100,13 @@ RSpec.describe Exports::CaseLogExportService do
end
context "and multiple case logs are available for export on different periods" do
+ let(:expected_zip_filename2) { "core_2022_2023_apr_jun_f0001_inc0001.zip" }
before { FactoryBot.create(:case_log, startdate: Time.zone.local(2022, 4, 1)) }
context "when case logs are across multiple quarters" do
it "generates multiple ZIP export files with the expected filenames" do
- expect(storage_service).to receive(:write_file).with("core_2021_2022_jan_mar_f0001_inc001.zip", any_args)
- expect(storage_service).to receive(:write_file).with("core_2022_2023_apr_jun_f0001_inc001.zip", any_args)
+ expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args)
+ expect(storage_service).to receive(:write_file).with(expected_zip_filename2, any_args)
export_service.export_case_logs
end
@@ -117,7 +118,7 @@ RSpec.describe Exports::CaseLogExportService do
it "generates an XML manifest file with the expected content within the ZIP file" do
expected_content = replace_record_number(local_manifest_file.read, 2)
- allow(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) do |_, content|
+ expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) do |_, content|
entry = Zip::File.open_buffer(content).find_entry(expected_manifest_filename)
expect(entry).not_to be_nil
expect(entry.get_input_stream.read).to eq(expected_content)
@@ -136,17 +137,17 @@ RSpec.describe Exports::CaseLogExportService do
it "records a ZIP archive in the master manifest (existing case logs)" do
expect(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) do |_, csv_content|
csv = CSV.parse(csv_content, headers: true)
- expect(csv&.count).to eq(1)
+ expect(csv&.count).to be > 0
end
export_service.export_case_logs
end
end
- context "when this is an partial export" do
+ context "when this is a second export (partial)" do
it "does not add any entry in the master manifest (no case logs)" do
start_time = Time.zone.local(2022, 4, 1)
- LogsExport.new(started_at: start_time, daily_run_number: 1).save!
+ LogsExport.new(started_at: start_time).save!
expect(storage_service).to receive(:write_file).with(expected_master_manifest_rerun, any_args) do |_, csv_content|
csv = CSV.parse(csv_content, headers: true)
@@ -159,9 +160,7 @@ RSpec.describe Exports::CaseLogExportService do
end
context "and a previous export has run the same day" do
- before do
- export_service.export_case_logs
- end
+ before { export_service.export_case_logs }
it "increments the master manifest number by 1" do
expect(storage_service).to receive(:write_file).with(expected_master_manifest_rerun, any_args)
@@ -170,12 +169,12 @@ RSpec.describe Exports::CaseLogExportService do
end
context "when export has an error" do
+ before { allow(storage_service).to receive(:write_file).and_raise(StandardError.new("This is an exception")) }
+
it "does not save a record in the database" do
- allow(storage_service).to receive(:write_file).and_raise(StandardError.new("This is an exception"))
- export = LogsExport.new
- allow(LogsExport).to receive(:new).and_return(export)
- expect(export).not_to receive(:save!)
- expect { export_service.export_case_logs }.to raise_error(StandardError)
+ expect { export_service.export_case_logs }
+ .to raise_error(StandardError)
+ .and(change(LogsExport, :count).by(0))
end
end
end