Add support for base and increment number

app/services/exports/case_log_export_service.rb

@@ -8,6 +8,7 @@ module Exports
     }.freeze
 
     LOG_ID_OFFSET = 300_000_000_000
+    MAX_XML_RECORDS = 10_000
 
     def initialize(storage_service, logger = Rails.logger)
       @storage_service = storage_service
@@ -18,8 +19,9 @@ module Exports
       current_time = Time.zone.now
       case_logs = retrieve_case_logs(current_time)
       export = build_export_run(current_time)
-      write_master_manifest(export.daily_run_number)
+      daily_run = get_daily_run_number
-      write_export_archive(case_logs)
+      archive_list = write_export_archive(case_logs)
+      write_master_manifest(daily_run, archive_list)
       export.save!
     end
 
@@ -31,20 +33,32 @@ module Exports
 
   private
 
-    def build_export_run(current_time)
+    def get_daily_run_number
       today = Time.zone.today
-      last_daily_run_number = LogsExport.where(created_at: today.beginning_of_day..today.end_of_day).maximum(:daily_run_number)
+      LogsExport.where(created_at: today.beginning_of_day..today.end_of_day).count + 1
-      last_daily_run_number = 0 if last_daily_run_number.nil?
+    end
+
+    def build_export_run(current_time, full_update = false)
+      if LogsExport.count == 0
+        return LogsExport.new(started_at: current_time)
+      end
 
-      export = LogsExport.new
+      base_number = LogsExport.maximum(:base_number)
-      export.daily_run_number = last_daily_run_number + 1
+      increment_number = LogsExport.where(base_number:).maximum(:increment_number)
-      export.started_at = current_time
+
-      export
+      if full_update
+        base_number += 1
+        increment_number = 1
+      else
+        increment_number += 1
       end
 
-    def write_master_manifest(daily_run_number)
+      LogsExport.new(started_at: current_time, base_number:, increment_number:)
+    end
+
+    def write_master_manifest(daily_run, archive_list)
       today = Time.zone.today
-      increment_number = daily_run_number.to_s.rjust(4, "0")
+      increment_number = daily_run.to_s.rjust(4, "0")
       month = today.month.to_s.rjust(2, "0")
       day = today.day.to_s.rjust(2, "0")
       file_path = "Manifest_#{today.year}_#{month}_#{day}_#{increment_number}.csv"
@@ -57,7 +71,7 @@ module Exports
       month = case_log.startdate.month
       quarter = QUARTERS[(month - 1) / 3]
       base_number_str = "f#{base_number.to_s.rjust(4, '0')}"
-      increment_str = "inc#{increment.to_s.rjust(3, '0')}"
+      increment_str = "inc#{increment.to_s.rjust(4, '0')}"
       "core_#{collection_start}_#{collection_start + 1}_#{quarter}_#{base_number_str}_#{increment_str}"
     end
 
@@ -75,13 +89,22 @@ module Exports
 
       # Write all archives
       case_logs_per_archive.each do |archive, case_logs_to_export|
-        data_xml = build_export_xml(case_logs_to_export)
         manifest_xml = build_manifest_xml(case_logs_to_export.count)
         zip_io = Zip::File.open_buffer(StringIO.new)
-        zip_io.add("#{archive}.xml", data_xml)
         zip_io.add("manifest.xml", manifest_xml)
+
+        part_number = 1
+        case_logs_to_export.each_slice(MAX_XML_RECORDS) do |case_logs_slice|
+          data_xml = build_export_xml(case_logs_slice)
+          part_number_str = "pt#{part_number.to_s.rjust(3, '0')}"
+          zip_io.add("#{archive}_#{part_number_str}.xml", data_xml)
+          part_number += 1
+        end
+
         @storage_service.write_file("#{archive}.zip", zip_io.write_buffer)
       end
+
+      case_logs_per_archive.keys
     end
 
     def retrieve_case_logs(current_time)

db/migrate/20220518115438_add_missing_export_fields.rb

@@ -0,0 +1,9 @@
+class AddMissingExportFields < ActiveRecord::Migration[7.0]
+  def change
+    change_table :logs_exports, bulk: true do |t|
+      t.column :base_number, :integer, default: 1, null: false
+      t.column :increment_number, :integer, default: 1, null: false
+      t.remove :daily_run_number, type: :integer
+    end
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2022_05_16_111514) do
+ActiveRecord::Schema[7.0].define(version: 2022_05_18_115438) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
@@ -259,9 +259,10 @@ ActiveRecord::Schema[7.0].define(version: 2022_05_16_111514) do
   end
 
   create_table "logs_exports", force: :cascade do |t|
-    t.integer "daily_run_number"
     t.datetime "created_at", default: -> { "CURRENT_TIMESTAMP" }
     t.datetime "started_at"
+    t.integer "base_number", default: 1, null: false
+    t.integer "increment_number", default: 1, null: false
   end
 
   create_table "organisation_las", force: :cascade do |t|

spec/services/exports/case_log_export_service_spec.rb

@@ -8,7 +8,7 @@ RSpec.describe Exports::CaseLogExportService do
 
   let(:expected_master_manifest_filename) { "Manifest_2022_05_01_0001.csv" }
   let(:expected_master_manifest_rerun) { "Manifest_2022_05_01_0002.csv" }
-  let(:expected_zip_filename) { "core_2021_2022_jan_mar_f0001_inc001.zip" }
+  let(:expected_zip_filename) { "core_2021_2022_jan_mar_f0001_inc0001.zip" }
   let(:expected_manifest_filename) { "manifest.xml" }
 
   let!(:case_log) { FactoryBot.create(:case_log, :completed) }
@@ -51,7 +51,7 @@ RSpec.describe Exports::CaseLogExportService do
     end
 
     context "and one case log is available for export" do
-      let(:expected_data_filename) { "core_2021_2022_jan_mar_f0001_inc001.xml" }
+      let(:expected_data_filename) { "core_2021_2022_jan_mar_f0001_inc0001_pt001.xml" }
 
       it "generates a ZIP export file with the expected filename" do
         expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args)
@@ -100,12 +100,13 @@ RSpec.describe Exports::CaseLogExportService do
     end
 
     context "and multiple case logs are available for export on different periods" do
+      let(:expected_zip_filename2) { "core_2022_2023_apr_jun_f0001_inc0001.zip" }
       before { FactoryBot.create(:case_log, startdate: Time.zone.local(2022, 4, 1)) }
 
       context "when case logs are across multiple quarters" do
         it "generates multiple ZIP export files with the expected filenames" do
-          expect(storage_service).to receive(:write_file).with("core_2021_2022_jan_mar_f0001_inc001.zip", any_args)
+          expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args)
-          expect(storage_service).to receive(:write_file).with("core_2022_2023_apr_jun_f0001_inc001.zip", any_args)
+          expect(storage_service).to receive(:write_file).with(expected_zip_filename2, any_args)
 
           export_service.export_case_logs
         end
@@ -117,7 +118,7 @@ RSpec.describe Exports::CaseLogExportService do
 
       it "generates an XML manifest file with the expected content within the ZIP file" do
         expected_content = replace_record_number(local_manifest_file.read, 2)
-        allow(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) do |_, content|
+        expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) do |_, content|
           entry = Zip::File.open_buffer(content).find_entry(expected_manifest_filename)
           expect(entry).not_to be_nil
           expect(entry.get_input_stream.read).to eq(expected_content)
@@ -136,17 +137,17 @@ RSpec.describe Exports::CaseLogExportService do
         it "records a ZIP archive in the master manifest (existing case logs)" do
           expect(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) do |_, csv_content|
             csv = CSV.parse(csv_content, headers: true)
-            expect(csv&.count).to eq(1)
+            expect(csv&.count).to be > 0
           end
 
           export_service.export_case_logs
         end
       end
 
-      context "when this is an partial export" do
+      context "when this is a second export (partial)" do
         it "does not add any entry in the master manifest (no case logs)" do
           start_time = Time.zone.local(2022, 4, 1)
-          LogsExport.new(started_at: start_time, daily_run_number: 1).save!
+          LogsExport.new(started_at: start_time).save!
 
           expect(storage_service).to receive(:write_file).with(expected_master_manifest_rerun, any_args) do |_, csv_content|
             csv = CSV.parse(csv_content, headers: true)
@@ -159,9 +160,7 @@ RSpec.describe Exports::CaseLogExportService do
     end
 
     context "and a previous export has run the same day" do
-      before do
+      before { export_service.export_case_logs }
-        export_service.export_case_logs
-      end
 
       it "increments the master manifest number by 1" do
         expect(storage_service).to receive(:write_file).with(expected_master_manifest_rerun, any_args)
@@ -170,12 +169,12 @@ RSpec.describe Exports::CaseLogExportService do
     end
 
     context "when export has an error" do
+      before { allow(storage_service).to receive(:write_file).and_raise(StandardError.new("This is an exception")) }
+
       it "does not save a record in the database" do
-        allow(storage_service).to receive(:write_file).and_raise(StandardError.new("This is an exception"))
+        expect { export_service.export_case_logs }
-        export = LogsExport.new
+          .to raise_error(StandardError)
-        allow(LogsExport).to receive(:new).and_return(export)
+                .and(change(LogsExport, :count).by(0))
-        expect(export).not_to receive(:save!)
-        expect { export_service.export_case_logs }.to raise_error(StandardError)
       end
     end
   end