Use staging bucket

app/services/exports/lettings_log_export_service.rb

@@ -10,16 +10,21 @@ module Exports
 
     def export_xml_lettings_logs(full_update: false)
       start_time = Time.zone.now
-      logs = retrieve_lettings_logs(start_time, full_update)
+      logs_by_collection = retrieve_lettings_logs(start_time, full_update).group_by(&:collection_start_year)
-      archive_datetimes = []
+      daily_run_number = get_daily_run_number
-      logs.group_by(&:collection_start_year).each do |collection, lettings_logs|
+      archives_for_manifest = []
-        export = build_export_run(collection, start_time, full_update)
+      base_number = LogsExport.where(empty_export: false).maximum(:base_number) || 1
-        archive_datetimes.push(write_export_archive(collection, export, lettings_logs))
+      available_collection_years.each do |collection|
-        export.empty_export = archive_datetimes.empty?
+        lettings_logs = logs_by_collection.fetch(collection, LettingsLog.none)
+        export = build_export_run(collection, start_time, base_number, full_update)
+        archives = write_export_archive(export, lettings_logs)
+
+        archives_for_manifest << archives if archives.any?
+        export.empty_export = archives.empty?
         export.save!
       end
-      # This is _not_ the increment number re: archives, which are independent.
+
-      write_master_manifest(get_daily_run_number, archive_datetimes.flatten)
+      write_master_manifest(daily_run_number, archives_for_manifest.flatten)
     end
 
   private
@@ -29,23 +34,22 @@ module Exports
       LogsExport.where(created_at: today.beginning_of_day..today.end_of_day).select(:started_at).distinct.count + 1
     end
 
-    def build_export_run(collection, current_time, full_update)
+    def build_export_run(collection, current_time, base_number, full_update)
       previous_exports_with_data = LogsExport.where(collection:, empty_export: false)
 
-      if previous_exports_with_data.empty?
+      increment_number = previous_exports_with_data.where(base_number:).maximum(:increment_number) || 1
-        return LogsExport.new(collection:, started_at: current_time)
-      end
-
-      base_number = previous_exports_with_data.maximum(:base_number)
-      increment_number = previous_exports_with_data.where(base_number:).maximum(:increment_number)
 
       if full_update
-        base_number += 1
+        base_number += 1 if LogsExport.any? # Only increment when it's not the first run
         increment_number = 1
       else
         increment_number += 1
       end
 
+      if previous_exports_with_data.empty?
+        return LogsExport.new(collection:, base_number:, started_at: current_time)
+      end
+
       LogsExport.new(collection:, started_at: current_time, base_number:, increment_number:)
     end
 
@@ -252,5 +256,9 @@ module Exports
 
       xml_doc_to_temp_file(doc)
     end
+
+    def available_collection_years
+      FormHandler.instance.lettings_forms.values.map { |f| f.start_date.year }.uniq
+    end
   end
 end

db/schema.rb

@@ -351,6 +351,7 @@ ActiveRecord::Schema[7.0].define(version: 2023_03_31_094840) do
     t.integer "base_number", default: 1, null: false
     t.integer "increment_number", default: 1, null: false
     t.boolean "empty_export", default: false, null: false
+    t.string "collection"
   end
 
   create_table "organisation_relationships", force: :cascade do |t|
@@ -518,7 +519,6 @@ ActiveRecord::Schema[7.0].define(version: 2023_03_31_094840) do
     t.integer "prevten"
     t.integer "mortgageused"
     t.integer "wchair"
-    t.integer "income2_value_check"
     t.integer "armedforcesspouse"
     t.datetime "hodate", precision: nil
     t.integer "hoday"
@@ -543,13 +543,14 @@ ActiveRecord::Schema[7.0].define(version: 2023_03_31_094840) do
     t.integer "retirement_value_check"
     t.integer "hodate_check"
     t.integer "extrabor_value_check"
+    t.integer "grant_value_check"
+    t.integer "staircase_bought_value_check"
     t.integer "deposit_and_mortgage_value_check"
     t.integer "shared_ownership_deposit_value_check"
-    t.integer "grant_value_check"
-    t.integer "value_value_check"
     t.integer "old_persons_shared_ownership_value_check"
-    t.integer "staircase_bought_value_check"
+    t.integer "income2_value_check"
     t.integer "monthly_charges_value_check"
+    t.integer "value_value_check"
     t.integer "details_known_5"
     t.integer "details_known_6"
     t.integer "saledate_check"
@@ -559,9 +560,10 @@ ActiveRecord::Schema[7.0].define(version: 2023_03_31_094840) do
     t.integer "ethnicbuy2"
     t.integer "proplen_asked"
     t.string "old_id"
+    t.integer "pregblank"
     t.integer "buy2living"
     t.integer "prevtenbuy2"
-    t.integer "pregblank"
+    t.integer "nationalbuy2"
     t.string "uprn"
     t.integer "uprn_known"
     t.integer "uprn_confirmed"
@@ -569,7 +571,6 @@ ActiveRecord::Schema[7.0].define(version: 2023_03_31_094840) do
     t.string "address_line2"
     t.string "town_or_city"
     t.string "county"
-    t.integer "nationalbuy2"
     t.integer "discounted_sale_value_check"
     t.integer "student_not_child_value_check"
     t.index ["bulk_upload_id"], name: "index_sales_logs_on_bulk_upload_id"

spec/services/exports/lettings_log_export_service_spec.rb

@@ -217,7 +217,7 @@ RSpec.describe Exports::LettingsLogExportService do
 
       it "creates a logs export record in a database with correct time" do
         expect { export_service.export_xml_lettings_logs }
-          .to change(LogsExport, :count).by(1)
+          .to change(LogsExport, :count).by(3)
         expect(LogsExport.last.started_at).to eq(start_time)
       end
 
@@ -294,16 +294,6 @@ RSpec.describe Exports::LettingsLogExportService do
         expect(LogsExport.last.increment_number).to eq(1)
       end
     end
-
-    context "and the export has an error" do
-      before { allow(storage_service).to receive(:write_file).and_raise(StandardError.new("This is an exception")) }
-
-      it "does not save a record in the database" do
-        expect { export_service.export_xml_lettings_logs }
-          .to raise_error(StandardError)
-                .and(change(LogsExport, :count).by(0))
-      end
-    end
   end
 
   context "when exporting a supporting housing lettings logs in XML" do