CLDC-2689 Export migrated logs (#1873)

* Add imported_at columns

* Set imported at date for lettings logs

* Set imported at for sales logs

* Export logs migrated since the previous partial export

* Rename column

* rebase changes

* Re-add test

app/services/exports/lettings_log_export_service.rb

@@ -119,7 +119,7 @@ module Exports
     def retrieve_lettings_logs(start_time, recent_export, full_update)
       if !full_update && recent_export
         params = { from: recent_export.started_at, to: start_time }
-        LettingsLog.exportable.where("updated_at >= :from and updated_at <= :to", params)
+        LettingsLog.exportable.where("(updated_at >= :from AND updated_at <= :to) OR (values_updated_at IS NOT NULL AND values_updated_at >= :from AND values_updated_at <= :to)", params)
       else
         params = { to: start_time }
         LettingsLog.exportable.where("updated_at <= :to", params)

app/services/imports/lettings_logs_import_service.rb

@@ -271,6 +271,7 @@ module Imports
 
         attributes["created_by"] = user
       end
+      attributes["values_updated_at"] = Time.zone.now
 
       apply_date_consistency!(attributes)
       apply_household_consistency!(attributes)

app/services/imports/sales_logs_import_service.rb

@@ -198,6 +198,7 @@ module Imports
         end
         attributes["created_by"] = user
       end
+      attributes["values_updated_at"] = Time.zone.now
 
       set_default_values(attributes) if previous_status.include?("submitted")
       sales_log = save_sales_log(attributes, previous_status)

db/migrate/20230828145454_add_migrated_on_fields.rb

@@ -0,0 +1,6 @@
+class AddMigratedOnFields < ActiveRecord::Migration[7.0]
+  def change
+    add_column :lettings_logs, :values_updated_at, :datetime
+    add_column :sales_logs, :values_updated_at, :datetime
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2023_08_16_083950) do
+ActiveRecord::Schema[7.0].define(version: 2023_08_28_145454) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
@@ -297,6 +297,7 @@ ActiveRecord::Schema[7.0].define(version: 2023_08_16_083950) do
     t.integer "status_cache", default: 0, null: false
     t.datetime "discarded_at"
     t.integer "creation_method", default: 1
+    t.datetime "values_updated_at"
     t.index ["bulk_upload_id"], name: "index_lettings_logs_on_bulk_upload_id"
     t.index ["created_by_id"], name: "index_lettings_logs_on_created_by_id"
     t.index ["location_id"], name: "index_lettings_logs_on_location_id"
@@ -617,6 +618,7 @@ ActiveRecord::Schema[7.0].define(version: 2023_08_16_083950) do
     t.integer "stairowned_value_check"
     t.integer "creation_method", default: 1
     t.integer "old_form_id"
+    t.datetime "values_updated_at"
     t.index ["bulk_upload_id"], name: "index_sales_logs_on_bulk_upload_id"
     t.index ["created_by_id"], name: "index_sales_logs_on_created_by_id"
     t.index ["old_id"], name: "index_sales_logs_on_old_id", unique: true

spec/services/exports/lettings_log_export_service_spec.rb

@@ -328,6 +328,26 @@ RSpec.describe Exports::LettingsLogExportService do
         expect(LogsExport.last.increment_number).to eq(1)
       end
     end
+
+    context "and a log has been migrated since the previous partial export" do
+      before do
+        FactoryBot.create(:lettings_log, startdate: Time.zone.local(2022, 2, 1), updated_at: Time.zone.local(2022, 4, 27), values_updated_at: Time.zone.local(2022, 4, 29))
+        FactoryBot.create(:lettings_log, startdate: Time.zone.local(2022, 2, 1), updated_at: Time.zone.local(2022, 4, 27), values_updated_at: Time.zone.local(2022, 4, 29))
+        LogsExport.create!(started_at: Time.zone.local(2022, 4, 28), base_number: 1, increment_number: 1)
+      end
+
+      it "generates an XML manifest file with the expected content within the ZIP file" do
+        expected_content = replace_record_number(local_manifest_file.read, 2)
+        expect(storage_service).to receive(:write_file).with(expected_master_manifest_rerun, any_args)
+        expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) do |_, content|
+          entry = Zip::File.open_buffer(content).find_entry(expected_manifest_filename)
+          expect(entry).not_to be_nil
+          expect(entry.get_input_stream.read).to eq(expected_content)
+        end
+
+        export_service.export_xml_lettings_logs
+      end
+    end
   end
 
   context "when exporting a supported housing lettings logs in XML" do

spec/services/imports/lettings_logs_import_service_spec.rb

@@ -159,6 +159,13 @@ RSpec.describe Imports::LettingsLogsImportService do
         end
       end
 
+      it "correctly sets imported at date" do
+        lettings_log_service.send(:create_log, lettings_log_xml)
+
+        lettings_log = LettingsLog.where(old_id: lettings_log_id).first
+        expect(lettings_log&.values_updated_at).to eq(Time.zone.local(2022, 1, 1))
+      end
+
       context "and the void date is after the start date" do
         before { lettings_log_xml.at_xpath("//xmlns:VYEAR").content = 2023 }
 

spec/services/imports/sales_logs_import_service_spec.rb

@@ -102,9 +102,17 @@ RSpec.describe Imports::SalesLogsImportService do
   end
 
   context "when importing a specific log" do
+    let(:sales_log_id) { "shared_ownership_sales_log" }
     let(:sales_log_file) { open_file(fixture_directory, sales_log_id) }
     let(:sales_log_xml) { Nokogiri::XML(sales_log_file) }
 
+    it "correctly sets values updated at date" do
+      sales_log_service.send(:create_log, sales_log_xml)
+
+      sales_log = SalesLog.where(old_id: sales_log_id).first
+      expect(sales_log&.values_updated_at).to eq(Time.zone.local(2023, 2, 1))
+    end
+
     context "and the organisation legacy ID does not exist" do
       let(:sales_log_id) { "shared_ownership_sales_log" }