Merge branch 'main' into full-import-optimisation

# Conflicts:
#	lib/tasks/full_import.rake
#	spec/services/imports/organisation_import_service_spec.rb

app/helpers/platform_helper.rb

@@ -0,0 +1,5 @@
+module PlatformHelper
+  def self.is_paas?
+    !ENV["VCAP_SERVICES"].nil?
+  end
+end

app/jobs/data_export_xml_job.rb

@@ -2,7 +2,7 @@ class DataExportXmlJob < ApplicationJob
   queue_as :default
 
   def perform(full_update: false)
-    storage_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["EXPORT_PAAS_INSTANCE"])
+    storage_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["EXPORT_PAAS_INSTANCE"])
     export_service = Exports::LettingsLogExportService.new(storage_service)
 
     export_service.export_xml_lettings_logs(full_update:)

app/models/forms/bulk_upload_lettings/upload_your_file.rb

@@ -57,7 +57,7 @@ module Forms
       def storage_service
         @storage_service ||= if upload_enabled?
                                Storage::S3Service.new(
-                                 Configuration::PaasConfigurationService.new,
+                                 PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new,
                                  ENV["CSV_DOWNLOAD_PAAS_INSTANCE"],
                                )
                              else

app/models/forms/bulk_upload_sales/upload_your_file.rb

@@ -50,7 +50,7 @@ module Forms
       def storage_service
         @storage_service ||= if FeatureToggle.upload_enabled?
                                Storage::S3Service.new(
-                                 Configuration::PaasConfigurationService.new,
+                                 PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new,
                                  ENV["CSV_DOWNLOAD_PAAS_INSTANCE"],
                                )
                              else

app/models/scheme.rb

@@ -34,7 +34,6 @@ class Scheme < ApplicationRecord
     if scopes.any?
       filtered_records = filtered_records
       .left_outer_joins(:scheme_deactivation_periods)
-      .order("scheme_deactivation_periods.created_at DESC")
       .merge(scopes.reduce(&:or))
     end
 
@@ -44,9 +43,9 @@ class Scheme < ApplicationRecord
   scope :incomplete, lambda {
     where.not(confirmed: true)
     .or(where.not(id: Location.select(:scheme_id).where(confirmed: true).distinct))
-    .where.not(id: joins(:scheme_deactivation_periods).reactivating_soon.pluck(:id))
-    .where.not(id: joins(:scheme_deactivation_periods).deactivated.pluck(:id))
-    .where.not(id: joins(:scheme_deactivation_periods).deactivating_soon.pluck(:id))
+    .where.not(id: joins(:scheme_deactivation_periods).reactivating_soon.pluck(:id, :service_name, :confirmed))
+    .where.not(id: joins(:scheme_deactivation_periods).deactivated.pluck(:id, :service_name, :confirmed))
+    .where.not(id: joins(:scheme_deactivation_periods).deactivating_soon.pluck(:id, :service_name, :confirmed))
   }
 
   scope :deactivated, lambda {
@@ -65,10 +64,10 @@ class Scheme < ApplicationRecord
   }
 
   scope :active_status, lambda {
-    where.not(id: joins(:scheme_deactivation_periods).reactivating_soon.pluck(:id))
-    .where.not(id: joins(:scheme_deactivation_periods).deactivated.pluck(:id))
-    .where.not(id: incomplete.pluck(:id))
-    .where.not(id: joins(:scheme_deactivation_periods).deactivating_soon.pluck(:id))
+    where.not(id: joins(:scheme_deactivation_periods).reactivating_soon.pluck(:id, :service_name, :confirmed))
+    .where.not(id: joins(:scheme_deactivation_periods).deactivated.pluck(:id, :service_name, :confirmed))
+    .where.not(id: incomplete.pluck(:id, :service_name, :confirmed))
+    .where.not(id: joins(:scheme_deactivation_periods).deactivating_soon.pluck(:id, :service_name, :confirmed))
   }
 
   validate :validate_confirmed

app/models/validations/sales/setup_validations.rb

@@ -24,6 +24,40 @@ module Validations::Sales::SetupValidations
     end
   end
 
+  def validate_merged_organisations_saledate(record)
+    return unless record.saledate && date_valid?("saledate", record)
+
+    if merged_owning_organisation_inactive?(record)
+      record.errors.add :saledate, I18n.t("validations.setup.saledate.invalid_merged_organisations_saledate",
+                                          owning_organisation: record.owning_organisation.name,
+                                          owning_organisation_merge_date: record.owning_organisation.merge_date.to_formatted_s(:govuk_date),
+                                          owning_absorbing_organisation: record.owning_organisation.absorbing_organisation.name)
+    end
+
+    if absorbing_owning_organisation_inactive?(record)
+      record.errors.add :saledate, I18n.t("validations.setup.saledate.invalid_absorbing_organisations_saledate",
+                                          owning_organisation: record.owning_organisation.name,
+                                          owning_organisation_available_from: record.owning_organisation.created_at.to_formatted_s(:govuk_date))
+    end
+  end
+
+  def validate_organisation(record)
+    return unless record.saledate && record.owning_organisation
+
+    if record.owning_organisation.present?
+      if record.owning_organisation&.merge_date.present? && record.owning_organisation.merge_date <= record.saledate
+        record.errors.add :owning_organisation_id, I18n.t("validations.setup.owning_organisation.inactive_merged_organisation_sales",
+                                                          owning_organisation: record.owning_organisation.name,
+                                                          owning_organisation_merge_date: record.owning_organisation.merge_date.to_formatted_s(:govuk_date),
+                                                          owning_absorbing_organisation: record.owning_organisation.absorbing_organisation.name)
+      elsif record.owning_organisation&.absorbed_organisations.present? && record.owning_organisation.created_at.to_date > record.saledate.to_date
+        record.errors.add :owning_organisation_id, I18n.t("validations.setup.owning_organisation.inactive_absorbing_organisation_sales",
+                                                          owning_organisation: record.owning_organisation.name,
+                                                          owning_organisation_available_from: record.owning_organisation.created_at.to_formatted_s(:govuk_date))
+      end
+    end
+  end
+
 private
 
   def active_collection_start_date
@@ -64,4 +98,12 @@ private
       )
     end
   end
+
+  def merged_owning_organisation_inactive?(record)
+    record.owning_organisation&.merge_date.present? && record.owning_organisation.merge_date <= record.saledate
+  end
+
+  def absorbing_owning_organisation_inactive?(record)
+    record.owning_organisation&.absorbed_organisations.present? && record.owning_organisation.created_at.to_date > record.saledate.to_date
+  end
 end

app/services/bulk_upload/downloader.rb

@@ -38,7 +38,7 @@ private
 
   def s3_storage_service
     Storage::S3Service.new(
-      Configuration::PaasConfigurationService.new,
+      PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new,
       ENV["CSV_DOWNLOAD_PAAS_INSTANCE"],
     )
   end

config/initializers/multi_logger.rb

@@ -0,0 +1,17 @@
+class MultiLogger
+  def initialize(*targets)
+    @targets = targets
+  end
+
+  def info(data)
+    @targets.each { |t| t.info(data) }
+  end
+
+  def warn(data)
+    @targets.each { |t| t.warn(data) }
+  end
+
+  def error(data)
+    @targets.each { |t| t.error(data) }
+  end
+end

config/initializers/rack_attack.rb

@@ -1,14 +1,18 @@
 require "configuration/configuration_service"
 require "configuration/paas_configuration_service"
+require "configuration/env_configuration_service"
+require Rails.root.join("app/helpers/platform_helper")
+
+configuration_service = PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new
 
 if Rails.env.development? || Rails.env.test?
   Rack::Attack.cache.store = ActiveSupport::Cache::MemoryStore.new
   Rack::Attack.enabled = false
 elsif Rails.env.review?
-  redis_url = Configuration::PaasConfigurationService.new.redis_uris.to_a[0][1]
+  redis_url = configuration_service.redis_uris.to_a[0][1]
   Rack::Attack.cache.store = ActiveSupport::Cache::RedisCacheStore.new(url: redis_url)
 else
-  redis_url = Configuration::PaasConfigurationService.new.redis_uris[:"dluhc-core-#{Rails.env}-redis"]
+  redis_url = configuration_service.redis_uris[:"dluhc-core-#{Rails.env}-redis"]
   Rack::Attack.cache.store = ActiveSupport::Cache::RedisCacheStore.new(url: redis_url)
 end
 

config/initializers/sidekiq.rb

@@ -1,8 +1,10 @@
 require "sidekiq/web"
 require "sidekiq/cron/web"
 
+configuration_service = PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new
+
 if Rails.env.staging? || Rails.env.production?
-  redis_url = Configuration::PaasConfigurationService.new.redis_uris[:"dluhc-core-#{Rails.env}-redis"]
+  redis_url = configuration_service.redis_uris[:"dluhc-core-#{Rails.env}-redis"]
 
   Sidekiq.configure_server do |config|
     config.redis = { url: redis_url }
@@ -14,7 +16,7 @@ if Rails.env.staging? || Rails.env.production?
 end
 
 if Rails.env.review?
-  redis_url = Configuration::PaasConfigurationService.new.redis_uris.to_a[0][1]
+  redis_url = configuration_service.redis_uris.to_a[0][1]
 
   Sidekiq.configure_server do |config|
     config.redis = { url: redis_url }

config/locales/en.yml

@@ -239,6 +239,8 @@ en:
         previous_and_current_collection_year:
           "Enter a date within the %{previous_start_year_short}/%{previous_end_year_short} or %{previous_end_year_short}/%{current_end_year_short} collection years, which is between %{previous_start_year_long} and %{current_end_year_long}"
         year_not_two_digits: "Sale completion year must be 2 digits"
+        invalid_merged_organisations_saledate: "Enter a date when the owning organisation was active. %{owning_organisation} became inactive on %{owning_organisation_merge_date} and was replaced by %{owning_absorbing_organisation}."
+        invalid_absorbing_organisations_saledate:  "Enter a date when the owning organisation was active. %{owning_organisation} became active on %{owning_organisation_available_from}."
       type:
         percentage_bought_must_be_at_least_threshold: "The minimum increase in equity while staircasing is %{threshold}% for this shared ownership type"
 
@@ -288,6 +290,8 @@ en:
         data_sharing_agreement_not_signed: "The organisation must accept the Data Sharing Agreement before it can be selected as the owning organisation."
         inactive_merged_organisation: "The owning organisation must be active on the tenancy start date. %{owning_organisation} became inactive on %{owning_organisation_merge_date} and was replaced by %{owning_absorbing_organisation}."
         inactive_absorbing_organisation: "The owning organisation must be active on the tenancy start date. %{owning_organisation} became active on %{owning_organisation_available_from}."
+        inactive_merged_organisation_sales: "The owning organisation must be active on the sale completion date. %{owning_organisation} became inactive on %{owning_organisation_merge_date} and was replaced by %{owning_absorbing_organisation}."
+        inactive_absorbing_organisation_sales: "The owning organisation must be active on the sale completion date. %{owning_organisation} became active on %{owning_organisation_available_from}."
       managing_organisation:
         invalid: "Please select the owning organisation or managing organisation that you belong to"
         data_sharing_agreement_not_signed: "The organisation must accept the Data Sharing Agreement before it can be selected as the managing organisation."

lib/tasks/correct_incref_values.rake

@@ -1,6 +1,6 @@
 desc "Alter incref values for non imported lettings logs in the database"
 task correct_incref_values: :environment do
-  LettingsLog.where(old_id: nil, net_income_known: 0).update!(incref: 0)
-  LettingsLog.where(old_id: nil, net_income_known: 1).update!(incref: 2)
-  LettingsLog.where(old_id: nil, net_income_known: 2).update!(incref: 1)
+  LettingsLog.where(old_id: nil, net_income_known: 0).update_all(incref: 0)
+  LettingsLog.where(old_id: nil, net_income_known: 1).update_all(incref: 2)
+  LettingsLog.where(old_id: nil, net_income_known: 2).update_all(incref: 1)
 end

lib/tasks/data_import.rake

@@ -5,7 +5,7 @@ namespace :core do
     path = args[:path]
     raise "Usage: rake core:data_import['data_type', 'path/to/xml_files']" if path.blank? || type.blank?
 
-    storage_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
+    storage_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
 
     case type
     when "organisation"

lib/tasks/data_import_field.rake

@@ -8,7 +8,7 @@ namespace :core do
     # We only allow a reduced list of known fields to be updatable
     case field
     when "tenancycode", "major_repairs", "lettings_allocation", "offered"
-      s3_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
+      s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
       archive_io = s3_service.get_file_io(path)
       archive_service = Storage::ArchiveService.new(archive_io)
       if archive_service.folder_present?("logs")

lib/tasks/full_import.rake

@@ -1,4 +1,4 @@
-Import = Struct.new("Import", :import_class, :import_method, :folder)
+Import = Struct.new("Import", :import_class, :import_method, :folder, :logger)
 
 namespace :import do
   desc "Run initial import steps - orgs, schemes, users etc (without triggering user invite emails)"
@@ -6,37 +6,43 @@ namespace :import do
     institutions_csv_name = args[:institutions_csv_name]
     raise "Usage: rake import:initial['institutions_csv_name']" if institutions_csv_name.blank?
 
-    s3_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
+    s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
     csv = CSV.parse(s3_service.get_file_io(institutions_csv_name), headers: true)
+    logs_string = StringIO.new
+    logger = MultiLogger.new(Rails.logger, Logger.new(logs_string))
     org_count = csv.length
 
     initial_import_list = [
-      Import.new(Imports::OrganisationImportService, :create_organisations, "institution"),
-      Import.new(Imports::OrganisationRelationshipImportService, :create_organisation_relationships, "institution-link"),
-      Import.new(Imports::SchemeImportService, :create_schemes, "mgmtgroups"),
-      Import.new(Imports::SchemeLocationImportService, :create_scheme_locations, "schemes"),
-      Import.new(Imports::UserImportService, :create_users, "user"),
-      Import.new(Imports::DataProtectionConfirmationImportService, :create_data_protection_confirmations, "dataprotect"),
-      Import.new(Imports::OrganisationRentPeriodImportService, :create_organisation_rent_periods, "rent-period"),
+      Import.new(Imports::OrganisationImportService, :create_organisations, "institution", logger),
+      Import.new(Imports::OrganisationRelationshipImportService, :create_organisation_relationships, "institution-link", logger),
+      Import.new(Imports::SchemeImportService, :create_schemes, "mgmtgroups", logger),
+      Import.new(Imports::SchemeLocationImportService, :create_scheme_locations, "schemes", logger),
+      Import.new(Imports::UserImportService, :create_users, "user", logger),
+      Import.new(Imports::DataProtectionConfirmationImportService, :create_data_protection_confirmations, "dataprotect", logger),
+      Import.new(Imports::OrganisationRentPeriodImportService, :create_organisation_rent_periods, "rent-period", logger),
     ]
 
-    Rails.logger.info("Beginning initial imports for #{org_count} organisations")
+    logger.info("Beginning initial imports for #{org_count} organisations")
 
     csv.each do |row|
       archive_path = row[1]
       archive_io = s3_service.get_file_io(archive_path)
       archive_service = Storage::ArchiveService.new(archive_io)
-
-      Rails.logger.info("Performing initial imports for organisation #{row[0]}")
+      logger.info("Performing initial imports for organisation #{row[0]}")
 
       initial_import_list.each do |step|
         if archive_service.folder_present?(step.folder)
-          step.import_class.new(archive_service).send(step.import_method, step.folder)
+          step.import_class.new(archive_service).send(step.import_method, step.folder, step.logger)
         end
       end
+
+      log_file = "#{File.basename(institutions_csv_name, File.extname(institutions_csv_name))}_#{File.basename(archive_path, File.extname(archive_path))}_initial.log"
+      s3_service.write_file(log_file, logs_string.string)
+      logs_string.rewind
+      logs_string.truncate(0)
     end
 
-    Rails.logger.info("Finished initial imports")
+    logger.info("Finished initial imports")
   end
 
   desc "Run logs import steps"
@@ -44,16 +50,18 @@ namespace :import do
     institutions_csv_name = args[:institutions_csv_name]
     raise "Usage: rake import:logs['institutions_csv_name']" if institutions_csv_name.blank?
 
-    s3_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
+    s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
     csv = CSV.parse(s3_service.get_file_io(institutions_csv_name), headers: true)
     org_count = csv.length
+    logs_string = StringIO.new
+    logger = MultiLogger.new(Rails.logger, Logger.new(logs_string))
 
     logs_import_list = [
-      Import.new(Imports::LettingsLogsImportService, :create_logs, "logs"),
-      Import.new(Imports::SalesLogsImportService, :create_logs, "logs"),
+      Import.new(Imports::LettingsLogsImportService, :create_logs, "logs", logger),
+      Import.new(Imports::SalesLogsImportService, :create_logs, "logs", logger),
     ]
 
-    Rails.logger.info("Beginning log imports for #{org_count} organisations")
+    logger.info("Beginning log imports for #{org_count} organisations")
 
     csv.each do |row|
       archive_path = row[1]
@@ -61,16 +69,21 @@ namespace :import do
       archive_service = Storage::ArchiveService.new(archive_io)
 
       log_count = row[2].to_i + row[3].to_i + row[4].to_i + row[5].to_i
-      Rails.logger.info("Importing logs for organisation #{row[0]}, expecting #{log_count} logs")
+      logger.info("Importing logs for organisation #{row[0]}, expecting #{log_count} logs")
 
       logs_import_list.each do |step|
         if archive_service.folder_present?(step.folder)
-          step.import_class.new(archive_service).send(step.import_method, step.folder)
+          step.import_class.new(archive_service).send(step.import_method, step.folder, step.logger)
         end
       end
+
+      log_file = "#{File.basename(institutions_csv_name, File.extname(institutions_csv_name))}_#{File.basename(archive_path, File.extname(archive_path))}_logs.log"
+      s3_service.write_file(log_file, logs_string.string)
+      logs_string.rewind
+      logs_string.truncate(0)
     end
 
-    Rails.logger.info("Log import complete")
+    logger.info("Log import complete")
   end
 
   desc "Trigger invite emails for imported users"
@@ -78,7 +91,7 @@ namespace :import do
     institutions_csv_name = args[:institutions_csv_name]
     raise "Usage: rake import:trigger_invites['institutions_csv_name']" if institutions_csv_name.blank?
 
-    s3_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
+    s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
     csv = CSV.parse(s3_service.get_file_io(institutions_csv_name), headers: true)
 
     Rails.logger.info("Triggering user invite emails")
@@ -99,7 +112,7 @@ namespace :import do
     institutions_csv_name = args[:institutions_csv_name]
     raise "Usage: rake import:generate_reports['institutions_csv_name']" if institutions_csv_name.blank?
 
-    s3_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
+    s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
     institutions_csv = CSV.parse(s3_service.get_file_io(institutions_csv_name), headers: true)
 
     Imports::ImportReportService.new(s3_service, institutions_csv).create_reports(institutions_csv_name)

spec/helpers/platform_helper_spec.rb

@@ -0,0 +1,15 @@
+require "rails_helper"
+
+RSpec.describe PlatformHelper do
+  describe "is_paas?" do
+    it "returns true if the VCAP_SERVICES environment variable exists" do
+      allow(ENV).to receive(:[]).with("VCAP_SERVICES").and_return("dummy")
+      expect(described_class.is_paas?).to eq(true)
+    end
+
+    it "returns false if the VCAP_SERVICES environment variable doesn't exist" do
+      allow(ENV).to receive(:[]).with("VCAP_SERVICES")
+      expect(described_class.is_paas?).to eq(false)
+    end
+  end
+end

spec/lib/tasks/correct_incref_values_spec.rb

@@ -31,6 +31,14 @@ RSpec.describe "correct_incref_values" do
         task.invoke
         expect(lettings_log.reload.incref).to eq(1)
       end
+
+      it "skips validations for previous years" do
+        lettings_log.update!(net_income_known: 2, incref: nil)
+        lettings_log.startdate = Time.zone.local(2021, 3, 3)
+        lettings_log.save!(validate: false)
+        task.invoke
+        expect(lettings_log.reload.incref).to eq(1)
+      end
     end
   end
 end

spec/lib/tasks/data_import_spec.rb

@@ -3,6 +3,7 @@ require "rake"
 
 describe "data import", type: :task do
   let(:instance_name) { "paas_import_instance" }
+  let(:env_config_service) { instance_double(Configuration::EnvConfigurationService) }
   let(:paas_config_service) { instance_double(Configuration::PaasConfigurationService) }
   let(:storage_service) { instance_double(Storage::S3Service) }
 
@@ -16,8 +17,10 @@ describe "data import", type: :task do
 
       allow(Storage::S3Service).to receive(:new).and_return(storage_service)
       allow(Configuration::PaasConfigurationService).to receive(:new).and_return(paas_config_service)
+      allow(Configuration::EnvConfigurationService).to receive(:new).and_return(env_config_service)
       allow(ENV).to receive(:[])
       allow(ENV).to receive(:[]).with("IMPORT_PAAS_INSTANCE").and_return(instance_name)
+      allow(ENV).to receive(:[]).with("VCAP_SERVICES").and_return("dummy")
     end
 
     context "when importing organisation data" do
@@ -29,13 +32,22 @@ describe "data import", type: :task do
         allow(Imports::OrganisationImportService).to receive(:new).and_return(import_service)
       end
 
-      it "creates an organisation from the given XML file" do
+      it "creates an organisation from the given XML file when the VCAP_SERVICES environment variable exists" do
         expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name)
         expect(Imports::OrganisationImportService).to receive(:new).with(storage_service)
         expect(import_service).to receive(:create_organisations).with(fixture_path)
 
         task.invoke(type, fixture_path)
       end
+
+      it "creates an organisation from the given XML file when the VCAP_SERVICES environment variable does not exist" do
+        allow(ENV).to receive(:[]).with("VCAP_SERVICES")
+        expect(Storage::S3Service).to receive(:new).with(env_config_service, instance_name)
+        expect(Imports::OrganisationImportService).to receive(:new).with(storage_service)
+        expect(import_service).to receive(:create_organisations).with(fixture_path)
+
+        task.invoke(type, fixture_path)
+      end
     end
 
     context "when importing user data" do

spec/lib/tasks/date_import_field_spec.rb

@@ -6,6 +6,7 @@ describe "rake core:data_import_field", type: :task do
 
   let(:instance_name) { "paas_import_instance" }
   let(:storage_service) { instance_double(Storage::S3Service) }
+  let(:env_config_service) { instance_double(Configuration::EnvConfigurationService) }
   let(:paas_config_service) { instance_double(Configuration::PaasConfigurationService) }
 
   before do
@@ -14,9 +15,11 @@ describe "rake core:data_import_field", type: :task do
     task.reenable
 
     allow(Storage::S3Service).to receive(:new).and_return(storage_service)
+    allow(Configuration::EnvConfigurationService).to receive(:new).and_return(env_config_service)
     allow(Configuration::PaasConfigurationService).to receive(:new).and_return(paas_config_service)
     allow(ENV).to receive(:[])
     allow(ENV).to receive(:[]).with("IMPORT_PAAS_INSTANCE").and_return(instance_name)
+    allow(ENV).to receive(:[]).with("VCAP_SERVICES").and_return("dummy")
     allow(Imports::LettingsLogsFieldImportService).to receive(:new).and_return(import_service)
   end
 
@@ -34,13 +37,22 @@ describe "rake core:data_import_field", type: :task do
     context "and we update the tenancycode field" do
       let(:field) { "tenancycode" }
 
-      it "updates the logs from the given XML file" do
+      it "updates the logs from the given XML file when the VCAP_SERVICES environment variable exists" do
         expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name)
         expect(storage_service).to receive(:get_file_io).with("spec/fixtures/imports/logs")
         expect(Imports::LettingsLogsFieldImportService).to receive(:new).with(archive_service)
         expect(import_service).to receive(:update_field).with(field, "logs")
         task.invoke(field, fixture_path)
       end
+
+      it "updates the logs from the given XML file when the VCAP_SERVICES environment variable does not exist" do
+        allow(ENV).to receive(:[]).with("VCAP_SERVICES")
+        expect(Storage::S3Service).to receive(:new).with(env_config_service, instance_name)
+        expect(storage_service).to receive(:get_file_io).with("spec/fixtures/imports/logs")
+        expect(Imports::LettingsLogsFieldImportService).to receive(:new).with(archive_service)
+        expect(import_service).to receive(:update_field).with(field, "logs")
+        task.invoke(field, fixture_path)
+      end
     end
 
     context "and we update the lettings_allocation fields" do

spec/lib/tasks/full_import_spec.rb

@@ -4,6 +4,7 @@ require "rake"
 describe "full import", type: :task do
   let(:instance_name) { "paas_import_instance" }
   let(:paas_config_service) { instance_double(Configuration::PaasConfigurationService) }
+  let(:env_config_service) { instance_double(Configuration::EnvConfigurationService) }
   let(:storage_service) { instance_double(Storage::S3Service) }
   let(:orgs_list) { "Institution name,Id,Old Completed lettings logs,Old In progress lettings logs,Old Completed sales logs,Old In progress sales logs\norg1,1.zip,0,0,0,0\norg2,2.zip,0,0,0,0" }
 
@@ -12,6 +13,7 @@ describe "full import", type: :task do
     allow(storage_service).to receive(:write_file).and_return(nil)
     allow(storage_service).to receive(:get_file_io).and_return(orgs_list)
     allow(Configuration::PaasConfigurationService).to receive(:new).and_return(paas_config_service)
+    allow(Configuration::EnvConfigurationService).to receive(:new).and_return(env_config_service)
     allow(ENV).to receive(:[])
     allow(ENV).to receive(:[]).with("IMPORT_PAAS_INSTANCE").and_return(instance_name)
   end
@@ -32,13 +34,85 @@ describe "full import", type: :task do
         allow(Imports::ImportReportService).to receive(:new).and_return(import_report_service)
       end
 
-      it "creates a report using given organisation csv" do
+      it "creates a report using given organisation csv when the VCAP_SERVICES environment variable exists" do
+        allow(ENV).to receive(:[]).with("VCAP_SERVICES").and_return("dummy")
         expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name)
         expect(Imports::ImportReportService).to receive(:new).with(storage_service, CSV.parse(orgs_list, headers: true))
         expect(import_report_service).to receive(:create_reports).with("some_name")
 
         task.invoke("some_name")
       end
+
+      it "creates a report using given organisation csv when the VCAP_SERVICES environment variable does not exist" do
+        allow(ENV).to receive(:[]).with("VCAP_SERVICES")
+        expect(Storage::S3Service).to receive(:new).with(env_config_service, instance_name)
+        expect(Imports::ImportReportService).to receive(:new).with(storage_service, CSV.parse(orgs_list, headers: true))
+        expect(import_report_service).to receive(:create_reports).with("some_name")
+
+        task.invoke("some_name")
+      end
+    end
+  end
+
+  describe "import:initial" do
+    subject(:task) { Rake::Task["import:initial"] }
+
+    let(:archive_service) { instance_double(Storage::ArchiveService) }
+
+    before do
+      Rake.application.rake_require("tasks/full_import")
+      Rake::Task.define_task(:environment)
+      task.reenable
+    end
+
+    context "when calling the initial import" do
+      before do
+        allow(Storage::ArchiveService).to receive(:new).and_return(archive_service)
+        allow(archive_service).to receive(:folder_present?).and_return(false)
+        allow(Imports::OrganisationImportService).to receive(:new).and_return(instance_double(Imports::OrganisationImportService))
+        allow(Imports::SchemeImportService).to receive(:new).and_return(instance_double(Imports::SchemeImportService))
+        allow(Imports::SchemeLocationImportService).to receive(:new).and_return(instance_double(Imports::SchemeLocationImportService))
+        allow(Imports::UserImportService).to receive(:new).and_return(instance_double(Imports::UserImportService))
+        allow(Imports::DataProtectionConfirmationImportService).to receive(:new).and_return(instance_double(Imports::DataProtectionConfirmationImportService))
+        allow(Imports::OrganisationRentPeriodImportService).to receive(:new).and_return(instance_double(Imports::OrganisationRentPeriodImportService))
+      end
+
+      it "creates a report using given organisation csv" do
+        expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name)
+        expect(storage_service).to receive(:write_file).with("some_name_1_initial.log", / INFO -- : Performing initial imports for organisation org1/)
+        expect(storage_service).to receive(:write_file).with("some_name_2_initial.log", / INFO -- : Performing initial imports for organisation org2/)
+
+        task.invoke("some_name.csv")
+      end
+    end
+  end
+
+  describe "import:logs" do
+    subject(:task) { Rake::Task["import:logs"] }
+
+    let(:archive_service) { instance_double(Storage::ArchiveService) }
+
+    before do
+      Rake.application.rake_require("tasks/full_import")
+      Rake::Task.define_task(:environment)
+      task.reenable
+    end
+
+    context "when calling the logs import" do
+      before do
+        allow(Storage::ArchiveService).to receive(:new).and_return(archive_service)
+        allow(archive_service).to receive(:folder_present?).and_return(false)
+        allow(Imports::LettingsLogsImportService).to receive(:new).and_return(instance_double(Imports::LettingsLogsImportService))
+        allow(Imports::SalesLogsImportService).to receive(:new).and_return(instance_double(Imports::SalesLogsImportService))
+      end
+
+      it "creates a report using given organisation csv" do
+        expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name)
+        expect(storage_service).to receive(:write_file).with("some_name_1_logs.log", / INFO -- : Importing logs for organisation org1, expecting 0 logs/)
+        expect(storage_service).to receive(:write_file).with("some_name_2_logs.log", / INFO -- : Importing logs for organisation org2, expecting 0 logs/)
+
+        task.invoke("some_name.csv")
+      end
     end
   end
 end

spec/models/scheme_spec.rb

@@ -110,7 +110,7 @@ RSpec.describe Scheme, type: :model do
       end
 
       context "when filtering by status" do
-        let!(:incomplete_scheme) { FactoryBot.create(:scheme, :incomplete) }
+        let!(:incomplete_scheme) { FactoryBot.create(:scheme, :incomplete, service_name: "name") }
         let(:active_scheme) { FactoryBot.create(:scheme) }
         let(:deactivating_soon_scheme) { FactoryBot.create(:scheme) }
         let(:deactivated_scheme) { FactoryBot.create(:scheme) }
@@ -141,6 +141,13 @@ RSpec.describe Scheme, type: :model do
           end
         end
 
+        context "when filtering by incomplete status and searching" do
+          it "returns only incomplete schemes" do
+            expect(described_class.search_by("name").filter_by_status(%w[incomplete]).count).to eq(1)
+            expect(described_class.search_by("name").filter_by_status(%w[incomplete]).first).to eq(incomplete_scheme)
+          end
+        end
+
         context "when filtering by active status" do
           it "returns only active schemes" do
             expect(described_class.filter_by_status(%w[active]).count).to eq(1)

spec/models/validations/sales/setup_validations_spec.rb

@@ -185,4 +185,100 @@ RSpec.describe Validations::Sales::SetupValidations do
       end
     end
   end
+
+  describe "#validate_merged_organisations_saledate" do
+    let(:record) { build(:sales_log) }
+    let(:absorbing_organisation) { create(:organisation, created_at: Time.zone.local(2023, 2, 1), name: "Absorbing org") }
+    let(:merged_organisation) { create(:organisation, name: "Merged org") }
+
+    around do |example|
+      Timecop.freeze(Time.zone.local(2023, 5, 1))
+      example.run
+      Timecop.return
+    end
+
+    before do
+      merged_organisation.update!(absorbing_organisation:, merge_date: Time.zone.local(2023, 2, 2))
+    end
+
+    context "and owning organisation is no longer active" do
+      it "does not allow saledate after organisation has been merged" do
+        record.saledate = Time.zone.local(2023, 3, 1)
+        record.owning_organisation_id = merged_organisation.id
+        setup_validator.validate_merged_organisations_saledate(record)
+        expect(record.errors["saledate"]).to include(match "Enter a date when the owning organisation was active. Merged org became inactive on 2 February 2023 and was replaced by Absorbing org.")
+      end
+
+      it "allows saledate before organisation has been merged" do
+        record.saledate = Time.zone.local(2023, 1, 1)
+        record.owning_organisation_id = merged_organisation.id
+        setup_validator.validate_merged_organisations_saledate(record)
+        expect(record.errors["saledate"]).to be_empty
+      end
+    end
+
+    context "and owning organisation is not yet active during the saledate" do
+      it "does not allow saledate before absorbing organisation has been created" do
+        record.saledate = Time.zone.local(2023, 1, 1)
+        record.owning_organisation_id = absorbing_organisation.id
+        setup_validator.validate_merged_organisations_saledate(record)
+        expect(record.errors["saledate"]).to include(match "Enter a date when the owning organisation was active. Absorbing org became active on 1 February 2023.")
+      end
+
+      it "allows saledate after absorbing organisation has been created" do
+        record.saledate = Time.zone.local(2023, 2, 2)
+        record.owning_organisation_id = absorbing_organisation.id
+        setup_validator.validate_merged_organisations_saledate(record)
+        expect(record.errors["saledate"]).to be_empty
+      end
+    end
+  end
+
+  describe "#validate_organisation" do
+    let(:record) { build(:sales_log) }
+
+    context "when organisations are merged" do
+      let(:absorbing_organisation) { create(:organisation, created_at: Time.zone.local(2023, 2, 1), name: "Absorbing org") }
+      let(:merged_organisation) { create(:organisation, name: "Merged org") }
+
+      around do |example|
+        Timecop.freeze(Time.zone.local(2023, 5, 1))
+        merged_organisation.update!(merge_date: Time.zone.local(2023, 2, 2), absorbing_organisation:)
+        example.run
+        Timecop.return
+      end
+
+      context "and owning organisation is no longer active" do
+        it "does not allow organisation that has been merged" do
+          record.saledate = Time.zone.local(2023, 3, 1)
+          record.owning_organisation_id = merged_organisation.id
+          setup_validator.validate_organisation(record)
+          expect(record.errors["owning_organisation_id"]).to include(match "The owning organisation must be active on the sale completion date. Merged org became inactive on 2 February 2023 and was replaced by Absorbing org.")
+        end
+
+        it "allows organisation before it has been merged" do
+          record.saledate = Time.zone.local(2023, 1, 1)
+          record.owning_organisation_id = merged_organisation.id
+          setup_validator.validate_organisation(record)
+          expect(record.errors["owning_organisation_id"]).to be_empty
+        end
+      end
+
+      context "and owning organisation is not yet active during the saledate" do
+        it "does not allow absorbing organisation before it had been created" do
+          record.saledate = Time.zone.local(2023, 1, 1)
+          record.owning_organisation_id = absorbing_organisation.id
+          setup_validator.validate_organisation(record)
+          expect(record.errors["owning_organisation_id"]).to include(match "The owning organisation must be active on the sale completion date. Absorbing org became active on 1 February 2023.")
+        end
+
+        it "allows absorbing organisation after it has been created" do
+          record.saledate = Time.zone.local(2023, 2, 2)
+          record.owning_organisation_id = absorbing_organisation.id
+          setup_validator.validate_organisation(record)
+          expect(record.errors["owning_organisation_id"]).to be_empty
+        end
+      end
+    end
+  end
 end

spec/services/imports/organisation_import_service_spec.rb

@@ -2,7 +2,8 @@ require "rails_helper"
 
 RSpec.describe Imports::OrganisationImportService do
   let(:storage_service) { instance_double(Storage::S3Service) }
-  let(:logger) { instance_double(Rails::Rack::Logger) }
+  let(:logs_string) { StringIO.new }
+  let(:logger) { MultiLogger.new(Rails.logger, Logger.new(logs_string)) }
   let(:folder_name) { "organisations" }
   let(:filenames) { %w[my_folder/my_file1.xml my_folder/my_file2.xml] }
   let(:fixture_directory) { "spec/fixtures/imports/institution" }
@@ -82,9 +83,11 @@ RSpec.describe Imports::OrganisationImportService do
       expect(storage_service).to receive(:list_files).with(folder_name).twice
       expect(storage_service).to receive(:get_file_io).with(filenames[0]).twice
       expect(logger).not_to receive(:warn)
+      expect(Rails.logger).not_to receive(:warn)
 
       expect { import_service.create_organisations(folder_name) }.to change(Organisation, :count).by(1)
       expect { import_service.create_organisations(folder_name) }.to change(Organisation, :count).by(0)
+      expect(logs_string.string).to include("Organisation my_new_organisation is already present with old visible ID 1, skipping.")
 
       expect(Organisation).to exist(old_visible_id: "1", name: "HA Ltd")
     end