diff --git a/app/helpers/platform_helper.rb b/app/helpers/platform_helper.rb new file mode 100644 index 000000000..08297f53b --- /dev/null +++ b/app/helpers/platform_helper.rb @@ -0,0 +1,5 @@ +module PlatformHelper + def self.is_paas? + !ENV["VCAP_SERVICES"].nil? + end +end diff --git a/app/jobs/data_export_xml_job.rb b/app/jobs/data_export_xml_job.rb index b26b65364..b882308ed 100644 --- a/app/jobs/data_export_xml_job.rb +++ b/app/jobs/data_export_xml_job.rb @@ -2,7 +2,7 @@ class DataExportXmlJob < ApplicationJob queue_as :default def perform(full_update: false) - storage_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["EXPORT_PAAS_INSTANCE"]) + storage_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["EXPORT_PAAS_INSTANCE"]) export_service = Exports::LettingsLogExportService.new(storage_service) export_service.export_xml_lettings_logs(full_update:) diff --git a/app/models/forms/bulk_upload_lettings/upload_your_file.rb b/app/models/forms/bulk_upload_lettings/upload_your_file.rb index 57ac017a3..bb7e9fad6 100644 --- a/app/models/forms/bulk_upload_lettings/upload_your_file.rb +++ b/app/models/forms/bulk_upload_lettings/upload_your_file.rb @@ -57,7 +57,7 @@ module Forms def storage_service @storage_service ||= if upload_enabled? Storage::S3Service.new( - Configuration::PaasConfigurationService.new, + PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["CSV_DOWNLOAD_PAAS_INSTANCE"], ) else diff --git a/app/models/forms/bulk_upload_sales/upload_your_file.rb b/app/models/forms/bulk_upload_sales/upload_your_file.rb index 117e612b1..b08983d82 100644 --- a/app/models/forms/bulk_upload_sales/upload_your_file.rb +++ b/app/models/forms/bulk_upload_sales/upload_your_file.rb @@ -50,7 +50,7 @@ module Forms def storage_service @storage_service ||= if FeatureToggle.upload_enabled? Storage::S3Service.new( - Configuration::PaasConfigurationService.new, + PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["CSV_DOWNLOAD_PAAS_INSTANCE"], ) else diff --git a/app/models/scheme.rb b/app/models/scheme.rb index 4767b92cf..08d7dedd5 100644 --- a/app/models/scheme.rb +++ b/app/models/scheme.rb @@ -34,7 +34,6 @@ class Scheme < ApplicationRecord if scopes.any? filtered_records = filtered_records .left_outer_joins(:scheme_deactivation_periods) - .order("scheme_deactivation_periods.created_at DESC") .merge(scopes.reduce(&:or)) end @@ -44,9 +43,9 @@ class Scheme < ApplicationRecord scope :incomplete, lambda { where.not(confirmed: true) .or(where.not(id: Location.select(:scheme_id).where(confirmed: true).distinct)) - .where.not(id: joins(:scheme_deactivation_periods).reactivating_soon.pluck(:id)) - .where.not(id: joins(:scheme_deactivation_periods).deactivated.pluck(:id)) - .where.not(id: joins(:scheme_deactivation_periods).deactivating_soon.pluck(:id)) + .where.not(id: joins(:scheme_deactivation_periods).reactivating_soon.pluck(:id, :service_name, :confirmed)) + .where.not(id: joins(:scheme_deactivation_periods).deactivated.pluck(:id, :service_name, :confirmed)) + .where.not(id: joins(:scheme_deactivation_periods).deactivating_soon.pluck(:id, :service_name, :confirmed)) } scope :deactivated, lambda { @@ -65,10 +64,10 @@ class Scheme < ApplicationRecord } scope :active_status, lambda { - where.not(id: joins(:scheme_deactivation_periods).reactivating_soon.pluck(:id)) - .where.not(id: joins(:scheme_deactivation_periods).deactivated.pluck(:id)) - .where.not(id: incomplete.pluck(:id)) - .where.not(id: joins(:scheme_deactivation_periods).deactivating_soon.pluck(:id)) + where.not(id: joins(:scheme_deactivation_periods).reactivating_soon.pluck(:id, :service_name, :confirmed)) + .where.not(id: joins(:scheme_deactivation_periods).deactivated.pluck(:id, :service_name, :confirmed)) + .where.not(id: incomplete.pluck(:id, :service_name, :confirmed)) + .where.not(id: joins(:scheme_deactivation_periods).deactivating_soon.pluck(:id, :service_name, :confirmed)) } validate :validate_confirmed diff --git a/app/models/validations/sales/setup_validations.rb b/app/models/validations/sales/setup_validations.rb index 661b5b058..bb99f02ed 100644 --- a/app/models/validations/sales/setup_validations.rb +++ b/app/models/validations/sales/setup_validations.rb @@ -24,6 +24,40 @@ module Validations::Sales::SetupValidations end end + def validate_merged_organisations_saledate(record) + return unless record.saledate && date_valid?("saledate", record) + + if merged_owning_organisation_inactive?(record) + record.errors.add :saledate, I18n.t("validations.setup.saledate.invalid_merged_organisations_saledate", + owning_organisation: record.owning_organisation.name, + owning_organisation_merge_date: record.owning_organisation.merge_date.to_formatted_s(:govuk_date), + owning_absorbing_organisation: record.owning_organisation.absorbing_organisation.name) + end + + if absorbing_owning_organisation_inactive?(record) + record.errors.add :saledate, I18n.t("validations.setup.saledate.invalid_absorbing_organisations_saledate", + owning_organisation: record.owning_organisation.name, + owning_organisation_available_from: record.owning_organisation.created_at.to_formatted_s(:govuk_date)) + end + end + + def validate_organisation(record) + return unless record.saledate && record.owning_organisation + + if record.owning_organisation.present? + if record.owning_organisation&.merge_date.present? && record.owning_organisation.merge_date <= record.saledate + record.errors.add :owning_organisation_id, I18n.t("validations.setup.owning_organisation.inactive_merged_organisation_sales", + owning_organisation: record.owning_organisation.name, + owning_organisation_merge_date: record.owning_organisation.merge_date.to_formatted_s(:govuk_date), + owning_absorbing_organisation: record.owning_organisation.absorbing_organisation.name) + elsif record.owning_organisation&.absorbed_organisations.present? && record.owning_organisation.created_at.to_date > record.saledate.to_date + record.errors.add :owning_organisation_id, I18n.t("validations.setup.owning_organisation.inactive_absorbing_organisation_sales", + owning_organisation: record.owning_organisation.name, + owning_organisation_available_from: record.owning_organisation.created_at.to_formatted_s(:govuk_date)) + end + end + end + private def active_collection_start_date @@ -64,4 +98,12 @@ private ) end end + + def merged_owning_organisation_inactive?(record) + record.owning_organisation&.merge_date.present? && record.owning_organisation.merge_date <= record.saledate + end + + def absorbing_owning_organisation_inactive?(record) + record.owning_organisation&.absorbed_organisations.present? && record.owning_organisation.created_at.to_date > record.saledate.to_date + end end diff --git a/app/services/bulk_upload/downloader.rb b/app/services/bulk_upload/downloader.rb index 8fcd9ccca..f3d5117e4 100644 --- a/app/services/bulk_upload/downloader.rb +++ b/app/services/bulk_upload/downloader.rb @@ -38,7 +38,7 @@ private def s3_storage_service Storage::S3Service.new( - Configuration::PaasConfigurationService.new, + PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["CSV_DOWNLOAD_PAAS_INSTANCE"], ) end diff --git a/config/initializers/multi_logger.rb b/config/initializers/multi_logger.rb new file mode 100644 index 000000000..d39d786c1 --- /dev/null +++ b/config/initializers/multi_logger.rb @@ -0,0 +1,17 @@ +class MultiLogger + def initialize(*targets) + @targets = targets + end + + def info(data) + @targets.each { |t| t.info(data) } + end + + def warn(data) + @targets.each { |t| t.warn(data) } + end + + def error(data) + @targets.each { |t| t.error(data) } + end +end diff --git a/config/initializers/rack_attack.rb b/config/initializers/rack_attack.rb index febd58a50..9b201b054 100644 --- a/config/initializers/rack_attack.rb +++ b/config/initializers/rack_attack.rb @@ -1,14 +1,18 @@ require "configuration/configuration_service" require "configuration/paas_configuration_service" +require "configuration/env_configuration_service" +require Rails.root.join("app/helpers/platform_helper") + +configuration_service = PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new if Rails.env.development? || Rails.env.test? Rack::Attack.cache.store = ActiveSupport::Cache::MemoryStore.new Rack::Attack.enabled = false elsif Rails.env.review? - redis_url = Configuration::PaasConfigurationService.new.redis_uris.to_a[0][1] + redis_url = configuration_service.redis_uris.to_a[0][1] Rack::Attack.cache.store = ActiveSupport::Cache::RedisCacheStore.new(url: redis_url) else - redis_url = Configuration::PaasConfigurationService.new.redis_uris[:"dluhc-core-#{Rails.env}-redis"] + redis_url = configuration_service.redis_uris[:"dluhc-core-#{Rails.env}-redis"] Rack::Attack.cache.store = ActiveSupport::Cache::RedisCacheStore.new(url: redis_url) end diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index c8e38e1d7..bc1b578ef 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -1,8 +1,10 @@ require "sidekiq/web" require "sidekiq/cron/web" +configuration_service = PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new + if Rails.env.staging? || Rails.env.production? - redis_url = Configuration::PaasConfigurationService.new.redis_uris[:"dluhc-core-#{Rails.env}-redis"] + redis_url = configuration_service.redis_uris[:"dluhc-core-#{Rails.env}-redis"] Sidekiq.configure_server do |config| config.redis = { url: redis_url } @@ -14,7 +16,7 @@ if Rails.env.staging? || Rails.env.production? end if Rails.env.review? - redis_url = Configuration::PaasConfigurationService.new.redis_uris.to_a[0][1] + redis_url = configuration_service.redis_uris.to_a[0][1] Sidekiq.configure_server do |config| config.redis = { url: redis_url } diff --git a/config/locales/en.yml b/config/locales/en.yml index f99d08c33..7ad173de8 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -239,6 +239,8 @@ en: previous_and_current_collection_year: "Enter a date within the %{previous_start_year_short}/%{previous_end_year_short} or %{previous_end_year_short}/%{current_end_year_short} collection years, which is between %{previous_start_year_long} and %{current_end_year_long}" year_not_two_digits: "Sale completion year must be 2 digits" + invalid_merged_organisations_saledate: "Enter a date when the owning organisation was active. %{owning_organisation} became inactive on %{owning_organisation_merge_date} and was replaced by %{owning_absorbing_organisation}." + invalid_absorbing_organisations_saledate: "Enter a date when the owning organisation was active. %{owning_organisation} became active on %{owning_organisation_available_from}." type: percentage_bought_must_be_at_least_threshold: "The minimum increase in equity while staircasing is %{threshold}% for this shared ownership type" @@ -288,6 +290,8 @@ en: data_sharing_agreement_not_signed: "The organisation must accept the Data Sharing Agreement before it can be selected as the owning organisation." inactive_merged_organisation: "The owning organisation must be active on the tenancy start date. %{owning_organisation} became inactive on %{owning_organisation_merge_date} and was replaced by %{owning_absorbing_organisation}." inactive_absorbing_organisation: "The owning organisation must be active on the tenancy start date. %{owning_organisation} became active on %{owning_organisation_available_from}." + inactive_merged_organisation_sales: "The owning organisation must be active on the sale completion date. %{owning_organisation} became inactive on %{owning_organisation_merge_date} and was replaced by %{owning_absorbing_organisation}." + inactive_absorbing_organisation_sales: "The owning organisation must be active on the sale completion date. %{owning_organisation} became active on %{owning_organisation_available_from}." managing_organisation: invalid: "Please select the owning organisation or managing organisation that you belong to" data_sharing_agreement_not_signed: "The organisation must accept the Data Sharing Agreement before it can be selected as the managing organisation." diff --git a/lib/tasks/correct_incref_values.rake b/lib/tasks/correct_incref_values.rake index 0b1921998..cc9278a72 100644 --- a/lib/tasks/correct_incref_values.rake +++ b/lib/tasks/correct_incref_values.rake @@ -1,6 +1,6 @@ desc "Alter incref values for non imported lettings logs in the database" task correct_incref_values: :environment do - LettingsLog.where(old_id: nil, net_income_known: 0).update!(incref: 0) - LettingsLog.where(old_id: nil, net_income_known: 1).update!(incref: 2) - LettingsLog.where(old_id: nil, net_income_known: 2).update!(incref: 1) + LettingsLog.where(old_id: nil, net_income_known: 0).update_all(incref: 0) + LettingsLog.where(old_id: nil, net_income_known: 1).update_all(incref: 2) + LettingsLog.where(old_id: nil, net_income_known: 2).update_all(incref: 1) end diff --git a/lib/tasks/data_import.rake b/lib/tasks/data_import.rake index 0dede82a3..7e71772e3 100644 --- a/lib/tasks/data_import.rake +++ b/lib/tasks/data_import.rake @@ -5,7 +5,7 @@ namespace :core do path = args[:path] raise "Usage: rake core:data_import['data_type', 'path/to/xml_files']" if path.blank? || type.blank? - storage_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"]) + storage_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"]) case type when "organisation" diff --git a/lib/tasks/data_import_field.rake b/lib/tasks/data_import_field.rake index 54d25517b..a30a2ec43 100644 --- a/lib/tasks/data_import_field.rake +++ b/lib/tasks/data_import_field.rake @@ -8,7 +8,7 @@ namespace :core do # We only allow a reduced list of known fields to be updatable case field when "tenancycode", "major_repairs", "lettings_allocation", "offered" - s3_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"]) + s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"]) archive_io = s3_service.get_file_io(path) archive_service = Storage::ArchiveService.new(archive_io) if archive_service.folder_present?("logs") diff --git a/lib/tasks/full_import.rake b/lib/tasks/full_import.rake index bacdb500a..7d230c160 100644 --- a/lib/tasks/full_import.rake +++ b/lib/tasks/full_import.rake @@ -1,4 +1,4 @@ -Import = Struct.new("Import", :import_class, :import_method, :folder) +Import = Struct.new("Import", :import_class, :import_method, :folder, :logger) namespace :import do desc "Run initial import steps - orgs, schemes, users etc (without triggering user invite emails)" @@ -6,37 +6,43 @@ namespace :import do institutions_csv_name = args[:institutions_csv_name] raise "Usage: rake import:initial['institutions_csv_name']" if institutions_csv_name.blank? - s3_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"]) + s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"]) csv = CSV.parse(s3_service.get_file_io(institutions_csv_name), headers: true) + logs_string = StringIO.new + logger = MultiLogger.new(Rails.logger, Logger.new(logs_string)) org_count = csv.length initial_import_list = [ - Import.new(Imports::OrganisationImportService, :create_organisations, "institution"), - Import.new(Imports::OrganisationRelationshipImportService, :create_organisation_relationships, "institution-link"), - Import.new(Imports::SchemeImportService, :create_schemes, "mgmtgroups"), - Import.new(Imports::SchemeLocationImportService, :create_scheme_locations, "schemes"), - Import.new(Imports::UserImportService, :create_users, "user"), - Import.new(Imports::DataProtectionConfirmationImportService, :create_data_protection_confirmations, "dataprotect"), - Import.new(Imports::OrganisationRentPeriodImportService, :create_organisation_rent_periods, "rent-period"), + Import.new(Imports::OrganisationImportService, :create_organisations, "institution", logger), + Import.new(Imports::OrganisationRelationshipImportService, :create_organisation_relationships, "institution-link", logger), + Import.new(Imports::SchemeImportService, :create_schemes, "mgmtgroups", logger), + Import.new(Imports::SchemeLocationImportService, :create_scheme_locations, "schemes", logger), + Import.new(Imports::UserImportService, :create_users, "user", logger), + Import.new(Imports::DataProtectionConfirmationImportService, :create_data_protection_confirmations, "dataprotect", logger), + Import.new(Imports::OrganisationRentPeriodImportService, :create_organisation_rent_periods, "rent-period", logger), ] - Rails.logger.info("Beginning initial imports for #{org_count} organisations") + logger.info("Beginning initial imports for #{org_count} organisations") csv.each do |row| archive_path = row[1] archive_io = s3_service.get_file_io(archive_path) archive_service = Storage::ArchiveService.new(archive_io) - - Rails.logger.info("Performing initial imports for organisation #{row[0]}") + logger.info("Performing initial imports for organisation #{row[0]}") initial_import_list.each do |step| if archive_service.folder_present?(step.folder) - step.import_class.new(archive_service).send(step.import_method, step.folder) + step.import_class.new(archive_service).send(step.import_method, step.folder, step.logger) end end + + log_file = "#{File.basename(institutions_csv_name, File.extname(institutions_csv_name))}_#{File.basename(archive_path, File.extname(archive_path))}_initial.log" + s3_service.write_file(log_file, logs_string.string) + logs_string.rewind + logs_string.truncate(0) end - Rails.logger.info("Finished initial imports") + logger.info("Finished initial imports") end desc "Run logs import steps" @@ -44,16 +50,18 @@ namespace :import do institutions_csv_name = args[:institutions_csv_name] raise "Usage: rake import:logs['institutions_csv_name']" if institutions_csv_name.blank? - s3_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"]) + s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"]) csv = CSV.parse(s3_service.get_file_io(institutions_csv_name), headers: true) org_count = csv.length + logs_string = StringIO.new + logger = MultiLogger.new(Rails.logger, Logger.new(logs_string)) logs_import_list = [ - Import.new(Imports::LettingsLogsImportService, :create_logs, "logs"), - Import.new(Imports::SalesLogsImportService, :create_logs, "logs"), + Import.new(Imports::LettingsLogsImportService, :create_logs, "logs", logger), + Import.new(Imports::SalesLogsImportService, :create_logs, "logs", logger), ] - Rails.logger.info("Beginning log imports for #{org_count} organisations") + logger.info("Beginning log imports for #{org_count} organisations") csv.each do |row| archive_path = row[1] @@ -61,16 +69,21 @@ namespace :import do archive_service = Storage::ArchiveService.new(archive_io) log_count = row[2].to_i + row[3].to_i + row[4].to_i + row[5].to_i - Rails.logger.info("Importing logs for organisation #{row[0]}, expecting #{log_count} logs") + logger.info("Importing logs for organisation #{row[0]}, expecting #{log_count} logs") logs_import_list.each do |step| if archive_service.folder_present?(step.folder) - step.import_class.new(archive_service).send(step.import_method, step.folder) + step.import_class.new(archive_service).send(step.import_method, step.folder, step.logger) end end + + log_file = "#{File.basename(institutions_csv_name, File.extname(institutions_csv_name))}_#{File.basename(archive_path, File.extname(archive_path))}_logs.log" + s3_service.write_file(log_file, logs_string.string) + logs_string.rewind + logs_string.truncate(0) end - Rails.logger.info("Log import complete") + logger.info("Log import complete") end desc "Trigger invite emails for imported users" @@ -78,7 +91,7 @@ namespace :import do institutions_csv_name = args[:institutions_csv_name] raise "Usage: rake import:trigger_invites['institutions_csv_name']" if institutions_csv_name.blank? - s3_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"]) + s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"]) csv = CSV.parse(s3_service.get_file_io(institutions_csv_name), headers: true) Rails.logger.info("Triggering user invite emails") @@ -99,7 +112,7 @@ namespace :import do institutions_csv_name = args[:institutions_csv_name] raise "Usage: rake import:generate_reports['institutions_csv_name']" if institutions_csv_name.blank? - s3_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"]) + s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"]) institutions_csv = CSV.parse(s3_service.get_file_io(institutions_csv_name), headers: true) Imports::ImportReportService.new(s3_service, institutions_csv).create_reports(institutions_csv_name) diff --git a/spec/helpers/platform_helper_spec.rb b/spec/helpers/platform_helper_spec.rb new file mode 100644 index 000000000..2a90f7762 --- /dev/null +++ b/spec/helpers/platform_helper_spec.rb @@ -0,0 +1,15 @@ +require "rails_helper" + +RSpec.describe PlatformHelper do + describe "is_paas?" do + it "returns true if the VCAP_SERVICES environment variable exists" do + allow(ENV).to receive(:[]).with("VCAP_SERVICES").and_return("dummy") + expect(described_class.is_paas?).to eq(true) + end + + it "returns false if the VCAP_SERVICES environment variable doesn't exist" do + allow(ENV).to receive(:[]).with("VCAP_SERVICES") + expect(described_class.is_paas?).to eq(false) + end + end +end diff --git a/spec/lib/tasks/correct_incref_values_spec.rb b/spec/lib/tasks/correct_incref_values_spec.rb index fc046dcf6..8acddb636 100644 --- a/spec/lib/tasks/correct_incref_values_spec.rb +++ b/spec/lib/tasks/correct_incref_values_spec.rb @@ -31,6 +31,14 @@ RSpec.describe "correct_incref_values" do task.invoke expect(lettings_log.reload.incref).to eq(1) end + + it "skips validations for previous years" do + lettings_log.update!(net_income_known: 2, incref: nil) + lettings_log.startdate = Time.zone.local(2021, 3, 3) + lettings_log.save!(validate: false) + task.invoke + expect(lettings_log.reload.incref).to eq(1) + end end end end diff --git a/spec/lib/tasks/data_import_spec.rb b/spec/lib/tasks/data_import_spec.rb index b8b56801a..0cba6952f 100644 --- a/spec/lib/tasks/data_import_spec.rb +++ b/spec/lib/tasks/data_import_spec.rb @@ -3,6 +3,7 @@ require "rake" describe "data import", type: :task do let(:instance_name) { "paas_import_instance" } + let(:env_config_service) { instance_double(Configuration::EnvConfigurationService) } let(:paas_config_service) { instance_double(Configuration::PaasConfigurationService) } let(:storage_service) { instance_double(Storage::S3Service) } @@ -16,8 +17,10 @@ describe "data import", type: :task do allow(Storage::S3Service).to receive(:new).and_return(storage_service) allow(Configuration::PaasConfigurationService).to receive(:new).and_return(paas_config_service) + allow(Configuration::EnvConfigurationService).to receive(:new).and_return(env_config_service) allow(ENV).to receive(:[]) allow(ENV).to receive(:[]).with("IMPORT_PAAS_INSTANCE").and_return(instance_name) + allow(ENV).to receive(:[]).with("VCAP_SERVICES").and_return("dummy") end context "when importing organisation data" do @@ -29,13 +32,22 @@ describe "data import", type: :task do allow(Imports::OrganisationImportService).to receive(:new).and_return(import_service) end - it "creates an organisation from the given XML file" do + it "creates an organisation from the given XML file when the VCAP_SERVICES environment variable exists" do expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name) expect(Imports::OrganisationImportService).to receive(:new).with(storage_service) expect(import_service).to receive(:create_organisations).with(fixture_path) task.invoke(type, fixture_path) end + + it "creates an organisation from the given XML file when the VCAP_SERVICES environment variable does not exist" do + allow(ENV).to receive(:[]).with("VCAP_SERVICES") + expect(Storage::S3Service).to receive(:new).with(env_config_service, instance_name) + expect(Imports::OrganisationImportService).to receive(:new).with(storage_service) + expect(import_service).to receive(:create_organisations).with(fixture_path) + + task.invoke(type, fixture_path) + end end context "when importing user data" do diff --git a/spec/lib/tasks/date_import_field_spec.rb b/spec/lib/tasks/date_import_field_spec.rb index a30920694..c34427e04 100644 --- a/spec/lib/tasks/date_import_field_spec.rb +++ b/spec/lib/tasks/date_import_field_spec.rb @@ -6,6 +6,7 @@ describe "rake core:data_import_field", type: :task do let(:instance_name) { "paas_import_instance" } let(:storage_service) { instance_double(Storage::S3Service) } + let(:env_config_service) { instance_double(Configuration::EnvConfigurationService) } let(:paas_config_service) { instance_double(Configuration::PaasConfigurationService) } before do @@ -14,9 +15,11 @@ describe "rake core:data_import_field", type: :task do task.reenable allow(Storage::S3Service).to receive(:new).and_return(storage_service) + allow(Configuration::EnvConfigurationService).to receive(:new).and_return(env_config_service) allow(Configuration::PaasConfigurationService).to receive(:new).and_return(paas_config_service) allow(ENV).to receive(:[]) allow(ENV).to receive(:[]).with("IMPORT_PAAS_INSTANCE").and_return(instance_name) + allow(ENV).to receive(:[]).with("VCAP_SERVICES").and_return("dummy") allow(Imports::LettingsLogsFieldImportService).to receive(:new).and_return(import_service) end @@ -34,13 +37,22 @@ describe "rake core:data_import_field", type: :task do context "and we update the tenancycode field" do let(:field) { "tenancycode" } - it "updates the logs from the given XML file" do + it "updates the logs from the given XML file when the VCAP_SERVICES environment variable exists" do expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name) expect(storage_service).to receive(:get_file_io).with("spec/fixtures/imports/logs") expect(Imports::LettingsLogsFieldImportService).to receive(:new).with(archive_service) expect(import_service).to receive(:update_field).with(field, "logs") task.invoke(field, fixture_path) end + + it "updates the logs from the given XML file when the VCAP_SERVICES environment variable does not exist" do + allow(ENV).to receive(:[]).with("VCAP_SERVICES") + expect(Storage::S3Service).to receive(:new).with(env_config_service, instance_name) + expect(storage_service).to receive(:get_file_io).with("spec/fixtures/imports/logs") + expect(Imports::LettingsLogsFieldImportService).to receive(:new).with(archive_service) + expect(import_service).to receive(:update_field).with(field, "logs") + task.invoke(field, fixture_path) + end end context "and we update the lettings_allocation fields" do diff --git a/spec/lib/tasks/full_import_spec.rb b/spec/lib/tasks/full_import_spec.rb index f263f15fc..9d530ef6c 100644 --- a/spec/lib/tasks/full_import_spec.rb +++ b/spec/lib/tasks/full_import_spec.rb @@ -4,6 +4,7 @@ require "rake" describe "full import", type: :task do let(:instance_name) { "paas_import_instance" } let(:paas_config_service) { instance_double(Configuration::PaasConfigurationService) } + let(:env_config_service) { instance_double(Configuration::EnvConfigurationService) } let(:storage_service) { instance_double(Storage::S3Service) } let(:orgs_list) { "Institution name,Id,Old Completed lettings logs,Old In progress lettings logs,Old Completed sales logs,Old In progress sales logs\norg1,1.zip,0,0,0,0\norg2,2.zip,0,0,0,0" } @@ -12,6 +13,7 @@ describe "full import", type: :task do allow(storage_service).to receive(:write_file).and_return(nil) allow(storage_service).to receive(:get_file_io).and_return(orgs_list) allow(Configuration::PaasConfigurationService).to receive(:new).and_return(paas_config_service) + allow(Configuration::EnvConfigurationService).to receive(:new).and_return(env_config_service) allow(ENV).to receive(:[]) allow(ENV).to receive(:[]).with("IMPORT_PAAS_INSTANCE").and_return(instance_name) end @@ -32,13 +34,85 @@ describe "full import", type: :task do allow(Imports::ImportReportService).to receive(:new).and_return(import_report_service) end - it "creates a report using given organisation csv" do + it "creates a report using given organisation csv when the VCAP_SERVICES environment variable exists" do + allow(ENV).to receive(:[]).with("VCAP_SERVICES").and_return("dummy") expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name) expect(Imports::ImportReportService).to receive(:new).with(storage_service, CSV.parse(orgs_list, headers: true)) expect(import_report_service).to receive(:create_reports).with("some_name") task.invoke("some_name") end + + it "creates a report using given organisation csv when the VCAP_SERVICES environment variable does not exist" do + allow(ENV).to receive(:[]).with("VCAP_SERVICES") + expect(Storage::S3Service).to receive(:new).with(env_config_service, instance_name) + expect(Imports::ImportReportService).to receive(:new).with(storage_service, CSV.parse(orgs_list, headers: true)) + expect(import_report_service).to receive(:create_reports).with("some_name") + + task.invoke("some_name") + end + end + end + + describe "import:initial" do + subject(:task) { Rake::Task["import:initial"] } + + let(:archive_service) { instance_double(Storage::ArchiveService) } + + before do + Rake.application.rake_require("tasks/full_import") + Rake::Task.define_task(:environment) + task.reenable + end + + context "when calling the initial import" do + before do + allow(Storage::ArchiveService).to receive(:new).and_return(archive_service) + allow(archive_service).to receive(:folder_present?).and_return(false) + allow(Imports::OrganisationImportService).to receive(:new).and_return(instance_double(Imports::OrganisationImportService)) + allow(Imports::SchemeImportService).to receive(:new).and_return(instance_double(Imports::SchemeImportService)) + allow(Imports::SchemeLocationImportService).to receive(:new).and_return(instance_double(Imports::SchemeLocationImportService)) + allow(Imports::UserImportService).to receive(:new).and_return(instance_double(Imports::UserImportService)) + allow(Imports::DataProtectionConfirmationImportService).to receive(:new).and_return(instance_double(Imports::DataProtectionConfirmationImportService)) + allow(Imports::OrganisationRentPeriodImportService).to receive(:new).and_return(instance_double(Imports::OrganisationRentPeriodImportService)) + end + + it "creates a report using given organisation csv" do + expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name) + expect(storage_service).to receive(:write_file).with("some_name_1_initial.log", / INFO -- : Performing initial imports for organisation org1/) + expect(storage_service).to receive(:write_file).with("some_name_2_initial.log", / INFO -- : Performing initial imports for organisation org2/) + + task.invoke("some_name.csv") + end + end + end + + describe "import:logs" do + subject(:task) { Rake::Task["import:logs"] } + + let(:archive_service) { instance_double(Storage::ArchiveService) } + + before do + Rake.application.rake_require("tasks/full_import") + Rake::Task.define_task(:environment) + task.reenable + end + + context "when calling the logs import" do + before do + allow(Storage::ArchiveService).to receive(:new).and_return(archive_service) + allow(archive_service).to receive(:folder_present?).and_return(false) + allow(Imports::LettingsLogsImportService).to receive(:new).and_return(instance_double(Imports::LettingsLogsImportService)) + allow(Imports::SalesLogsImportService).to receive(:new).and_return(instance_double(Imports::SalesLogsImportService)) + end + + it "creates a report using given organisation csv" do + expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name) + expect(storage_service).to receive(:write_file).with("some_name_1_logs.log", / INFO -- : Importing logs for organisation org1, expecting 0 logs/) + expect(storage_service).to receive(:write_file).with("some_name_2_logs.log", / INFO -- : Importing logs for organisation org2, expecting 0 logs/) + + task.invoke("some_name.csv") + end end end end diff --git a/spec/models/scheme_spec.rb b/spec/models/scheme_spec.rb index 049da077a..0001d55f7 100644 --- a/spec/models/scheme_spec.rb +++ b/spec/models/scheme_spec.rb @@ -110,7 +110,7 @@ RSpec.describe Scheme, type: :model do end context "when filtering by status" do - let!(:incomplete_scheme) { FactoryBot.create(:scheme, :incomplete) } + let!(:incomplete_scheme) { FactoryBot.create(:scheme, :incomplete, service_name: "name") } let(:active_scheme) { FactoryBot.create(:scheme) } let(:deactivating_soon_scheme) { FactoryBot.create(:scheme) } let(:deactivated_scheme) { FactoryBot.create(:scheme) } @@ -141,6 +141,13 @@ RSpec.describe Scheme, type: :model do end end + context "when filtering by incomplete status and searching" do + it "returns only incomplete schemes" do + expect(described_class.search_by("name").filter_by_status(%w[incomplete]).count).to eq(1) + expect(described_class.search_by("name").filter_by_status(%w[incomplete]).first).to eq(incomplete_scheme) + end + end + context "when filtering by active status" do it "returns only active schemes" do expect(described_class.filter_by_status(%w[active]).count).to eq(1) diff --git a/spec/models/validations/sales/setup_validations_spec.rb b/spec/models/validations/sales/setup_validations_spec.rb index ecbb7e0d3..f7f01eaf7 100644 --- a/spec/models/validations/sales/setup_validations_spec.rb +++ b/spec/models/validations/sales/setup_validations_spec.rb @@ -185,4 +185,100 @@ RSpec.describe Validations::Sales::SetupValidations do end end end + + describe "#validate_merged_organisations_saledate" do + let(:record) { build(:sales_log) } + let(:absorbing_organisation) { create(:organisation, created_at: Time.zone.local(2023, 2, 1), name: "Absorbing org") } + let(:merged_organisation) { create(:organisation, name: "Merged org") } + + around do |example| + Timecop.freeze(Time.zone.local(2023, 5, 1)) + example.run + Timecop.return + end + + before do + merged_organisation.update!(absorbing_organisation:, merge_date: Time.zone.local(2023, 2, 2)) + end + + context "and owning organisation is no longer active" do + it "does not allow saledate after organisation has been merged" do + record.saledate = Time.zone.local(2023, 3, 1) + record.owning_organisation_id = merged_organisation.id + setup_validator.validate_merged_organisations_saledate(record) + expect(record.errors["saledate"]).to include(match "Enter a date when the owning organisation was active. Merged org became inactive on 2 February 2023 and was replaced by Absorbing org.") + end + + it "allows saledate before organisation has been merged" do + record.saledate = Time.zone.local(2023, 1, 1) + record.owning_organisation_id = merged_organisation.id + setup_validator.validate_merged_organisations_saledate(record) + expect(record.errors["saledate"]).to be_empty + end + end + + context "and owning organisation is not yet active during the saledate" do + it "does not allow saledate before absorbing organisation has been created" do + record.saledate = Time.zone.local(2023, 1, 1) + record.owning_organisation_id = absorbing_organisation.id + setup_validator.validate_merged_organisations_saledate(record) + expect(record.errors["saledate"]).to include(match "Enter a date when the owning organisation was active. Absorbing org became active on 1 February 2023.") + end + + it "allows saledate after absorbing organisation has been created" do + record.saledate = Time.zone.local(2023, 2, 2) + record.owning_organisation_id = absorbing_organisation.id + setup_validator.validate_merged_organisations_saledate(record) + expect(record.errors["saledate"]).to be_empty + end + end + end + + describe "#validate_organisation" do + let(:record) { build(:sales_log) } + + context "when organisations are merged" do + let(:absorbing_organisation) { create(:organisation, created_at: Time.zone.local(2023, 2, 1), name: "Absorbing org") } + let(:merged_organisation) { create(:organisation, name: "Merged org") } + + around do |example| + Timecop.freeze(Time.zone.local(2023, 5, 1)) + merged_organisation.update!(merge_date: Time.zone.local(2023, 2, 2), absorbing_organisation:) + example.run + Timecop.return + end + + context "and owning organisation is no longer active" do + it "does not allow organisation that has been merged" do + record.saledate = Time.zone.local(2023, 3, 1) + record.owning_organisation_id = merged_organisation.id + setup_validator.validate_organisation(record) + expect(record.errors["owning_organisation_id"]).to include(match "The owning organisation must be active on the sale completion date. Merged org became inactive on 2 February 2023 and was replaced by Absorbing org.") + end + + it "allows organisation before it has been merged" do + record.saledate = Time.zone.local(2023, 1, 1) + record.owning_organisation_id = merged_organisation.id + setup_validator.validate_organisation(record) + expect(record.errors["owning_organisation_id"]).to be_empty + end + end + + context "and owning organisation is not yet active during the saledate" do + it "does not allow absorbing organisation before it had been created" do + record.saledate = Time.zone.local(2023, 1, 1) + record.owning_organisation_id = absorbing_organisation.id + setup_validator.validate_organisation(record) + expect(record.errors["owning_organisation_id"]).to include(match "The owning organisation must be active on the sale completion date. Absorbing org became active on 1 February 2023.") + end + + it "allows absorbing organisation after it has been created" do + record.saledate = Time.zone.local(2023, 2, 2) + record.owning_organisation_id = absorbing_organisation.id + setup_validator.validate_organisation(record) + expect(record.errors["owning_organisation_id"]).to be_empty + end + end + end + end end diff --git a/spec/services/imports/organisation_import_service_spec.rb b/spec/services/imports/organisation_import_service_spec.rb index ca7bbf492..f9a47abe8 100644 --- a/spec/services/imports/organisation_import_service_spec.rb +++ b/spec/services/imports/organisation_import_service_spec.rb @@ -2,7 +2,8 @@ require "rails_helper" RSpec.describe Imports::OrganisationImportService do let(:storage_service) { instance_double(Storage::S3Service) } - let(:logger) { instance_double(Rails::Rack::Logger) } + let(:logs_string) { StringIO.new } + let(:logger) { MultiLogger.new(Rails.logger, Logger.new(logs_string)) } let(:folder_name) { "organisations" } let(:filenames) { %w[my_folder/my_file1.xml my_folder/my_file2.xml] } let(:fixture_directory) { "spec/fixtures/imports/institution" } @@ -82,9 +83,11 @@ RSpec.describe Imports::OrganisationImportService do expect(storage_service).to receive(:list_files).with(folder_name).twice expect(storage_service).to receive(:get_file_io).with(filenames[0]).twice expect(logger).not_to receive(:warn) + expect(Rails.logger).not_to receive(:warn) expect { import_service.create_organisations(folder_name) }.to change(Organisation, :count).by(1) expect { import_service.create_organisations(folder_name) }.to change(Organisation, :count).by(0) + expect(logs_string.string).to include("Organisation my_new_organisation is already present with old visible ID 1, skipping.") expect(Organisation).to exist(old_visible_id: "1", name: "HA Ltd") end