diff --git a/config/initializers/multi_logger.rb b/config/initializers/multi_logger.rb new file mode 100644 index 000000000..d39d786c1 --- /dev/null +++ b/config/initializers/multi_logger.rb @@ -0,0 +1,17 @@ +class MultiLogger + def initialize(*targets) + @targets = targets + end + + def info(data) + @targets.each { |t| t.info(data) } + end + + def warn(data) + @targets.each { |t| t.warn(data) } + end + + def error(data) + @targets.each { |t| t.error(data) } + end +end diff --git a/lib/tasks/full_import.rake b/lib/tasks/full_import.rake index 4fea7ef93..0313b57fc 100644 --- a/lib/tasks/full_import.rake +++ b/lib/tasks/full_import.rake @@ -1,4 +1,4 @@ -Import = Struct.new("Import", :import_class, :import_method, :folder) +Import = Struct.new("Import", :import_class, :import_method, :folder, :logger) namespace :import do desc "Run initial import steps - orgs, schemes, users etc (without triggering user invite emails)" @@ -8,34 +8,40 @@ namespace :import do s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"]) csv = CSV.parse(s3_service.get_file_io(institutions_csv_name), headers: true) + logs_string = StringIO.new + logger = MultiLogger.new(Rails.logger, Logger.new(logs_string)) org_count = csv.length initial_import_list = [ - Import.new(Imports::OrganisationImportService, :create_organisations, "institution"), - Import.new(Imports::SchemeImportService, :create_schemes, "mgmtgroups"), - Import.new(Imports::SchemeLocationImportService, :create_scheme_locations, "schemes"), - Import.new(Imports::UserImportService, :create_users, "user"), - Import.new(Imports::DataProtectionConfirmationImportService, :create_data_protection_confirmations, "dataprotect"), - Import.new(Imports::OrganisationRentPeriodImportService, :create_organisation_rent_periods, "rent-period"), + Import.new(Imports::OrganisationImportService, :create_organisations, "institution", logger), + Import.new(Imports::SchemeImportService, :create_schemes, "mgmtgroups", logger), + Import.new(Imports::SchemeLocationImportService, :create_scheme_locations, "schemes", logger), + Import.new(Imports::UserImportService, :create_users, "user", logger), + Import.new(Imports::DataProtectionConfirmationImportService, :create_data_protection_confirmations, "dataprotect", logger), + Import.new(Imports::OrganisationRentPeriodImportService, :create_organisation_rent_periods, "rent-period", logger), ] - Rails.logger.info("Beginning initial imports for #{org_count} organisations") + logger.info("Beginning initial imports for #{org_count} organisations") csv.each do |row| archive_path = row[1] archive_io = s3_service.get_file_io(archive_path) archive_service = Storage::ArchiveService.new(archive_io) - - Rails.logger.info("Performing initial imports for organisation #{row[0]}") + logger.info("Performing initial imports for organisation #{row[0]}") initial_import_list.each do |step| if archive_service.folder_present?(step.folder) - step.import_class.new(archive_service).send(step.import_method, step.folder) + step.import_class.new(archive_service).send(step.import_method, step.folder, step.logger) end end + + log_file = "#{File.basename(institutions_csv_name, File.extname(institutions_csv_name))}_#{File.basename(archive_path, File.extname(archive_path))}_initial.log" + s3_service.write_file(log_file, logs_string.string) + logs_string.rewind + logs_string.truncate(0) end - Rails.logger.info("Finished initial imports") + logger.info("Finished initial imports") end desc "Run logs import steps" @@ -46,13 +52,15 @@ namespace :import do s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"]) csv = CSV.parse(s3_service.get_file_io(institutions_csv_name), headers: true) org_count = csv.length + logs_string = StringIO.new + logger = MultiLogger.new(Rails.logger, Logger.new(logs_string)) logs_import_list = [ - Import.new(Imports::LettingsLogsImportService, :create_logs, "logs"), - Import.new(Imports::SalesLogsImportService, :create_logs, "logs"), + Import.new(Imports::LettingsLogsImportService, :create_logs, "logs", logger), + Import.new(Imports::SalesLogsImportService, :create_logs, "logs", logger), ] - Rails.logger.info("Beginning log imports for #{org_count} organisations") + logger.info("Beginning log imports for #{org_count} organisations") csv.each do |row| archive_path = row[1] @@ -60,16 +68,21 @@ namespace :import do archive_service = Storage::ArchiveService.new(archive_io) log_count = row[2].to_i + row[3].to_i + row[4].to_i + row[5].to_i - Rails.logger.info("Importing logs for organisation #{row[0]}, expecting #{log_count} logs") + logger.info("Importing logs for organisation #{row[0]}, expecting #{log_count} logs") logs_import_list.each do |step| if archive_service.folder_present?(step.folder) - step.import_class.new(archive_service).send(step.import_method, step.folder) + step.import_class.new(archive_service).send(step.import_method, step.folder, step.logger) end end + + log_file = "#{File.basename(institutions_csv_name, File.extname(institutions_csv_name))}_#{File.basename(archive_path, File.extname(archive_path))}_logs.log" + s3_service.write_file(log_file, logs_string.string) + logs_string.rewind + logs_string.truncate(0) end - Rails.logger.info("Log import complete") + logger.info("Log import complete") end desc "Trigger invite emails for imported users" diff --git a/spec/lib/tasks/full_import_spec.rb b/spec/lib/tasks/full_import_spec.rb index 4c123d5b5..9d530ef6c 100644 --- a/spec/lib/tasks/full_import_spec.rb +++ b/spec/lib/tasks/full_import_spec.rb @@ -53,4 +53,66 @@ describe "full import", type: :task do end end end + + describe "import:initial" do + subject(:task) { Rake::Task["import:initial"] } + + let(:archive_service) { instance_double(Storage::ArchiveService) } + + before do + Rake.application.rake_require("tasks/full_import") + Rake::Task.define_task(:environment) + task.reenable + end + + context "when calling the initial import" do + before do + allow(Storage::ArchiveService).to receive(:new).and_return(archive_service) + allow(archive_service).to receive(:folder_present?).and_return(false) + allow(Imports::OrganisationImportService).to receive(:new).and_return(instance_double(Imports::OrganisationImportService)) + allow(Imports::SchemeImportService).to receive(:new).and_return(instance_double(Imports::SchemeImportService)) + allow(Imports::SchemeLocationImportService).to receive(:new).and_return(instance_double(Imports::SchemeLocationImportService)) + allow(Imports::UserImportService).to receive(:new).and_return(instance_double(Imports::UserImportService)) + allow(Imports::DataProtectionConfirmationImportService).to receive(:new).and_return(instance_double(Imports::DataProtectionConfirmationImportService)) + allow(Imports::OrganisationRentPeriodImportService).to receive(:new).and_return(instance_double(Imports::OrganisationRentPeriodImportService)) + end + + it "creates a report using given organisation csv" do + expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name) + expect(storage_service).to receive(:write_file).with("some_name_1_initial.log", / INFO -- : Performing initial imports for organisation org1/) + expect(storage_service).to receive(:write_file).with("some_name_2_initial.log", / INFO -- : Performing initial imports for organisation org2/) + + task.invoke("some_name.csv") + end + end + end + + describe "import:logs" do + subject(:task) { Rake::Task["import:logs"] } + + let(:archive_service) { instance_double(Storage::ArchiveService) } + + before do + Rake.application.rake_require("tasks/full_import") + Rake::Task.define_task(:environment) + task.reenable + end + + context "when calling the logs import" do + before do + allow(Storage::ArchiveService).to receive(:new).and_return(archive_service) + allow(archive_service).to receive(:folder_present?).and_return(false) + allow(Imports::LettingsLogsImportService).to receive(:new).and_return(instance_double(Imports::LettingsLogsImportService)) + allow(Imports::SalesLogsImportService).to receive(:new).and_return(instance_double(Imports::SalesLogsImportService)) + end + + it "creates a report using given organisation csv" do + expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name) + expect(storage_service).to receive(:write_file).with("some_name_1_logs.log", / INFO -- : Importing logs for organisation org1, expecting 0 logs/) + expect(storage_service).to receive(:write_file).with("some_name_2_logs.log", / INFO -- : Importing logs for organisation org2, expecting 0 logs/) + + task.invoke("some_name.csv") + end + end + end end diff --git a/spec/services/imports/organisation_import_service_spec.rb b/spec/services/imports/organisation_import_service_spec.rb index c8fcd4829..8496c046f 100644 --- a/spec/services/imports/organisation_import_service_spec.rb +++ b/spec/services/imports/organisation_import_service_spec.rb @@ -2,7 +2,8 @@ require "rails_helper" RSpec.describe Imports::OrganisationImportService do let(:storage_service) { instance_double(Storage::S3Service) } - let(:logger) { instance_double(Rails::Rack::Logger) } + let(:logs_string) { StringIO.new } + let(:logger) { MultiLogger.new(Rails.logger, Logger.new(logs_string)) } let(:folder_name) { "organisations" } let(:filenames) { %w[my_folder/my_file1.xml my_folder/my_file2.xml] } let(:fixture_directory) { "spec/fixtures/imports/institution" } @@ -81,10 +82,12 @@ RSpec.describe Imports::OrganisationImportService do it "successfully create an organisation the first time, and does not update it" do expect(storage_service).to receive(:list_files).with(folder_name).twice expect(storage_service).to receive(:get_file_io).with(filenames[0]).twice - expect(logger).to receive(:warn).once + expect(logger).to receive(:warn).once.and_call_original + expect(Rails.logger).to receive(:warn).with("Organisation my_new_organisation is already present with old visible ID 1, skipping.") expect { import_service.create_organisations(folder_name) }.to change(Organisation, :count).by(1) expect { import_service.create_organisations(folder_name) }.to change(Organisation, :count).by(0) + expect(logs_string.string).to include("Organisation my_new_organisation is already present with old visible ID 1, skipping.") expect(Organisation).to exist(old_visible_id: "1", name: "HA Ltd") end