CLDC-2602 Save import logs to S3 (#1829)

* Add logger that writes to s3 file

* Add logger for logs import

* remove unused method

* Update organisation import service spec

* Partition log files per org

config/initializers/multi_logger.rb

@@ -0,0 +1,17 @@
+class MultiLogger
+  def initialize(*targets)
+    @targets = targets
+  end
+
+  def info(data)
+    @targets.each { |t| t.info(data) }
+  end
+
+  def warn(data)
+    @targets.each { |t| t.warn(data) }
+  end
+
+  def error(data)
+    @targets.each { |t| t.error(data) }
+  end
+end

lib/tasks/full_import.rake

@@ -1,4 +1,4 @@
-Import = Struct.new("Import", :import_class, :import_method, :folder)
+Import = Struct.new("Import", :import_class, :import_method, :folder, :logger)
 
 namespace :import do
   desc "Run initial import steps - orgs, schemes, users etc (without triggering user invite emails)"
@@ -8,34 +8,40 @@ namespace :import do
 
     s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
     csv = CSV.parse(s3_service.get_file_io(institutions_csv_name), headers: true)
+    logs_string = StringIO.new
+    logger = MultiLogger.new(Rails.logger, Logger.new(logs_string))
     org_count = csv.length
 
     initial_import_list = [
-      Import.new(Imports::OrganisationImportService, :create_organisations, "institution"),
+      Import.new(Imports::OrganisationImportService, :create_organisations, "institution", logger),
-      Import.new(Imports::SchemeImportService, :create_schemes, "mgmtgroups"),
+      Import.new(Imports::SchemeImportService, :create_schemes, "mgmtgroups", logger),
-      Import.new(Imports::SchemeLocationImportService, :create_scheme_locations, "schemes"),
+      Import.new(Imports::SchemeLocationImportService, :create_scheme_locations, "schemes", logger),
-      Import.new(Imports::UserImportService, :create_users, "user"),
+      Import.new(Imports::UserImportService, :create_users, "user", logger),
-      Import.new(Imports::DataProtectionConfirmationImportService, :create_data_protection_confirmations, "dataprotect"),
+      Import.new(Imports::DataProtectionConfirmationImportService, :create_data_protection_confirmations, "dataprotect", logger),
-      Import.new(Imports::OrganisationRentPeriodImportService, :create_organisation_rent_periods, "rent-period"),
+      Import.new(Imports::OrganisationRentPeriodImportService, :create_organisation_rent_periods, "rent-period", logger),
     ]
 
-    Rails.logger.info("Beginning initial imports for #{org_count} organisations")
+    logger.info("Beginning initial imports for #{org_count} organisations")
 
     csv.each do |row|
       archive_path = row[1]
       archive_io = s3_service.get_file_io(archive_path)
       archive_service = Storage::ArchiveService.new(archive_io)
-
+      logger.info("Performing initial imports for organisation #{row[0]}")
-      Rails.logger.info("Performing initial imports for organisation #{row[0]}")
 
       initial_import_list.each do |step|
         if archive_service.folder_present?(step.folder)
-          step.import_class.new(archive_service).send(step.import_method, step.folder)
+          step.import_class.new(archive_service).send(step.import_method, step.folder, step.logger)
         end
       end
+
+      log_file = "#{File.basename(institutions_csv_name, File.extname(institutions_csv_name))}_#{File.basename(archive_path, File.extname(archive_path))}_initial.log"
+      s3_service.write_file(log_file, logs_string.string)
+      logs_string.rewind
+      logs_string.truncate(0)
     end
 
-    Rails.logger.info("Finished initial imports")
+    logger.info("Finished initial imports")
   end
 
   desc "Run logs import steps"
@@ -46,13 +52,15 @@ namespace :import do
     s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
     csv = CSV.parse(s3_service.get_file_io(institutions_csv_name), headers: true)
     org_count = csv.length
+    logs_string = StringIO.new
+    logger = MultiLogger.new(Rails.logger, Logger.new(logs_string))
 
     logs_import_list = [
-      Import.new(Imports::LettingsLogsImportService, :create_logs, "logs"),
+      Import.new(Imports::LettingsLogsImportService, :create_logs, "logs", logger),
-      Import.new(Imports::SalesLogsImportService, :create_logs, "logs"),
+      Import.new(Imports::SalesLogsImportService, :create_logs, "logs", logger),
     ]
 
-    Rails.logger.info("Beginning log imports for #{org_count} organisations")
+    logger.info("Beginning log imports for #{org_count} organisations")
 
     csv.each do |row|
       archive_path = row[1]
@@ -60,16 +68,21 @@ namespace :import do
       archive_service = Storage::ArchiveService.new(archive_io)
 
       log_count = row[2].to_i + row[3].to_i + row[4].to_i + row[5].to_i
-      Rails.logger.info("Importing logs for organisation #{row[0]}, expecting #{log_count} logs")
+      logger.info("Importing logs for organisation #{row[0]}, expecting #{log_count} logs")
 
       logs_import_list.each do |step|
         if archive_service.folder_present?(step.folder)
-          step.import_class.new(archive_service).send(step.import_method, step.folder)
+          step.import_class.new(archive_service).send(step.import_method, step.folder, step.logger)
         end
       end
+
+      log_file = "#{File.basename(institutions_csv_name, File.extname(institutions_csv_name))}_#{File.basename(archive_path, File.extname(archive_path))}_logs.log"
+      s3_service.write_file(log_file, logs_string.string)
+      logs_string.rewind
+      logs_string.truncate(0)
     end
 
-    Rails.logger.info("Log import complete")
+    logger.info("Log import complete")
   end
 
   desc "Trigger invite emails for imported users"

spec/lib/tasks/full_import_spec.rb

@@ -53,4 +53,66 @@ describe "full import", type: :task do
       end
     end
   end
+
+  describe "import:initial" do
+    subject(:task) { Rake::Task["import:initial"] }
+
+    let(:archive_service) { instance_double(Storage::ArchiveService) }
+
+    before do
+      Rake.application.rake_require("tasks/full_import")
+      Rake::Task.define_task(:environment)
+      task.reenable
+    end
+
+    context "when calling the initial import" do
+      before do
+        allow(Storage::ArchiveService).to receive(:new).and_return(archive_service)
+        allow(archive_service).to receive(:folder_present?).and_return(false)
+        allow(Imports::OrganisationImportService).to receive(:new).and_return(instance_double(Imports::OrganisationImportService))
+        allow(Imports::SchemeImportService).to receive(:new).and_return(instance_double(Imports::SchemeImportService))
+        allow(Imports::SchemeLocationImportService).to receive(:new).and_return(instance_double(Imports::SchemeLocationImportService))
+        allow(Imports::UserImportService).to receive(:new).and_return(instance_double(Imports::UserImportService))
+        allow(Imports::DataProtectionConfirmationImportService).to receive(:new).and_return(instance_double(Imports::DataProtectionConfirmationImportService))
+        allow(Imports::OrganisationRentPeriodImportService).to receive(:new).and_return(instance_double(Imports::OrganisationRentPeriodImportService))
+      end
+
+      it "creates a report using given organisation csv" do
+        expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name)
+        expect(storage_service).to receive(:write_file).with("some_name_1_initial.log", / INFO -- : Performing initial imports for organisation org1/)
+        expect(storage_service).to receive(:write_file).with("some_name_2_initial.log", / INFO -- : Performing initial imports for organisation org2/)
+
+        task.invoke("some_name.csv")
+      end
+    end
+  end
+
+  describe "import:logs" do
+    subject(:task) { Rake::Task["import:logs"] }
+
+    let(:archive_service) { instance_double(Storage::ArchiveService) }
+
+    before do
+      Rake.application.rake_require("tasks/full_import")
+      Rake::Task.define_task(:environment)
+      task.reenable
+    end
+
+    context "when calling the logs import" do
+      before do
+        allow(Storage::ArchiveService).to receive(:new).and_return(archive_service)
+        allow(archive_service).to receive(:folder_present?).and_return(false)
+        allow(Imports::LettingsLogsImportService).to receive(:new).and_return(instance_double(Imports::LettingsLogsImportService))
+        allow(Imports::SalesLogsImportService).to receive(:new).and_return(instance_double(Imports::SalesLogsImportService))
+      end
+
+      it "creates a report using given organisation csv" do
+        expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name)
+        expect(storage_service).to receive(:write_file).with("some_name_1_logs.log", / INFO -- : Importing logs for organisation org1, expecting 0 logs/)
+        expect(storage_service).to receive(:write_file).with("some_name_2_logs.log", / INFO -- : Importing logs for organisation org2, expecting 0 logs/)
+
+        task.invoke("some_name.csv")
+      end
+    end
+  end
 end

spec/services/imports/organisation_import_service_spec.rb

@@ -2,7 +2,8 @@ require "rails_helper"
 
 RSpec.describe Imports::OrganisationImportService do
   let(:storage_service) { instance_double(Storage::S3Service) }
-  let(:logger) { instance_double(Rails::Rack::Logger) }
+  let(:logs_string) { StringIO.new }
+  let(:logger) { MultiLogger.new(Rails.logger, Logger.new(logs_string)) }
   let(:folder_name) { "organisations" }
   let(:filenames) { %w[my_folder/my_file1.xml my_folder/my_file2.xml] }
   let(:fixture_directory) { "spec/fixtures/imports/institution" }
@@ -81,10 +82,12 @@ RSpec.describe Imports::OrganisationImportService do
     it "successfully create an organisation the first time, and does not update it" do
       expect(storage_service).to receive(:list_files).with(folder_name).twice
       expect(storage_service).to receive(:get_file_io).with(filenames[0]).twice
-      expect(logger).to receive(:warn).once
+      expect(logger).to receive(:warn).once.and_call_original
+      expect(Rails.logger).to receive(:warn).with("Organisation my_new_organisation is already present with old visible ID 1, skipping.")
 
       expect { import_service.create_organisations(folder_name) }.to change(Organisation, :count).by(1)
       expect { import_service.create_organisations(folder_name) }.to change(Organisation, :count).by(0)
+      expect(logs_string.string).to include("Organisation my_new_organisation is already present with old visible ID 1, skipping.")
 
       expect(Organisation).to exist(old_visible_id: "1", name: "HA Ltd")
     end