baarkerlounger
/
submit-social-housing-lettings-and-sales-data
mirror of https://github.com/communitiesuk/submit-social-housing-lettings-and-sales-data.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Revert "CLDC-2602 Save import logs to S3 (#1829)" 

This reverts commit b93a6456d4.
revert-s3-errors
Kat 2 years ago
parent
7d6d17fe51
commit
a16c66295e
4 changed files with 20 additions and 115 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 17
      config/initializers/multi_logger.rb
  2. 49
      lib/tasks/full_import.rake
  3. 62
      spec/lib/tasks/full_import_spec.rb
  4. 7
      spec/services/imports/organisation_import_service_spec.rb

17
config/initializers/multi_logger.rb
Unescape View File

@ -1,17 +0,0 @@
class MultiLogger
def initialize(*targets)
@targets = targets
end
def info(data)
@targets.each { |t| t.info(data) }
end
def warn(data)
@targets.each { |t| t.warn(data) }
end
def error(data)
@targets.each { |t| t.error(data) }
end
end

49
lib/tasks/full_import.rake
Unescape View File

@ -1,4 +1,4 @@
Import = Struct.new("Import", :import_class, :import_method, :folder, :logger)
Import = Struct.new("Import", :import_class, :import_method, :folder)
namespace :import do
desc "Run initial import steps - orgs, schemes, users etc (without triggering user invite emails)"
@ -8,40 +8,34 @@ namespace :import do
s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
csv = CSV.parse(s3_service.get_file_io(institutions_csv_name), headers: true)
logs_string = StringIO.new
logger = MultiLogger.new(Rails.logger, Logger.new(logs_string))
org_count = csv.length
initial_import_list = [
Import.new(Imports::OrganisationImportService, :create_organisations, "institution", logger),
Import.new(Imports::SchemeImportService, :create_schemes, "mgmtgroups", logger),
Import.new(Imports::SchemeLocationImportService, :create_scheme_locations, "schemes", logger),
Import.new(Imports::UserImportService, :create_users, "user", logger),
Import.new(Imports::DataProtectionConfirmationImportService, :create_data_protection_confirmations, "dataprotect", logger),
Import.new(Imports::OrganisationRentPeriodImportService, :create_organisation_rent_periods, "rent-period", logger),
Import.new(Imports::OrganisationImportService, :create_organisations, "institution"),
Import.new(Imports::SchemeImportService, :create_schemes, "mgmtgroups"),
Import.new(Imports::SchemeLocationImportService, :create_scheme_locations, "schemes"),
Import.new(Imports::UserImportService, :create_users, "user"),
Import.new(Imports::DataProtectionConfirmationImportService, :create_data_protection_confirmations, "dataprotect"),
Import.new(Imports::OrganisationRentPeriodImportService, :create_organisation_rent_periods, "rent-period"),
]
logger.info("Beginning initial imports for #{org_count} organisations")
Rails.logger.info("Beginning initial imports for #{org_count} organisations")
csv.each do |row|
archive_path = row[1]
archive_io = s3_service.get_file_io(archive_path)
archive_service = Storage::ArchiveService.new(archive_io)
logger.info("Performing initial imports for organisation #{row[0]}")
Rails.logger.info("Performing initial imports for organisation #{row[0]}")
initial_import_list.each do |step|
if archive_service.folder_present?(step.folder)
step.import_class.new(archive_service, step.logger).send(step.import_method, step.folder)
step.import_class.new(archive_service).send(step.import_method, step.folder)
end
end
log_file = "#{File.basename(institutions_csv_name, File.extname(institutions_csv_name))}_#{File.basename(archive_path, File.extname(archive_path))}_initial.log"
s3_service.write_file(log_file, logs_string.string)
logs_string.rewind
logs_string.truncate(0)
end
logger.info("Finished initial imports")
Rails.logger.info("Finished initial imports")
end
desc "Run logs import steps"
@ -52,15 +46,13 @@ namespace :import do
s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
csv = CSV.parse(s3_service.get_file_io(institutions_csv_name), headers: true)
org_count = csv.length
logs_string = StringIO.new
logger = MultiLogger.new(Rails.logger, Logger.new(logs_string))
logs_import_list = [
Import.new(Imports::LettingsLogsImportService, :create_logs, "logs", logger),
Import.new(Imports::SalesLogsImportService, :create_logs, "logs", logger),
Import.new(Imports::LettingsLogsImportService, :create_logs, "logs"),
Import.new(Imports::SalesLogsImportService, :create_logs, "logs"),
]
logger.info("Beginning log imports for #{org_count} organisations")
Rails.logger.info("Beginning log imports for #{org_count} organisations")
csv.each do |row|
archive_path = row[1]
@ -68,21 +60,16 @@ namespace :import do
archive_service = Storage::ArchiveService.new(archive_io)
log_count = row[2].to_i + row[3].to_i + row[4].to_i + row[5].to_i
logger.info("Importing logs for organisation #{row[0]}, expecting #{log_count} logs")
Rails.logger.info("Importing logs for organisation #{row[0]}, expecting #{log_count} logs")
logs_import_list.each do |step|
if archive_service.folder_present?(step.folder)
step.import_class.new(archive_service, step.logger).send(step.import_method, step.folder)
step.import_class.new(archive_service).send(step.import_method, step.folder)
end
end
log_file = "#{File.basename(institutions_csv_name, File.extname(institutions_csv_name))}_#{File.basename(archive_path, File.extname(archive_path))}_logs.log"
s3_service.write_file(log_file, logs_string.string)
logs_string.rewind
logs_string.truncate(0)
end
logger.info("Log import complete")
Rails.logger.info("Log import complete")
end
desc "Trigger invite emails for imported users"

62
spec/lib/tasks/full_import_spec.rb
Unescape View File

@ -53,66 +53,4 @@ describe "full import", type: :task do
end
end
end
describe "import:initial" do
subject(:task) { Rake::Task["import:initial"] }
let(:archive_service) { instance_double(Storage::ArchiveService) }
before do
Rake.application.rake_require("tasks/full_import")
Rake::Task.define_task(:environment)
task.reenable
end
context "when calling the initial import" do
before do
allow(Storage::ArchiveService).to receive(:new).and_return(archive_service)
allow(archive_service).to receive(:folder_present?).and_return(false)
allow(Imports::OrganisationImportService).to receive(:new).and_return(instance_double(Imports::OrganisationImportService))
allow(Imports::SchemeImportService).to receive(:new).and_return(instance_double(Imports::SchemeImportService))
allow(Imports::SchemeLocationImportService).to receive(:new).and_return(instance_double(Imports::SchemeLocationImportService))
allow(Imports::UserImportService).to receive(:new).and_return(instance_double(Imports::UserImportService))
allow(Imports::DataProtectionConfirmationImportService).to receive(:new).and_return(instance_double(Imports::DataProtectionConfirmationImportService))
allow(Imports::OrganisationRentPeriodImportService).to receive(:new).and_return(instance_double(Imports::OrganisationRentPeriodImportService))
end
it "creates a report using given organisation csv" do
expect(Storage::S3Service).to receive(:new).with(env_config_service, instance_name)
expect(storage_service).to receive(:write_file).with("some_name_1_initial.log", / INFO -- : Performing initial imports for organisation org1/)
expect(storage_service).to receive(:write_file).with("some_name_2_initial.log", / INFO -- : Performing initial imports for organisation org2/)
task.invoke("some_name.csv")
end
end
end
describe "import:logs" do
subject(:task) { Rake::Task["import:logs"] }
let(:archive_service) { instance_double(Storage::ArchiveService) }
before do
Rake.application.rake_require("tasks/full_import")
Rake::Task.define_task(:environment)
task.reenable
end
context "when calling the logs import" do
before do
allow(Storage::ArchiveService).to receive(:new).and_return(archive_service)
allow(archive_service).to receive(:folder_present?).and_return(false)
allow(Imports::LettingsLogsImportService).to receive(:new).and_return(instance_double(Imports::LettingsLogsImportService))
allow(Imports::SalesLogsImportService).to receive(:new).and_return(instance_double(Imports::SalesLogsImportService))
end
it "creates a report using given organisation csv" do
expect(Storage::S3Service).to receive(:new).with(env_config_service, instance_name)
expect(storage_service).to receive(:write_file).with("some_name_1_logs.log", / INFO -- : Importing logs for organisation org1, expecting 0 logs/)
expect(storage_service).to receive(:write_file).with("some_name_2_logs.log", / INFO -- : Importing logs for organisation org2, expecting 0 logs/)
task.invoke("some_name.csv")
end
end
end
end

7
spec/services/imports/organisation_import_service_spec.rb
Unescape View File

@ -2,8 +2,7 @@ require "rails_helper"
RSpec.describe Imports::OrganisationImportService do
let(:storage_service) { instance_double(Storage::S3Service) }
let(:logs_string) { StringIO.new }
let(:logger) { MultiLogger.new(Rails.logger, Logger.new(logs_string)) }
let(:logger) { instance_double(Rails::Rack::Logger) }
let(:folder_name) { "organisations" }
let(:filenames) { %w[my_folder/my_file1.xml my_folder/my_file2.xml] }
let(:fixture_directory) { "spec/fixtures/imports/institution" }
@ -82,12 +81,10 @@ RSpec.describe Imports::OrganisationImportService do
it "successfully create an organisation the first time, and does not update it" do
expect(storage_service).to receive(:list_files).with(folder_name).twice
expect(storage_service).to receive(:get_file_io).with(filenames[0]).twice
expect(logger).to receive(:warn).once.and_call_original
expect(Rails.logger).to receive(:warn).with("Organisation my_new_organisation is already present with old visible ID 1, skipping.")
expect(logger).to receive(:warn).once
expect { import_service.create_organisations(folder_name) }.to change(Organisation, :count).by(1)
expect { import_service.create_organisations(folder_name) }.to change(Organisation, :count).by(0)
expect(logs_string.string).to include("Organisation my_new_organisation is already present with old visible ID 1, skipping.")
expect(Organisation).to exist(old_visible_id: "1", name: "HA Ltd")
end

Write Preview
Loading…
Cancel
Save