Browse Source

Fix bug in DPC import script (#1723)

* Add test as a proof of the bug we have in prod

* Refactor script to create users with unique email

* Add rake task to import missing data protection confirmations
pull/1731/head
Jack 2 years ago committed by GitHub
parent
commit
677714c336
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 19
      app/services/imports/data_protection_confirmation_import_service.rb
  2. 23
      lib/tasks/data_import.rake
  3. 48
      spec/lib/tasks/data_import_spec.rb
  4. 7
      spec/services/imports/data_protection_confirmation_import_service_spec.rb

19
app/services/imports/data_protection_confirmation_import_service.rb

@ -8,6 +8,10 @@ module Imports
def create_data_protection_confirmation(xml_document) def create_data_protection_confirmation(xml_document)
org = Organisation.find_by(old_org_id: record_field_value(xml_document, "institution")) org = Organisation.find_by(old_org_id: record_field_value(xml_document, "institution"))
return log_org_must_exist if org.blank?
return log_dpc_already_present(org) if org.data_protection_confirmed?
dp_officer = User.find_by( dp_officer = User.find_by(
name: record_field_value(xml_document, "dp-user"), name: record_field_value(xml_document, "dp-user"),
organisation: org, organisation: org,
@ -20,6 +24,9 @@ module Imports
organisation: org, organisation: org,
is_dpo: true, is_dpo: true,
encrypted_password: SecureRandom.hex(10), encrypted_password: SecureRandom.hex(10),
email: SecureRandom.uuid,
confirmed_at: Time.zone.now,
active: false,
) )
dp_officer.save!(validate: false) dp_officer.save!(validate: false)
end end
@ -32,14 +39,18 @@ module Imports
old_org_id: record_field_value(xml_document, "institution"), old_org_id: record_field_value(xml_document, "institution"),
created_at: record_field_value(xml_document, "change-date").to_time(:utc), created_at: record_field_value(xml_document, "change-date").to_time(:utc),
) )
rescue ActiveRecord::RecordNotUnique
id = record_field_value(xml_document, "id")
dp_officer_name = record_field_value(xml_document, "dp-user")
@logger.warn("Data protection confirmation #{id} created by #{dp_officer_name} for #{org.name} is already present, skipping.")
end end
def record_field_value(xml_document, field) def record_field_value(xml_document, field)
field_value(xml_document, "dataprotect", field) field_value(xml_document, "dataprotect", field)
end end
def log_dpc_already_present(org)
@logger.warn("Data protection confirmation for #{org.name} with id #{org.id} is already present, skipping.")
end
def log_org_must_exist
@logger.error("Organisation must exist")
end
end end
end end

23
lib/tasks/data_import.rake

@ -28,4 +28,27 @@ namespace :core do
raise "Type #{type} is not supported by data_import" raise "Type #{type} is not supported by data_import"
end end
end end
desc "Import missing data sharing agreement XMLs from legacy CORE"
task :data_protection_confirmation_import, %i[type path] => :environment do |_task|
orgs_without_dpc = Organisation.includes(:data_protection_confirmation)
.where.not(id: DataProtectionConfirmation.all.select(:organisation_id))
.where.not(old_org_id: nil)
puts "Processing #{orgs_without_dpc.count} orgs without data protection confirmation"
s3_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
orgs_without_dpc.each do |org|
archive_path = "#{org.old_org_id}.zip"
archive_io = s3_service.get_file_io(archive_path)
archive_service = Storage::ArchiveService.new(archive_io)
if archive_service.folder_present?("dataprotect")
Rails.logger.info("Start importing folder dataprotect")
Imports::DataProtectionConfirmationImportService.new(archive_service).create_data_protection_confirmations("dataprotect")
Rails.logger.info("Imported")
else
Rails.logger.info("dataprotect does not exist, skipping import")
end
end
end
end end

48
spec/lib/tasks/data_import_spec.rb

@ -1,12 +1,13 @@
require "rails_helper" require "rails_helper"
require "rake" require "rake"
describe "rake core:data_import", type: :task do describe "data import", type: :task do
subject(:task) { Rake::Task["core:data_import"] }
let(:instance_name) { "paas_import_instance" } let(:instance_name) { "paas_import_instance" }
let(:storage_service) { instance_double(Storage::S3Service) }
let(:paas_config_service) { instance_double(Configuration::PaasConfigurationService) } let(:paas_config_service) { instance_double(Configuration::PaasConfigurationService) }
let(:storage_service) { instance_double(Storage::S3Service) }
describe "core:data_import" do
subject(:task) { Rake::Task["core:data_import"] }
before do before do
Rake.application.rake_require("tasks/data_import") Rake.application.rake_require("tasks/data_import")
@ -175,3 +176,42 @@ describe "rake core:data_import", type: :task do
expect { task.invoke("unknown_type", "my_path") }.to raise_error(/Type unknown_type is not supported/) expect { task.invoke("unknown_type", "my_path") }.to raise_error(/Type unknown_type is not supported/)
end end
end end
describe "core:data_protection_confirmation_import" do
subject(:task) { Rake::Task["core:data_protection_confirmation_import"] }
let(:import_service) { instance_double(Imports::DataProtectionConfirmationImportService) }
# let(:fixture_path) { "spec/fixtures/imports/data_protection_confirmations" }
let(:storage_service) { instance_double(Storage::S3Service) }
let(:archive_service) { instance_double(Storage::ArchiveService) }
let!(:organisation_without_dpc_with_old_org_id) { create(:organisation, :without_dpc, old_org_id: 1) }
before do
Rake.application.rake_require("tasks/data_import")
Rake::Task.define_task(:environment)
task.reenable
allow(Storage::S3Service).to receive(:new).and_return(storage_service)
allow(Configuration::PaasConfigurationService).to receive(:new).and_return(paas_config_service)
allow(ENV).to receive(:[])
allow(ENV).to receive(:[]).with("IMPORT_PAAS_INSTANCE").and_return(instance_name)
allow(Imports::DataProtectionConfirmationImportService).to receive(:new).and_return(import_service)
allow(Storage::ArchiveService).to receive(:new).and_return(archive_service)
allow(archive_service).to receive(:folder_present?).with("dataprotect").and_return(true)
_org_without_old_org_id = create(:organisation, old_org_id: nil)
_organisation_without_dp = create(:organisation, :without_dpc, old_org_id: nil)
end
it "creates an organisation from the given XML file" do
expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name)
expect(storage_service).to receive(:get_file_io).with("#{organisation_without_dpc_with_old_org_id.old_org_id}.zip")
expect(Imports::DataProtectionConfirmationImportService).to receive(:new).with(archive_service)
expect(import_service).to receive(:create_data_protection_confirmations).with("dataprotect")
task.invoke
end
end
end

7
spec/services/imports/data_protection_confirmation_import_service_spec.rb

@ -23,7 +23,7 @@ RSpec.describe Imports::DataProtectionConfirmationImportService do
context "when the organisation in the import file doesn't exist in the system" do context "when the organisation in the import file doesn't exist in the system" do
it "does not create a data protection confirmation" do it "does not create a data protection confirmation" do
expect(logger).to receive(:error).with(/Organisation must exist/) expect(logger).to receive(:error).with("Organisation must exist")
import_service.create_data_protection_confirmations("data_protection_directory") import_service.create_data_protection_confirmations("data_protection_directory")
end end
end end
@ -32,11 +32,12 @@ RSpec.describe Imports::DataProtectionConfirmationImportService do
let!(:organisation) { create(:organisation, :without_dpc, old_org_id:) } let!(:organisation) { create(:organisation, :without_dpc, old_org_id:) }
context "when a data protection officer with matching name does not exists for the organisation" do context "when a data protection officer with matching name does not exists for the organisation" do
it "creates a data protection officer without sign in credentials" do it "creates an inactive data protection officer" do
expect { import_service.create_data_protection_confirmations("data_protection_directory") } expect { import_service.create_data_protection_confirmations("data_protection_directory") }
.to change(User, :count).by(1) .to change(User, :count).by(1)
data_protection_officer = User.find_by(organisation:, is_dpo: true) data_protection_officer = User.find_by(organisation:, is_dpo: true)
expect(data_protection_officer.email).to eq("") expect(data_protection_officer.confirmed_at).not_to be_nil
expect(data_protection_officer.active).to be false
end end
it "successfully create a data protection confirmation record with the expected data" do it "successfully create a data protection confirmation record with the expected data" do

Loading…
Cancel
Save