CLDC-2837 Update import for washup migration 2 (#1948)

* feat: update rake tasks for washup batch 2

* feat: update rake tasks for washup batch 2 and add tests

* feat: revert unrelated test change

* feat: add dpo check before importing user

app/services/imports/import_service.rb

@@ -22,6 +22,12 @@ module Imports
       end
     end
 
+    def dataprotect_xml
+      data_protect_filename = @storage_service.list_files("dataprotect").first
+      file_io = @storage_service.get_file_io(data_protect_filename)
+      Nokogiri::XML(file_io)
+    end
+
     def field_value(xml_document, namespace, field, *args)
       if namespace.present?
         xml_document.at_xpath("//#{namespace}:#{field}", *args)&.text

app/services/imports/user_import_service.rb

@@ -4,6 +4,10 @@ module Imports
       import_from(folder, :create_user)
     end
 
+    def create_users_who_signed_dpcs(folder)
+      import_from(folder, :create_user_who_signed_dpc)
+    end
+
   private
 
     PROVIDER_TYPE = {
@@ -51,6 +55,14 @@ module Imports
       end
     end
 
+    def create_user_who_signed_dpc(xml_document)
+      dpc_signer_name = field_value(dataprotect_xml, "dataprotect", "dp-user")
+      name = user_field_value(xml_document, "full-name")
+      if name == dpc_signer_name && is_dpo?(user_field_value(xml_document, "user-type"))
+        create_user(xml_document)
+      end
+    end
+
     def user_field_value(xml_document, field)
       field_value(xml_document, "user", field, { "user" => "dclg:user" })
     end

lib/tasks/full_import.rake

@@ -44,6 +44,47 @@ namespace :import do
     logger.info("Finished initial imports")
   end
 
+  desc "Specific subset of object imports for washup batch 2"
+  task :washup_imports, %i[institutions_csv_name] => :environment do |_task, args|
+    institutions_csv_name = args[:institutions_csv_name]
+    raise "Usage: rake import:washup_imports['institutions_csv_name']" if institutions_csv_name.blank?
+
+    s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
+    csv = CSV.parse(s3_service.get_file_io(institutions_csv_name), headers: true)
+    logs_string = StringIO.new
+    logger = MultiLogger.new(Logger.new(logs_string))
+    org_count = csv.length
+
+    initial_import_list = [
+      Import.new(Imports::OrganisationImportService, :create_organisations, "institution", logger),
+      Import.new(Imports::UserImportService, :create_users_who_signed_dpcs, "user", logger),
+      Import.new(Imports::DataProtectionConfirmationImportService, :create_data_protection_confirmations, "dataprotect", logger),
+      Import.new(Imports::OrganisationRentPeriodImportService, :create_organisation_rent_periods, "rent-period", logger),
+    ]
+
+    logger.info("Beginning imports for #{org_count} organisations")
+
+    csv.each do |row|
+      archive_path = row[1]
+      archive_io = s3_service.get_file_io(archive_path)
+      archive_service = Storage::ArchiveService.new(archive_io)
+      logger.info("Performing imports for organisation #{row[0]}")
+
+      initial_import_list.each do |step|
+        if archive_service.folder_present?(step.folder)
+          step.import_class.new(archive_service, step.logger).send(step.import_method, step.folder)
+        end
+      end
+
+      log_file = "#{File.basename(institutions_csv_name, File.extname(institutions_csv_name))}_#{File.basename(archive_path, File.extname(archive_path))}_initial.log"
+      s3_service.write_file(log_file, logs_string.string) if logs_string.string.present?
+      logs_string.rewind
+      logs_string.truncate(0)
+    end
+
+    logger.info("Finished washup imports")
+  end
+
   desc "Run logs import steps"
   task :logs, %i[institutions_csv_name] => :environment do |_task, args|
     institutions_csv_name = args[:institutions_csv_name]

spec/services/imports/user_import_service_spec.rb

@@ -1,10 +1,10 @@
 require "rails_helper"
 
 RSpec.describe Imports::UserImportService do
-  let(:fixture_directory) { "spec/fixtures/imports/user" }
+  let(:fixture_directory) { "spec/fixtures/imports" }
   let(:old_user_id) { "fc7625a02b24ae16162aa63ae7cb33feeec0c373" }
   let(:old_org_id) { "7c5bd5fb549c09a2c55d7cb90d7ba84927e64618" }
-  let(:user_file) { File.open("#{fixture_directory}/#{old_user_id}.xml") }
+  let(:user_file) { File.open("#{fixture_directory}/user/#{old_user_id}.xml") }
   let(:storage_service) { instance_double(Storage::S3Service) }
   let(:logger) { instance_double(ActiveSupport::Logger) }
   let(:notify_client) { instance_double(Notifications::Client) }
@@ -21,6 +21,7 @@ RSpec.describe Imports::UserImportService do
 
     before do
       allow(storage_service).to receive(:list_files)
+                                  .with("user_directory")
                                   .and_return(["user_directory/#{old_user_id}.xml"])
       allow(storage_service).to receive(:get_file_io)
                                   .with("user_directory/#{old_user_id}.xml")
@@ -201,5 +202,63 @@ RSpec.describe Imports::UserImportService do
         end
       end
     end
+
+    context "when only creating specific dsa signers" do
+      let(:old_dataprotect_id) { "7c5bd5fb549c09a2c55d7cb90d7ba84927e64618" }
+      let(:dataprotect_file) { File.open("#{fixture_directory}/dataprotect/#{old_dataprotect_id}.xml") }
+
+      before do
+        allow(storage_service).to receive(:list_files)
+                                    .with("dataprotect")
+                                    .and_return(["dataprotect_directory/#{old_dataprotect_id}.xml"])
+        allow(storage_service).to receive(:get_file_io)
+                                    .with("dataprotect_directory/#{old_dataprotect_id}.xml")
+                                    .and_return(dataprotect_file)
+        allow(logger).to receive(:info)
+      end
+
+      context "when the user is to be imported" do
+        let(:old_user_id) { "10c887710550844e2551b3e0fb88dc9b4a8a642b" }
+
+        it "creates a user with the correct details" do
+          FactoryBot.create(:organisation, old_org_id:)
+          import_service.create_users_who_signed_dpcs("user_directory")
+
+          user = LegacyUser.find_by(old_user_id:)&.user
+          expect(user.name).to eq("John Doe")
+          expect(user.email).to eq("john.doe@gov.uk")
+          expect(user.encrypted_password).not_to be_nil
+          expect(user.phone).to eq("02012345678")
+          expect(user.is_data_protection_officer?).to be true
+          expect(user.organisation.old_org_id).to eq(old_org_id)
+          expect(user.is_key_contact?).to be false
+          expect(user.active).to be true
+        end
+      end
+
+      context "when the user has a different name to the dsa signer" do
+        let(:old_user_id) { "80d9b73aa1c88b6e5c36ee49be9050b923b4a1bb" }
+
+        it "does not create a user" do
+          FactoryBot.create(:organisation, old_org_id:)
+          import_service.create_users_who_signed_dpcs("user_directory")
+
+          user = LegacyUser.find_by(old_user_id:)&.user
+          expect(user).to be_nil
+        end
+      end
+
+      context "when the user has the name of the dsa signer but is not a dpo" do
+        let(:old_user_id) { "9ed81a262215a1634f0809effa683e38924d8bcb" }
+
+        it "does not create a user" do
+          FactoryBot.create(:organisation, old_org_id:)
+          import_service.create_users_who_signed_dpcs("user_directory")
+
+          user = LegacyUser.find_by(old_user_id:)&.user
+          expect(user).to be_nil
+        end
+      end
+    end
   end
 end