CLDC-2621 Flag orgs without coordinators (#1818)

* Add import report service

* Call ImportReportService from rake task

* Move generate_logs_report to import service

* Add organisations without active data coordinators to the report

* pluralize method names

* update suffixes

app/services/imports/import_report_service.rb

@@ -0,0 +1,57 @@
+module Imports
+  class ImportReportService
+    def initialize(storage_service, institutions_csv, logger = Rails.logger)
+      @storage_service = storage_service
+      @logger = logger
+      @institutions_csv = institutions_csv
+    end
+
+    BYTE_ORDER_MARK = "\uFEFF".freeze # Required to ensure Excel always reads CSV as UTF-8
+
+    def create_reports(report_suffix)
+      generate_missing_data_coordinators_report(report_suffix)
+      generate_logs_report(report_suffix)
+    end
+
+    def generate_missing_data_coordinators_report(report_suffix)
+      report_csv = "Organisation ID,Old Organisation ID,Organisation Name\n"
+      organisations = @institutions_csv.map { |row| Organisation.find_by(name: row[0]) }.compact
+      organisations.each do |organisation|
+        if organisation.users.none? { |user| user.data_coordinator? && user.active? }
+          report_csv += "#{organisation.id},#{organisation.old_visible_id},#{organisation.name}\n"
+        end
+      end
+
+      report_name = "OrganisationsWithoutDataCoordinators_#{report_suffix}"
+      @storage_service.write_file(report_name, BYTE_ORDER_MARK + report_csv)
+
+      @logger.info("Missing data coordinators report available in s3 import bucket at #{report_name}")
+    end
+
+    def generate_logs_report(report_suffix)
+      Rails.logger.info("Generating migrated logs report")
+
+      rep = CSV.generate do |report|
+        headers = ["Institution name", "Id", "Old Completed lettings logs", "Old In progress lettings logs", "Old Completed sales logs", "Old In progress sales logs", "New Completed lettings logs", "New In Progress lettings logs", "New Completed sales logs", "New In Progress sales logs"]
+        report << headers
+
+        @institutions_csv.each do |row|
+          name = row[0]
+          organisation = Organisation.find_by(name:)
+          next unless organisation
+
+          completed_sales_logs = organisation.owned_sales_logs.where(status: "completed").count
+          in_progress_sales_logs = organisation.owned_sales_logs.where(status: "in_progress").count
+          completed_lettings_logs = organisation.owned_lettings_logs.where(status: "completed").count
+          in_progress_lettings_logs = organisation.owned_lettings_logs.where(status: "in_progress").count
+          report << row.push(completed_lettings_logs, in_progress_lettings_logs, completed_sales_logs, in_progress_sales_logs)
+        end
+      end
+
+      report_name = "MigratedLogsReport_#{report_suffix}"
+      @storage_service.write_file(report_name, BYTE_ORDER_MARK + rep)
+
+      @logger.info("Logs report available in s3 import bucket at #{report_name}")
+    end
+  end
+end

lib/tasks/full_import.rake

@@ -94,41 +94,19 @@ namespace :import do
   end
 
   desc "Generate migrated logs report"
-  task :generate_report, %i[institutions_csv_name] => :environment do |_task, args|
+  task :generate_reports, %i[institutions_csv_name] => :environment do |_task, args|
     institutions_csv_name = args[:institutions_csv_name]
-    raise "Usage: rake import:generate_report['institutions_csv_name']" if institutions_csv_name.blank?
+    raise "Usage: rake import:generate_reports['institutions_csv_name']" if institutions_csv_name.blank?
 
     s3_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
-    csv = CSV.parse(s3_service.get_file_io(institutions_csv_name), headers: true)
+    institutions_csv = CSV.parse(s3_service.get_file_io(institutions_csv_name), headers: true)
-
-    Rails.logger.info("Generating migrated logs report")
-
-    rep = CSV.generate do |report|
-      headers = ["Institution name", "Id", "Old Completed lettings logs", "Old In progress lettings logs", "Old Completed sales logs", "Old In progress sales logs", "New Completed lettings logs", "New In Progress lettings logs", "New Completed sales logs", "New In Progress sales logs"]
-      report << headers
-
-      csv.each do |row|
-        name = row[0]
-        organisation = Organisation.find_by(name:)
-        next unless organisation
-
-        completed_sales_logs = organisation.owned_sales_logs.where(status: "completed").count
-        in_progress_sales_logs = organisation.owned_sales_logs.where(status: "in_progress").count
-        completed_lettings_logs = organisation.owned_lettings_logs.where(status: "completed").count
-        in_progress_lettings_logs = organisation.owned_lettings_logs.where(status: "in_progress").count
-        report << row.push(completed_lettings_logs, in_progress_lettings_logs, completed_sales_logs, in_progress_sales_logs)
-      end
-    end
-
-    report_name = "MigratedLogsReport_#{institutions_csv_name}"
-    s3_service.write_file(report_name, rep)
 
-    Rails.logger.info("Logs report available in s3 import bucket at #{report_name}")
+    Imports::ImportReportService.new(s3_service, institutions_csv).create_reports(institutions_csv_name)
   end
 
   desc "Run import from logs step to end"
-  task :logs_onwards, %i[institutions_csv_name] => %i[environment logs trigger_invites generate_report]
+  task :logs_onwards, %i[institutions_csv_name] => %i[environment logs trigger_invites generate_reports]
 
   desc "Run a full import for the institutions listed in the named file on s3"
-  task :full, %i[institutions_csv_name] => %i[environment initial logs trigger_invites generate_report]
+  task :full, %i[institutions_csv_name] => %i[environment initial logs trigger_invites generate_reports]
 end

spec/lib/tasks/full_import_spec.rb

@@ -0,0 +1,44 @@
+require "rails_helper"
+require "rake"
+
+describe "full import", type: :task do
+  let(:instance_name) { "paas_import_instance" }
+  let(:paas_config_service) { instance_double(Configuration::PaasConfigurationService) }
+  let(:storage_service) { instance_double(Storage::S3Service) }
+  let(:orgs_list) { "Institution name,Id,Old Completed lettings logs,Old In progress lettings logs,Old Completed sales logs,Old In progress sales logs\norg1,1.zip,0,0,0,0\norg2,2.zip,0,0,0,0" }
+
+  before do
+    allow(Storage::S3Service).to receive(:new).and_return(storage_service)
+    allow(storage_service).to receive(:write_file).and_return(nil)
+    allow(storage_service).to receive(:get_file_io).and_return(orgs_list)
+    allow(Configuration::PaasConfigurationService).to receive(:new).and_return(paas_config_service)
+    allow(ENV).to receive(:[])
+    allow(ENV).to receive(:[]).with("IMPORT_PAAS_INSTANCE").and_return(instance_name)
+  end
+
+  describe "import:generate_reports" do
+    subject(:task) { Rake::Task["import:generate_reports"] }
+
+    before do
+      Rake.application.rake_require("tasks/full_import")
+      Rake::Task.define_task(:environment)
+      task.reenable
+    end
+
+    context "when generating report" do
+      let(:import_report_service) { instance_double(Imports::ImportReportService) }
+
+      before do
+        allow(Imports::ImportReportService).to receive(:new).and_return(import_report_service)
+      end
+
+      it "creates a report using given organisation csv" do
+        expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name)
+        expect(Imports::ImportReportService).to receive(:new).with(storage_service, CSV.parse(orgs_list, headers: true))
+        expect(import_report_service).to receive(:create_reports).with("some_name")
+
+        task.invoke("some_name")
+      end
+    end
+  end
+end

spec/services/imports/import_report_service_spec.rb

@@ -0,0 +1,70 @@
+require "rails_helper"
+
+RSpec.describe Imports::ImportReportService do
+  subject(:report_service) { described_class.new(storage_service, institutions_csv) }
+
+  let(:storage_service) { instance_double(Storage::S3Service) }
+
+  describe "#generate_missing_data_coordinators_report" do
+    context "when all organisations have data coordinators" do
+      let!(:organisation) { create(:organisation, old_visible_id: "1", name: "org1") }
+      let(:institutions_csv) { CSV.parse("Institution name,Id,Old Completed lettings logs,Old In progress lettings logs,Old Completed sales logs,Old In progress sales logs\norg1,1,2,1,4,3", headers: true) }
+
+      before do
+        create(:user, :data_coordinator, organisation:)
+      end
+
+      it "writes an empty organisations without a data coordinators report" do
+        expect(storage_service).to receive(:write_file).with("OrganisationsWithoutDataCoordinators_report_suffix.csv", "\uFEFFOrganisation ID,Old Organisation ID,Organisation Name\n")
+
+        report_service.generate_missing_data_coordinators_report("report_suffix.csv")
+      end
+    end
+
+    context "when some organisations have no data coordinators" do
+      let!(:organisation) { create(:organisation, old_visible_id: "1", name: "org1") }
+      let!(:organisation2) { create(:organisation, old_visible_id: "2", name: "org2") }
+      let!(:organisation3) { create(:organisation, old_visible_id: "3", name: "org3") }
+      let(:institutions_csv) { CSV.parse("Institution name,Id,Old Completed lettings logs,Old In progress lettings logs,Old Completed sales logs,Old In progress sales logs\norg1,1,2,1,4,3\norg2,2,5,6,5,7\norg3,3,5,6,5,7", headers: true) }
+
+      before do
+        create(:user, :data_coordinator, organisation:)
+      end
+
+      it "writes an empty organisations without a data coordinators report" do
+        expect(storage_service).to receive(:write_file).with("OrganisationsWithoutDataCoordinators_report_suffix.csv", "\uFEFFOrganisation ID,Old Organisation ID,Organisation Name\n#{organisation2.id},2,org2\n#{organisation3.id},3,org3\n")
+
+        report_service.generate_missing_data_coordinators_report("report_suffix.csv")
+      end
+    end
+
+    context "when organisation has an inactive data coordinator" do
+      let!(:organisation) { create(:organisation, old_visible_id: "1", name: "org1") }
+      let(:institutions_csv) { CSV.parse("Institution name,Id,Old Completed lettings logs,Old In progress lettings logs,Old Completed sales logs,Old In progress sales logs\norg1,1,2,1,4,3", headers: true) }
+
+      before do
+        create(:user, :data_coordinator, organisation:, active: false)
+      end
+
+      it "includes that organisation in the data coordinators report" do
+        expect(storage_service).to receive(:write_file).with("OrganisationsWithoutDataCoordinators_report_suffix.csv", "\uFEFFOrganisation ID,Old Organisation ID,Organisation Name\n#{organisation.id},1,org1\n")
+
+        report_service.generate_missing_data_coordinators_report("report_suffix.csv")
+      end
+    end
+  end
+
+  describe "#generate_logs_report" do
+    let(:institutions_csv) { CSV.parse("Institution name,Id,Old Completed lettings logs,Old In progress lettings logs,Old Completed sales logs,Old In progress sales logs\norg1,1,2,1,4,3\norg2,2,5,6,5,7", headers: true) }
+
+    before do
+      create(:organisation, old_visible_id: "1", name: "org1")
+      create(:organisation, old_visible_id: "2", name: "org2")
+    end
+
+    it "generates a report with imported logs" do
+      expect(storage_service).to receive(:write_file).with("MigratedLogsReport_report_suffix.csv", "\uFEFFInstitution name,Id,Old Completed lettings logs,Old In progress lettings logs,Old Completed sales logs,Old In progress sales logs,New Completed lettings logs,New In Progress lettings logs,New Completed sales logs,New In Progress sales logs\norg1,1,2,1,4,3,0,0,0,0\norg2,2,5,6,5,7,0,0,0,0\n")
+      report_service.generate_logs_report("report_suffix.csv")
+    end
+  end
+end