From 8a3f18d3acc24750654cdcd94fae7f0ec4e4632f Mon Sep 17 00:00:00 2001 From: kosiakkatrina <54268893+kosiakkatrina@users.noreply.github.com> Date: Wed, 9 Aug 2023 09:45:08 +0100 Subject: [PATCH] CLDC-2621 Flag orgs without coordinators (#1818) * Add import report service * Call ImportReportService from rake task * Move generate_logs_report to import service * Add organisations without active data coordinators to the report * pluralize method names * update suffixes --- app/services/imports/import_report_service.rb | 57 +++++++++++++++ lib/tasks/full_import.rake | 34 ++------- spec/lib/tasks/full_import_spec.rb | 44 ++++++++++++ .../imports/import_report_service_spec.rb | 70 +++++++++++++++++++ 4 files changed, 177 insertions(+), 28 deletions(-) create mode 100644 app/services/imports/import_report_service.rb create mode 100644 spec/lib/tasks/full_import_spec.rb create mode 100644 spec/services/imports/import_report_service_spec.rb diff --git a/app/services/imports/import_report_service.rb b/app/services/imports/import_report_service.rb new file mode 100644 index 000000000..61d43a9eb --- /dev/null +++ b/app/services/imports/import_report_service.rb @@ -0,0 +1,57 @@ +module Imports + class ImportReportService + def initialize(storage_service, institutions_csv, logger = Rails.logger) + @storage_service = storage_service + @logger = logger + @institutions_csv = institutions_csv + end + + BYTE_ORDER_MARK = "\uFEFF".freeze # Required to ensure Excel always reads CSV as UTF-8 + + def create_reports(report_suffix) + generate_missing_data_coordinators_report(report_suffix) + generate_logs_report(report_suffix) + end + + def generate_missing_data_coordinators_report(report_suffix) + report_csv = "Organisation ID,Old Organisation ID,Organisation Name\n" + organisations = @institutions_csv.map { |row| Organisation.find_by(name: row[0]) }.compact + organisations.each do |organisation| + if organisation.users.none? { |user| user.data_coordinator? && user.active? } + report_csv += "#{organisation.id},#{organisation.old_visible_id},#{organisation.name}\n" + end + end + + report_name = "OrganisationsWithoutDataCoordinators_#{report_suffix}" + @storage_service.write_file(report_name, BYTE_ORDER_MARK + report_csv) + + @logger.info("Missing data coordinators report available in s3 import bucket at #{report_name}") + end + + def generate_logs_report(report_suffix) + Rails.logger.info("Generating migrated logs report") + + rep = CSV.generate do |report| + headers = ["Institution name", "Id", "Old Completed lettings logs", "Old In progress lettings logs", "Old Completed sales logs", "Old In progress sales logs", "New Completed lettings logs", "New In Progress lettings logs", "New Completed sales logs", "New In Progress sales logs"] + report << headers + + @institutions_csv.each do |row| + name = row[0] + organisation = Organisation.find_by(name:) + next unless organisation + + completed_sales_logs = organisation.owned_sales_logs.where(status: "completed").count + in_progress_sales_logs = organisation.owned_sales_logs.where(status: "in_progress").count + completed_lettings_logs = organisation.owned_lettings_logs.where(status: "completed").count + in_progress_lettings_logs = organisation.owned_lettings_logs.where(status: "in_progress").count + report << row.push(completed_lettings_logs, in_progress_lettings_logs, completed_sales_logs, in_progress_sales_logs) + end + end + + report_name = "MigratedLogsReport_#{report_suffix}" + @storage_service.write_file(report_name, BYTE_ORDER_MARK + rep) + + @logger.info("Logs report available in s3 import bucket at #{report_name}") + end + end +end diff --git a/lib/tasks/full_import.rake b/lib/tasks/full_import.rake index 780fe46fc..28af8c4dc 100644 --- a/lib/tasks/full_import.rake +++ b/lib/tasks/full_import.rake @@ -94,41 +94,19 @@ namespace :import do end desc "Generate migrated logs report" - task :generate_report, %i[institutions_csv_name] => :environment do |_task, args| + task :generate_reports, %i[institutions_csv_name] => :environment do |_task, args| institutions_csv_name = args[:institutions_csv_name] - raise "Usage: rake import:generate_report['institutions_csv_name']" if institutions_csv_name.blank? + raise "Usage: rake import:generate_reports['institutions_csv_name']" if institutions_csv_name.blank? s3_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"]) - csv = CSV.parse(s3_service.get_file_io(institutions_csv_name), headers: true) - - Rails.logger.info("Generating migrated logs report") - - rep = CSV.generate do |report| - headers = ["Institution name", "Id", "Old Completed lettings logs", "Old In progress lettings logs", "Old Completed sales logs", "Old In progress sales logs", "New Completed lettings logs", "New In Progress lettings logs", "New Completed sales logs", "New In Progress sales logs"] - report << headers - - csv.each do |row| - name = row[0] - organisation = Organisation.find_by(name:) - next unless organisation - - completed_sales_logs = organisation.owned_sales_logs.where(status: "completed").count - in_progress_sales_logs = organisation.owned_sales_logs.where(status: "in_progress").count - completed_lettings_logs = organisation.owned_lettings_logs.where(status: "completed").count - in_progress_lettings_logs = organisation.owned_lettings_logs.where(status: "in_progress").count - report << row.push(completed_lettings_logs, in_progress_lettings_logs, completed_sales_logs, in_progress_sales_logs) - end - end - - report_name = "MigratedLogsReport_#{institutions_csv_name}" - s3_service.write_file(report_name, rep) + institutions_csv = CSV.parse(s3_service.get_file_io(institutions_csv_name), headers: true) - Rails.logger.info("Logs report available in s3 import bucket at #{report_name}") + Imports::ImportReportService.new(s3_service, institutions_csv).create_reports(institutions_csv_name) end desc "Run import from logs step to end" - task :logs_onwards, %i[institutions_csv_name] => %i[environment logs trigger_invites generate_report] + task :logs_onwards, %i[institutions_csv_name] => %i[environment logs trigger_invites generate_reports] desc "Run a full import for the institutions listed in the named file on s3" - task :full, %i[institutions_csv_name] => %i[environment initial logs trigger_invites generate_report] + task :full, %i[institutions_csv_name] => %i[environment initial logs trigger_invites generate_reports] end diff --git a/spec/lib/tasks/full_import_spec.rb b/spec/lib/tasks/full_import_spec.rb new file mode 100644 index 000000000..f263f15fc --- /dev/null +++ b/spec/lib/tasks/full_import_spec.rb @@ -0,0 +1,44 @@ +require "rails_helper" +require "rake" + +describe "full import", type: :task do + let(:instance_name) { "paas_import_instance" } + let(:paas_config_service) { instance_double(Configuration::PaasConfigurationService) } + let(:storage_service) { instance_double(Storage::S3Service) } + let(:orgs_list) { "Institution name,Id,Old Completed lettings logs,Old In progress lettings logs,Old Completed sales logs,Old In progress sales logs\norg1,1.zip,0,0,0,0\norg2,2.zip,0,0,0,0" } + + before do + allow(Storage::S3Service).to receive(:new).and_return(storage_service) + allow(storage_service).to receive(:write_file).and_return(nil) + allow(storage_service).to receive(:get_file_io).and_return(orgs_list) + allow(Configuration::PaasConfigurationService).to receive(:new).and_return(paas_config_service) + allow(ENV).to receive(:[]) + allow(ENV).to receive(:[]).with("IMPORT_PAAS_INSTANCE").and_return(instance_name) + end + + describe "import:generate_reports" do + subject(:task) { Rake::Task["import:generate_reports"] } + + before do + Rake.application.rake_require("tasks/full_import") + Rake::Task.define_task(:environment) + task.reenable + end + + context "when generating report" do + let(:import_report_service) { instance_double(Imports::ImportReportService) } + + before do + allow(Imports::ImportReportService).to receive(:new).and_return(import_report_service) + end + + it "creates a report using given organisation csv" do + expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name) + expect(Imports::ImportReportService).to receive(:new).with(storage_service, CSV.parse(orgs_list, headers: true)) + expect(import_report_service).to receive(:create_reports).with("some_name") + + task.invoke("some_name") + end + end + end +end diff --git a/spec/services/imports/import_report_service_spec.rb b/spec/services/imports/import_report_service_spec.rb new file mode 100644 index 000000000..1d241754d --- /dev/null +++ b/spec/services/imports/import_report_service_spec.rb @@ -0,0 +1,70 @@ +require "rails_helper" + +RSpec.describe Imports::ImportReportService do + subject(:report_service) { described_class.new(storage_service, institutions_csv) } + + let(:storage_service) { instance_double(Storage::S3Service) } + + describe "#generate_missing_data_coordinators_report" do + context "when all organisations have data coordinators" do + let!(:organisation) { create(:organisation, old_visible_id: "1", name: "org1") } + let(:institutions_csv) { CSV.parse("Institution name,Id,Old Completed lettings logs,Old In progress lettings logs,Old Completed sales logs,Old In progress sales logs\norg1,1,2,1,4,3", headers: true) } + + before do + create(:user, :data_coordinator, organisation:) + end + + it "writes an empty organisations without a data coordinators report" do + expect(storage_service).to receive(:write_file).with("OrganisationsWithoutDataCoordinators_report_suffix.csv", "\uFEFFOrganisation ID,Old Organisation ID,Organisation Name\n") + + report_service.generate_missing_data_coordinators_report("report_suffix.csv") + end + end + + context "when some organisations have no data coordinators" do + let!(:organisation) { create(:organisation, old_visible_id: "1", name: "org1") } + let!(:organisation2) { create(:organisation, old_visible_id: "2", name: "org2") } + let!(:organisation3) { create(:organisation, old_visible_id: "3", name: "org3") } + let(:institutions_csv) { CSV.parse("Institution name,Id,Old Completed lettings logs,Old In progress lettings logs,Old Completed sales logs,Old In progress sales logs\norg1,1,2,1,4,3\norg2,2,5,6,5,7\norg3,3,5,6,5,7", headers: true) } + + before do + create(:user, :data_coordinator, organisation:) + end + + it "writes an empty organisations without a data coordinators report" do + expect(storage_service).to receive(:write_file).with("OrganisationsWithoutDataCoordinators_report_suffix.csv", "\uFEFFOrganisation ID,Old Organisation ID,Organisation Name\n#{organisation2.id},2,org2\n#{organisation3.id},3,org3\n") + + report_service.generate_missing_data_coordinators_report("report_suffix.csv") + end + end + + context "when organisation has an inactive data coordinator" do + let!(:organisation) { create(:organisation, old_visible_id: "1", name: "org1") } + let(:institutions_csv) { CSV.parse("Institution name,Id,Old Completed lettings logs,Old In progress lettings logs,Old Completed sales logs,Old In progress sales logs\norg1,1,2,1,4,3", headers: true) } + + before do + create(:user, :data_coordinator, organisation:, active: false) + end + + it "includes that organisation in the data coordinators report" do + expect(storage_service).to receive(:write_file).with("OrganisationsWithoutDataCoordinators_report_suffix.csv", "\uFEFFOrganisation ID,Old Organisation ID,Organisation Name\n#{organisation.id},1,org1\n") + + report_service.generate_missing_data_coordinators_report("report_suffix.csv") + end + end + end + + describe "#generate_logs_report" do + let(:institutions_csv) { CSV.parse("Institution name,Id,Old Completed lettings logs,Old In progress lettings logs,Old Completed sales logs,Old In progress sales logs\norg1,1,2,1,4,3\norg2,2,5,6,5,7", headers: true) } + + before do + create(:organisation, old_visible_id: "1", name: "org1") + create(:organisation, old_visible_id: "2", name: "org2") + end + + it "generates a report with imported logs" do + expect(storage_service).to receive(:write_file).with("MigratedLogsReport_report_suffix.csv", "\uFEFFInstitution name,Id,Old Completed lettings logs,Old In progress lettings logs,Old Completed sales logs,Old In progress sales logs,New Completed lettings logs,New In Progress lettings logs,New Completed sales logs,New In Progress sales logs\norg1,1,2,1,4,3,0,0,0,0\norg2,2,5,6,5,7,0,0,0,0\n") + report_service.generate_logs_report("report_suffix.csv") + end + end +end