diff --git a/app/services/imports/import_report_service.rb b/app/services/imports/import_report_service.rb index 4e387c4c4..7dcde699c 100644 --- a/app/services/imports/import_report_service.rb +++ b/app/services/imports/import_report_service.rb @@ -84,5 +84,52 @@ module Imports @logger.info("Unassigned logs report available in s3 import bucket at #{report_name}") end + + def generate_missing_answers_report(report_suffix) + Rails.logger.info("Generating missing imported lettings logs answers report") + unanswered_question_counts = {} + missing_answers_example_sets = {} + + LettingsLog.where.not(old_id: nil).where(status: "in_progress").each do |lettings_log| + applicable_questions = lettings_log.form.subsections.map { |s| s.applicable_questions(lettings_log).select { |q| q.enabled?(lettings_log) } }.flatten + unanswered_questions = (applicable_questions.filter { |q| q.unanswered?(lettings_log) }.map(&:id) - lettings_log.optional_fields).join(", ") + + if unanswered_question_counts[unanswered_questions].present? + unanswered_question_counts[unanswered_questions] += 1 + missing_answers_example_sets[unanswered_questions] << { id: lettings_log.id, old_form_id: lettings_log.old_form_id, owning_organisation_id: lettings_log.owning_organisation_id } unless unanswered_question_counts[unanswered_questions] > 10 + else + unanswered_question_counts[unanswered_questions] = 1 + missing_answers_example_sets[unanswered_questions] = [{ id: lettings_log.id, old_form_id: lettings_log.old_form_id, owning_organisation_id: lettings_log.owning_organisation_id }] + end + end + + rep = CSV.generate do |report| + headers = ["Missing answers", "Total number of affected logs"] + report << headers + + unanswered_question_counts.each do |missing_answers, count| + report << [missing_answers, count] + end + end + + missing_answers_examples = CSV.generate do |report| + headers = ["Missing answers", "Organisation ID", "Log ID", "Old Form ID"] + report << headers + + missing_answers_example_sets.each do |missing_answers, examples| + examples.each do |example| + report << [missing_answers, example[:owning_organisation_id], example[:id], example[:old_form_id]] + end + end + end + + report_name = "MissingAnswersReport_#{report_suffix}.csv" + @storage_service.write_file(report_name, BYTE_ORDER_MARK + rep) + + examples_report_name = "MissingAnswersExamples_#{report_suffix}.csv" + @storage_service.write_file(examples_report_name, BYTE_ORDER_MARK + missing_answers_examples) + + @logger.info("Missing answers report available in s3 import bucket at #{report_name}") + end end end diff --git a/lib/tasks/full_import.rake b/lib/tasks/full_import.rake index 3b74b458a..4c5fcf380 100644 --- a/lib/tasks/full_import.rake +++ b/lib/tasks/full_import.rake @@ -117,6 +117,15 @@ namespace :import do Imports::ImportReportService.new(s3_service, institutions_csv).create_reports(institutions_csv_name) end + desc "Generate missing answers report" + task :generate_missing_answers_report, %i[file_suffix] => :environment do |_task, args| + file_suffix = args[:file_suffix] + raise "Usage: rake import:generate_reports['file_suffix']" if file_suffix.blank? + + s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"]) + Imports::ImportReportService.new(s3_service, nil).generate_missing_answers_report(file_suffix) + end + desc "Run import from logs step to end" task :logs_onwards, %i[institutions_csv_name] => %i[environment logs trigger_invites generate_reports] diff --git a/spec/fixtures/files/imported_lettings_logs_missing_answers_examples.csv b/spec/fixtures/files/imported_lettings_logs_missing_answers_examples.csv new file mode 100644 index 000000000..0c35bae0b --- /dev/null +++ b/spec/fixtures/files/imported_lettings_logs_missing_answers_examples.csv @@ -0,0 +1,16 @@ +Missing answers,Organisation ID,Log ID,Old Form ID +age1_known,{org_id0},{id0},1000 +age1_known,{org_id1},{id1},1001 +age1_known,{org_id2},{id2},1002 +age1_known,{org_id3},{id3},1003 +age1_known,{org_id4},{id4},1004 +age1_known,{org_id5},{id5},1005 +age1_known,{org_id6},{id6},1006 +age1_known,{org_id7},{id7},1007 +age1_known,{org_id8},{id8},1008 +age1_known,{org_id9},{id9},1009 +beds,{org_id2_0},{id2_0},2000 +beds,{org_id2_1},{id2_1},2001 +beds,{org_id2_2},{id2_2},2002 +beds,{org_id2_3},{id2_3},2003 +"beds, age1_known",{org_id},{id},300 diff --git a/spec/fixtures/files/imported_lettings_logs_missing_answers_report.csv b/spec/fixtures/files/imported_lettings_logs_missing_answers_report.csv new file mode 100644 index 000000000..936dc82e0 --- /dev/null +++ b/spec/fixtures/files/imported_lettings_logs_missing_answers_report.csv @@ -0,0 +1,4 @@ +Missing answers,Total number of affected logs +age1_known,11 +beds,4 +"beds, age1_known",1 diff --git a/spec/lib/tasks/full_import_spec.rb b/spec/lib/tasks/full_import_spec.rb index b95b370cd..2bb85e9b6 100644 --- a/spec/lib/tasks/full_import_spec.rb +++ b/spec/lib/tasks/full_import_spec.rb @@ -54,6 +54,32 @@ describe "full import", type: :task do end end + describe "import:generate_missing_answers_report" do + subject(:task) { Rake::Task["import:generate_missing_answers_report"] } + + before do + Rake.application.rake_require("tasks/full_import") + Rake::Task.define_task(:environment) + task.reenable + end + + context "when generating a missing answers report" do + let(:import_report_service) { instance_double(Imports::ImportReportService) } + + before do + allow(Imports::ImportReportService).to receive(:new).and_return(import_report_service) + allow(ENV).to receive(:[]).with("VCAP_SERVICES").and_return("dummy") + end + + it "creates a missing answers report" do + expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name) + expect(Imports::ImportReportService).to receive(:new).with(storage_service, nil) + expect(import_report_service).to receive(:generate_missing_answers_report).with("some_name") + task.invoke("some_name") + end + end + end + describe "import:initial" do subject(:task) { Rake::Task["import:initial"] } diff --git a/spec/services/imports/import_report_service_spec.rb b/spec/services/imports/import_report_service_spec.rb index abd318a8d..ca6e81bc2 100644 --- a/spec/services/imports/import_report_service_spec.rb +++ b/spec/services/imports/import_report_service_spec.rb @@ -98,4 +98,42 @@ RSpec.describe Imports::ImportReportService do end end end + + describe "#generate_missing_answers_report" do + context "when there are in progress imported logs" do + let(:institutions_csv) { nil } + let(:expected_content) { File.read("spec/fixtures/files/imported_lettings_logs_missing_answers_report.csv") } + let(:expected__answers_examples_content) { File.read("spec/fixtures/files/imported_lettings_logs_missing_answers_examples.csv") } + + before do + create_list(:lettings_log, 11, :completed, age1_known: nil) do |log, i| + log.old_form_id = "100#{i}" + log.old_id = "old_id_age1_known_#{i}" + log.save! + expected__answers_examples_content.sub!("{id#{i}}", log.id.to_s) + expected__answers_examples_content.sub!("{org_id#{i}}", log.owning_organisation_id.to_s) + end + create_list(:lettings_log, 4, :completed, beds: nil) do |log, i| + log.old_form_id = "200#{i}" + log.old_id = "old_id_beds_#{i}" + expected__answers_examples_content.sub!("{id2_#{i}}", log.id.to_s) + expected__answers_examples_content.sub!("{org_id2_#{i}}", log.owning_organisation_id.to_s) + log.save! + end + create(:lettings_log, :completed, age1_known: nil, beds: nil, old_form_id: "300", old_id: "123") do |log| + expected__answers_examples_content.sub!("{id}", log.id.to_s) + expected__answers_examples_content.sub!("{org_id}", log.owning_organisation_id.to_s) + end + + create_list(:lettings_log, 2, :completed, age1_known: nil) + end + + it "generates a csv with expected missing fields" do + expect(storage_service).to receive(:write_file).with("MissingAnswersReport_report_suffix.csv", "#{expected_content}") + expect(storage_service).to receive(:write_file).with("MissingAnswersExamples_report_suffix.csv", "#{expected__answers_examples_content}") + + report_service.generate_missing_answers_report("report_suffix") + end + end + end end