Browse Source

CLDC-2724 Add missing data reporting for imported lettings logs (#1892)

* Add generate_missing_answers_report task and method

* Create example report

* change task description

* refactor

* Report old form id instead of old id
pull/1893/head
kosiakkatrina 1 year ago committed by GitHub
parent
commit
11ca1bbcff
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 47
      app/services/imports/import_report_service.rb
  2. 9
      lib/tasks/full_import.rake
  3. 16
      spec/fixtures/files/imported_lettings_logs_missing_answers_examples.csv
  4. 4
      spec/fixtures/files/imported_lettings_logs_missing_answers_report.csv
  5. 26
      spec/lib/tasks/full_import_spec.rb
  6. 38
      spec/services/imports/import_report_service_spec.rb

47
app/services/imports/import_report_service.rb

@ -84,5 +84,52 @@ module Imports
@logger.info("Unassigned logs report available in s3 import bucket at #{report_name}") @logger.info("Unassigned logs report available in s3 import bucket at #{report_name}")
end end
def generate_missing_answers_report(report_suffix)
Rails.logger.info("Generating missing imported lettings logs answers report")
unanswered_question_counts = {}
missing_answers_example_sets = {}
LettingsLog.where.not(old_id: nil).where(status: "in_progress").each do |lettings_log|
applicable_questions = lettings_log.form.subsections.map { |s| s.applicable_questions(lettings_log).select { |q| q.enabled?(lettings_log) } }.flatten
unanswered_questions = (applicable_questions.filter { |q| q.unanswered?(lettings_log) }.map(&:id) - lettings_log.optional_fields).join(", ")
if unanswered_question_counts[unanswered_questions].present?
unanswered_question_counts[unanswered_questions] += 1
missing_answers_example_sets[unanswered_questions] << { id: lettings_log.id, old_form_id: lettings_log.old_form_id, owning_organisation_id: lettings_log.owning_organisation_id } unless unanswered_question_counts[unanswered_questions] > 10
else
unanswered_question_counts[unanswered_questions] = 1
missing_answers_example_sets[unanswered_questions] = [{ id: lettings_log.id, old_form_id: lettings_log.old_form_id, owning_organisation_id: lettings_log.owning_organisation_id }]
end
end
rep = CSV.generate do |report|
headers = ["Missing answers", "Total number of affected logs"]
report << headers
unanswered_question_counts.each do |missing_answers, count|
report << [missing_answers, count]
end
end
missing_answers_examples = CSV.generate do |report|
headers = ["Missing answers", "Organisation ID", "Log ID", "Old Form ID"]
report << headers
missing_answers_example_sets.each do |missing_answers, examples|
examples.each do |example|
report << [missing_answers, example[:owning_organisation_id], example[:id], example[:old_form_id]]
end
end
end
report_name = "MissingAnswersReport_#{report_suffix}.csv"
@storage_service.write_file(report_name, BYTE_ORDER_MARK + rep)
examples_report_name = "MissingAnswersExamples_#{report_suffix}.csv"
@storage_service.write_file(examples_report_name, BYTE_ORDER_MARK + missing_answers_examples)
@logger.info("Missing answers report available in s3 import bucket at #{report_name}")
end
end end
end end

9
lib/tasks/full_import.rake

@ -117,6 +117,15 @@ namespace :import do
Imports::ImportReportService.new(s3_service, institutions_csv).create_reports(institutions_csv_name) Imports::ImportReportService.new(s3_service, institutions_csv).create_reports(institutions_csv_name)
end end
desc "Generate missing answers report"
task :generate_missing_answers_report, %i[file_suffix] => :environment do |_task, args|
file_suffix = args[:file_suffix]
raise "Usage: rake import:generate_reports['file_suffix']" if file_suffix.blank?
s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
Imports::ImportReportService.new(s3_service, nil).generate_missing_answers_report(file_suffix)
end
desc "Run import from logs step to end" desc "Run import from logs step to end"
task :logs_onwards, %i[institutions_csv_name] => %i[environment logs trigger_invites generate_reports] task :logs_onwards, %i[institutions_csv_name] => %i[environment logs trigger_invites generate_reports]

16
spec/fixtures/files/imported_lettings_logs_missing_answers_examples.csv vendored

@ -0,0 +1,16 @@
Missing answers,Organisation ID,Log ID,Old Form ID
age1_known,{org_id0},{id0},1000
age1_known,{org_id1},{id1},1001
age1_known,{org_id2},{id2},1002
age1_known,{org_id3},{id3},1003
age1_known,{org_id4},{id4},1004
age1_known,{org_id5},{id5},1005
age1_known,{org_id6},{id6},1006
age1_known,{org_id7},{id7},1007
age1_known,{org_id8},{id8},1008
age1_known,{org_id9},{id9},1009
beds,{org_id2_0},{id2_0},2000
beds,{org_id2_1},{id2_1},2001
beds,{org_id2_2},{id2_2},2002
beds,{org_id2_3},{id2_3},2003
"beds, age1_known",{org_id},{id},300
1 Missing answers Organisation ID Log ID Old Form ID
2 age1_known {org_id0} {id0} 1000
3 age1_known {org_id1} {id1} 1001
4 age1_known {org_id2} {id2} 1002
5 age1_known {org_id3} {id3} 1003
6 age1_known {org_id4} {id4} 1004
7 age1_known {org_id5} {id5} 1005
8 age1_known {org_id6} {id6} 1006
9 age1_known {org_id7} {id7} 1007
10 age1_known {org_id8} {id8} 1008
11 age1_known {org_id9} {id9} 1009
12 beds {org_id2_0} {id2_0} 2000
13 beds {org_id2_1} {id2_1} 2001
14 beds {org_id2_2} {id2_2} 2002
15 beds {org_id2_3} {id2_3} 2003
16 beds, age1_known {org_id} {id} 300

4
spec/fixtures/files/imported_lettings_logs_missing_answers_report.csv vendored

@ -0,0 +1,4 @@
Missing answers,Total number of affected logs
age1_known,11
beds,4
"beds, age1_known",1
1 Missing answers Total number of affected logs
2 age1_known 11
3 beds 4
4 beds, age1_known 1

26
spec/lib/tasks/full_import_spec.rb

@ -54,6 +54,32 @@ describe "full import", type: :task do
end end
end end
describe "import:generate_missing_answers_report" do
subject(:task) { Rake::Task["import:generate_missing_answers_report"] }
before do
Rake.application.rake_require("tasks/full_import")
Rake::Task.define_task(:environment)
task.reenable
end
context "when generating a missing answers report" do
let(:import_report_service) { instance_double(Imports::ImportReportService) }
before do
allow(Imports::ImportReportService).to receive(:new).and_return(import_report_service)
allow(ENV).to receive(:[]).with("VCAP_SERVICES").and_return("dummy")
end
it "creates a missing answers report" do
expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name)
expect(Imports::ImportReportService).to receive(:new).with(storage_service, nil)
expect(import_report_service).to receive(:generate_missing_answers_report).with("some_name")
task.invoke("some_name")
end
end
end
describe "import:initial" do describe "import:initial" do
subject(:task) { Rake::Task["import:initial"] } subject(:task) { Rake::Task["import:initial"] }

38
spec/services/imports/import_report_service_spec.rb

@ -98,4 +98,42 @@ RSpec.describe Imports::ImportReportService do
end end
end end
end end
describe "#generate_missing_answers_report" do
context "when there are in progress imported logs" do
let(:institutions_csv) { nil }
let(:expected_content) { File.read("spec/fixtures/files/imported_lettings_logs_missing_answers_report.csv") }
let(:expected__answers_examples_content) { File.read("spec/fixtures/files/imported_lettings_logs_missing_answers_examples.csv") }
before do
create_list(:lettings_log, 11, :completed, age1_known: nil) do |log, i|
log.old_form_id = "100#{i}"
log.old_id = "old_id_age1_known_#{i}"
log.save!
expected__answers_examples_content.sub!("{id#{i}}", log.id.to_s)
expected__answers_examples_content.sub!("{org_id#{i}}", log.owning_organisation_id.to_s)
end
create_list(:lettings_log, 4, :completed, beds: nil) do |log, i|
log.old_form_id = "200#{i}"
log.old_id = "old_id_beds_#{i}"
expected__answers_examples_content.sub!("{id2_#{i}}", log.id.to_s)
expected__answers_examples_content.sub!("{org_id2_#{i}}", log.owning_organisation_id.to_s)
log.save!
end
create(:lettings_log, :completed, age1_known: nil, beds: nil, old_form_id: "300", old_id: "123") do |log|
expected__answers_examples_content.sub!("{id}", log.id.to_s)
expected__answers_examples_content.sub!("{org_id}", log.owning_organisation_id.to_s)
end
create_list(:lettings_log, 2, :completed, age1_known: nil)
end
it "generates a csv with expected missing fields" do
expect(storage_service).to receive(:write_file).with("MissingAnswersReport_report_suffix.csv", "#{expected_content}")
expect(storage_service).to receive(:write_file).with("MissingAnswersExamples_report_suffix.csv", "#{expected__answers_examples_content}")
report_service.generate_missing_answers_report("report_suffix")
end
end
end
end end

Loading…
Cancel
Save