kosiakkatrina
1 year ago
committed by
GitHub
11 changed files with 429 additions and 0 deletions
@ -0,0 +1,17 @@ |
|||||||
|
class CreateIllnessCsvJob < ApplicationJob |
||||||
|
queue_as :default |
||||||
|
|
||||||
|
BYTE_ORDER_MARK = "\uFEFF".freeze # Required to ensure Excel always reads CSV as UTF-8 |
||||||
|
|
||||||
|
def perform(organisation) |
||||||
|
csv_service = Csv::MissingIllnessCsvService.new(organisation) |
||||||
|
|
||||||
|
csv_string = csv_service.create_illness_csv |
||||||
|
filename = "#{['missing-illness', organisation.name, Time.zone.now].compact.join('-')}.csv" |
||||||
|
|
||||||
|
storage_service = Storage::S3Service.new(Configuration::EnvConfigurationService.new, ENV["CSV_DOWNLOAD_PAAS_INSTANCE"]) |
||||||
|
storage_service.write_file(filename, BYTE_ORDER_MARK + csv_string) |
||||||
|
|
||||||
|
Rails.logger.info("Created illness file: #{filename}") |
||||||
|
end |
||||||
|
end |
||||||
@ -0,0 +1,48 @@ |
|||||||
|
module Csv |
||||||
|
class MissingIllnessCsvService |
||||||
|
def initialize(organisation) |
||||||
|
@organisation = organisation |
||||||
|
end |
||||||
|
|
||||||
|
def create_illness_csv |
||||||
|
logs = @organisation.managed_lettings_logs |
||||||
|
.imported |
||||||
|
.after_date(Time.zone.local(2022, 4, 1)) |
||||||
|
.with_illness_without_type |
||||||
|
|
||||||
|
CSV.generate(headers: false) do |csv| |
||||||
|
csv << ["Question", "Log ID", "Tenancy start date", "Tenant code", "Property reference", "Log owner", "Owning organisation", "Managing organisation", "Does anybody in the household have a physical or mental health condition (or other illness) expected to last 12 months or more?", "Does this person's condition affect their vision?", "Does this person's condition affect their hearing?", "Does this person's condition affect their mobility?", "Does this person's condition affect their dexterity?", "Does this person's condition affect their learning or understanding or concentrating?", "Does this person's condition affect their memory?", "Does this person's condition affect their mental health?", "Does this person's condition affect their stamina or breathing or fatigue?", "Does this person's condition affect them socially or behaviourally?", "Does this person's condition affect them in another way?"] |
||||||
|
csv << ["Additional info", nil, nil, nil, nil, nil, nil, nil, nil, "For example, blindness or partial sight", "For example, deafness or partial hearing", nil, "For example, lifting and carrying objects, or using a keyboard", nil, nil, "For example, depression or anxiety", nil, "Anything associated with autism spectrum disorder (ASD), including Asperger's or attention deficit hyperactivity disorder (ADHD)", nil, nil] |
||||||
|
csv << ["How to answer", "Do not change the answers for this field", "Do not change the answers for this field", "Do not change the answers for this field", "Do not change the answers for this field", "Do not change the answers for this field", "Do not change the answers for this field", "Do not change the answers for this field", "1 = Yes; 2 = No; 3 = Prefers not to say", "1 = Yes; blank = No", "1 = Yes; blank = No", "1 = Yes; blank = No", "1 = Yes; blank = No", "1 = Yes; blank = No", "1 = Yes; blank = No", "1 = Yes; blank = No", "1 = Yes; blank = No", "1 = Yes; blank = No", "1 = Yes; blank = No"] |
||||||
|
|
||||||
|
logs.each do |log| |
||||||
|
csv << log_to_csv_row(log) |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
private |
||||||
|
|
||||||
|
def log_to_csv_row(log) |
||||||
|
[nil, |
||||||
|
log.id, |
||||||
|
log.startdate&.to_date, |
||||||
|
log.tenancycode, |
||||||
|
log.propcode, |
||||||
|
log.created_by&.email, |
||||||
|
log.owning_organisation&.name, |
||||||
|
log.managing_organisation&.name, |
||||||
|
1, |
||||||
|
nil, |
||||||
|
nil, |
||||||
|
nil, |
||||||
|
nil, |
||||||
|
nil, |
||||||
|
nil, |
||||||
|
nil, |
||||||
|
nil, |
||||||
|
nil, |
||||||
|
nil] |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
@ -0,0 +1,61 @@ |
|||||||
|
namespace :correct_illness do |
||||||
|
desc "Export data CSVs for import into Central Data System (CDS)" |
||||||
|
task :create_illness_csv, %i[organisation_id] => :environment do |_task, args| |
||||||
|
organisation_id = args[:organisation_id] |
||||||
|
raise "Usage: rake correct_illness:create_illness_csv['organisation_id']" if organisation_id.blank? |
||||||
|
|
||||||
|
organisation = Organisation.find_by(id: organisation_id) |
||||||
|
if organisation.present? |
||||||
|
CreateIllnessCsvJob.perform_later(organisation) |
||||||
|
Rails.logger.info("Creating illness CSV for #{organisation.name}") |
||||||
|
else |
||||||
|
Rails.logger.error("Organisation with ID #{organisation_id} not found") |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
desc "Export data CSVs for import into Central Data System (CDS)" |
||||||
|
task :correct_illness_from_csv, %i[file_name] => :environment do |_task, args| |
||||||
|
file_name = args[:file_name] |
||||||
|
|
||||||
|
raise "Usage: rake correct_illness:correct_illness_from_csv['csv_file_name']" if file_name.blank? |
||||||
|
|
||||||
|
s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"]) |
||||||
|
illness_csv = CSV.parse(s3_service.get_file_io(file_name), headers: false) |
||||||
|
|
||||||
|
illness_csv.each_with_index do |row, index| |
||||||
|
next if index < 3 |
||||||
|
|
||||||
|
lettings_log_id = row[1] |
||||||
|
|
||||||
|
if lettings_log_id.blank? |
||||||
|
Rails.logger.info("Lettings log ID not provided") |
||||||
|
next |
||||||
|
end |
||||||
|
|
||||||
|
lettings_log = LettingsLog.find_by(id: lettings_log_id) |
||||||
|
if lettings_log.blank? |
||||||
|
Rails.logger.info("Could not find a lettings log with id #{lettings_log_id}") |
||||||
|
next |
||||||
|
end |
||||||
|
|
||||||
|
lettings_log.illness = row[8] |
||||||
|
lettings_log.illness_type_1 = row[9].presence || 0 |
||||||
|
lettings_log.illness_type_2 = row[10].presence || 0 |
||||||
|
lettings_log.illness_type_3 = row[11].presence || 0 |
||||||
|
lettings_log.illness_type_4 = row[12].presence || 0 |
||||||
|
lettings_log.illness_type_5 = row[13].presence || 0 |
||||||
|
lettings_log.illness_type_6 = row[14].presence || 0 |
||||||
|
lettings_log.illness_type_7 = row[15].presence || 0 |
||||||
|
lettings_log.illness_type_8 = row[16].presence || 0 |
||||||
|
lettings_log.illness_type_9 = row[17].presence || 0 |
||||||
|
lettings_log.illness_type_10 = row[18].presence || 0 |
||||||
|
lettings_log.values_updated_at = Time.zone.now |
||||||
|
|
||||||
|
if lettings_log.save |
||||||
|
Rails.logger.info("Updated lettings log #{lettings_log_id}, with illness: #{([lettings_log.illness] + %w[illness_type_1 illness_type_2 illness_type_3 illness_type_4 illness_type_5 illness_type_6 illness_type_7 illness_type_8 illness_type_9 illness_type_10].select { |illness_type| lettings_log[illness_type] == 1 }).join(', ')}") |
||||||
|
else |
||||||
|
Rails.logger.error("Validation failed for lettings log with ID #{lettings_log.id}: #{lettings_log.errors.full_messages.join(', ')}}") |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
Can't render this file because it has a wrong number of fields in line 2.
|
|
Can't render this file because it has a wrong number of fields in line 2.
|
|
Can't render this file because it has a wrong number of fields in line 2.
|
@ -0,0 +1,33 @@ |
|||||||
|
require "rails_helper" |
||||||
|
|
||||||
|
describe CreateIllnessCsvJob do |
||||||
|
include Helpers |
||||||
|
|
||||||
|
let(:job) { described_class.new } |
||||||
|
let(:storage_service) { instance_double(Storage::S3Service) } |
||||||
|
let(:mailer) { instance_double(CsvDownloadMailer) } |
||||||
|
let(:missing_illness_csv_service) { instance_double(Csv::MissingIllnessCsvService) } |
||||||
|
let(:organisation) { build(:organisation) } |
||||||
|
let(:users) { create_list(:user, 2) } |
||||||
|
|
||||||
|
before do |
||||||
|
allow(Storage::S3Service).to receive(:new).and_return(storage_service) |
||||||
|
allow(storage_service).to receive(:write_file) |
||||||
|
|
||||||
|
allow(Csv::MissingIllnessCsvService).to receive(:new).and_return(missing_illness_csv_service) |
||||||
|
allow(missing_illness_csv_service).to receive(:create_illness_csv).and_return("") |
||||||
|
end |
||||||
|
|
||||||
|
context "when creating illness logs csv" do |
||||||
|
it "uses an appropriate filename in S3" do |
||||||
|
expect(storage_service).to receive(:write_file).with(/missing-illness-#{organisation.name}-.*\.csv/, anything) |
||||||
|
job.perform(organisation) |
||||||
|
end |
||||||
|
|
||||||
|
it "creates a MissingIllnessCsvService with the correct organisation and calls create illness csv" do |
||||||
|
expect(Csv::MissingIllnessCsvService).to receive(:new).with(organisation) |
||||||
|
expect(missing_illness_csv_service).to receive(:create_illness_csv) |
||||||
|
job.perform(organisation) |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
@ -0,0 +1,178 @@ |
|||||||
|
require "rails_helper" |
||||||
|
require "rake" |
||||||
|
RSpec.describe "correct_illness" do |
||||||
|
describe ":create_illness_csv", type: :task do |
||||||
|
subject(:task) { Rake::Task["correct_illness:create_illness_csv"] } |
||||||
|
|
||||||
|
before do |
||||||
|
organisation.users.destroy_all |
||||||
|
Rake.application.rake_require("tasks/correct_illness_from_csv") |
||||||
|
Rake::Task.define_task(:environment) |
||||||
|
task.reenable |
||||||
|
end |
||||||
|
|
||||||
|
context "when the rake task is run" do |
||||||
|
let(:organisation) { create(:organisation, name: "test organisation") } |
||||||
|
|
||||||
|
context "and organisation ID is provided" do |
||||||
|
it "enqueues the job with correct organisation" do |
||||||
|
expect { task.invoke(organisation.id) }.to enqueue_job(CreateIllnessCsvJob).with(organisation) |
||||||
|
end |
||||||
|
|
||||||
|
it "prints out the jobs enqueued" do |
||||||
|
expect(Rails.logger).to receive(:info).with(nil) |
||||||
|
expect(Rails.logger).to receive(:info).with("Creating illness CSV for test organisation") |
||||||
|
task.invoke(organisation.id) |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
context "when organisation with given ID cannot be found" do |
||||||
|
it "prints out error" do |
||||||
|
expect(Rails.logger).to receive(:error).with("Organisation with ID fake not found") |
||||||
|
task.invoke("fake") |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
context "when organisation ID is not given" do |
||||||
|
it "raises an error" do |
||||||
|
expect { task.invoke }.to raise_error(RuntimeError, "Usage: rake correct_illness:create_illness_csv['organisation_id']") |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
describe ":correct_illness_from_csv", type: :task do |
||||||
|
def replace_entity_ids(lettings_log, second_lettings_log, third_lettings_log, export_template) |
||||||
|
export_template.sub!(/\{id\}/, lettings_log.id.to_s) |
||||||
|
export_template.sub!(/\{id2\}/, second_lettings_log.id.to_s) |
||||||
|
export_template.sub!(/\{id3\}/, third_lettings_log.id.to_s) |
||||||
|
end |
||||||
|
|
||||||
|
subject(:task) { Rake::Task["correct_illness:correct_illness_from_csv"] } |
||||||
|
|
||||||
|
let(:instance_name) { "paas_import_instance" } |
||||||
|
let(:storage_service) { instance_double(Storage::S3Service) } |
||||||
|
let(:env_config_service) { instance_double(Configuration::EnvConfigurationService) } |
||||||
|
let(:paas_config_service) { instance_double(Configuration::PaasConfigurationService) } |
||||||
|
|
||||||
|
before do |
||||||
|
allow(Storage::S3Service).to receive(:new).and_return(storage_service) |
||||||
|
allow(Configuration::EnvConfigurationService).to receive(:new).and_return(env_config_service) |
||||||
|
allow(Configuration::PaasConfigurationService).to receive(:new).and_return(paas_config_service) |
||||||
|
allow(ENV).to receive(:[]) |
||||||
|
allow(ENV).to receive(:[]).with("IMPORT_PAAS_INSTANCE").and_return(instance_name) |
||||||
|
allow(ENV).to receive(:[]).with("VCAP_SERVICES").and_return("dummy") |
||||||
|
|
||||||
|
Rake.application.rake_require("tasks/correct_illness_from_csv") |
||||||
|
Rake::Task.define_task(:environment) |
||||||
|
task.reenable |
||||||
|
end |
||||||
|
|
||||||
|
context "when the rake task is run" do |
||||||
|
let(:illness_csv_path) { "illness_123.csv" } |
||||||
|
let(:wrong_file_path) { "/test/no_csv_here.csv" } |
||||||
|
let!(:lettings_log) do |
||||||
|
create(:lettings_log, |
||||||
|
:completed, |
||||||
|
:with_illness_without_type) |
||||||
|
end |
||||||
|
|
||||||
|
let!(:second_lettings_log) do |
||||||
|
create(:lettings_log, |
||||||
|
:completed, |
||||||
|
illness: 1, |
||||||
|
illness_type_1: true, |
||||||
|
illness_type_2: true) |
||||||
|
end |
||||||
|
|
||||||
|
let!(:third_lettings_log) do |
||||||
|
create(:lettings_log, |
||||||
|
:completed, |
||||||
|
illness: 1, |
||||||
|
illness_type_3: true, |
||||||
|
illness_type_4: true) |
||||||
|
end |
||||||
|
|
||||||
|
before do |
||||||
|
allow(storage_service).to receive(:get_file_io) |
||||||
|
.with("illness_123.csv") |
||||||
|
.and_return(replace_entity_ids(lettings_log, second_lettings_log, third_lettings_log, File.open("./spec/fixtures/files/illness_update.csv").read)) |
||||||
|
end |
||||||
|
|
||||||
|
it "sets illness to yes and sets correct illness type" do |
||||||
|
task.invoke(illness_csv_path) |
||||||
|
lettings_log.reload |
||||||
|
expect(lettings_log.illness).to eq(1) |
||||||
|
expect(lettings_log.illness_type_2).to eq(1) |
||||||
|
%w[illness_type_1 |
||||||
|
illness_type_3 |
||||||
|
illness_type_4 |
||||||
|
illness_type_5 |
||||||
|
illness_type_6 |
||||||
|
illness_type_7 |
||||||
|
illness_type_8 |
||||||
|
illness_type_9 |
||||||
|
illness_type_10].each do |illness_type| |
||||||
|
expect(lettings_log[illness_type]).to eq(0) |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
it "sets illness to no" do |
||||||
|
task.invoke(illness_csv_path) |
||||||
|
second_lettings_log.reload |
||||||
|
expect(second_lettings_log.illness).to eq(2) |
||||||
|
%w[illness_type_1 |
||||||
|
illness_type_2 |
||||||
|
illness_type_3 |
||||||
|
illness_type_4 |
||||||
|
illness_type_5 |
||||||
|
illness_type_6 |
||||||
|
illness_type_7 |
||||||
|
illness_type_8 |
||||||
|
illness_type_9 |
||||||
|
illness_type_10].each do |illness_type| |
||||||
|
expect(second_lettings_log[illness_type]).to eq(nil) |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
it "sets illness to don't know" do |
||||||
|
task.invoke(illness_csv_path) |
||||||
|
third_lettings_log.reload |
||||||
|
expect(third_lettings_log.illness).to eq(3) |
||||||
|
%w[illness_type_1 |
||||||
|
illness_type_2 |
||||||
|
illness_type_3 |
||||||
|
illness_type_4 |
||||||
|
illness_type_5 |
||||||
|
illness_type_6 |
||||||
|
illness_type_7 |
||||||
|
illness_type_8 |
||||||
|
illness_type_9 |
||||||
|
illness_type_10].each do |illness_type| |
||||||
|
expect(third_lettings_log[illness_type]).to eq(nil) |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
it "logs the progress of the update" do |
||||||
|
expect(Rails.logger).to receive(:info).with("Updated lettings log #{lettings_log.id}, with illness: 1, illness_type_2") |
||||||
|
expect(Rails.logger).to receive(:info).with("Updated lettings log #{second_lettings_log.id}, with illness: 2") |
||||||
|
expect(Rails.logger).to receive(:info).with("Updated lettings log #{third_lettings_log.id}, with illness: 3") |
||||||
|
expect(Rails.logger).to receive(:info).with("Lettings log ID not provided") |
||||||
|
expect(Rails.logger).to receive(:info).with("Could not find a lettings log with id fake_id") |
||||||
|
|
||||||
|
task.invoke(illness_csv_path) |
||||||
|
end |
||||||
|
|
||||||
|
it "raises an error when no path is given" do |
||||||
|
expect { task.invoke(nil) }.to raise_error(RuntimeError, "Usage: rake correct_illness:correct_illness_from_csv['csv_file_name']") |
||||||
|
end |
||||||
|
|
||||||
|
it "logs an error if a validation fails" do |
||||||
|
lettings_log.postcode_full = "invalid_format" |
||||||
|
lettings_log.save!(validate: false) |
||||||
|
expect(Rails.logger).to receive(:error).with(/Validation failed for lettings log with ID #{lettings_log.id}: Postcode full/) |
||||||
|
task.invoke(illness_csv_path) |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
@ -0,0 +1,50 @@ |
|||||||
|
require "rails_helper" |
||||||
|
RSpec.describe Csv::MissingIllnessCsvService do |
||||||
|
let(:organisation) { create(:organisation, name: "Illness org") } |
||||||
|
let(:user) { create(:user, organisation:, email: "testy@example.com") } |
||||||
|
let(:service) { described_class.new(organisation) } |
||||||
|
|
||||||
|
def replace_entity_ids(lettings_log, export_template) |
||||||
|
export_template.sub!(/\{id\}/, lettings_log.id.to_s) |
||||||
|
end |
||||||
|
|
||||||
|
describe "#create_illness_csv" do |
||||||
|
context "when the organisation has lettings logs" do |
||||||
|
let!(:lettings_log) do |
||||||
|
create(:lettings_log, |
||||||
|
:setup_completed, |
||||||
|
:with_illness_without_type, |
||||||
|
tenancycode: "tenancycode1", |
||||||
|
propcode: "propcode1", |
||||||
|
startdate: Time.zone.local(2023, 4, 5), |
||||||
|
created_by: user, |
||||||
|
owning_organisation: organisation, |
||||||
|
managing_organisation: organisation, |
||||||
|
old_id: "old_id_1") |
||||||
|
end |
||||||
|
|
||||||
|
before do |
||||||
|
create(:lettings_log, :setup_completed, :with_illness_without_type, startdate: Time.zone.local(2023, 4, 5), owning_organisation: organisation, managing_organisation: organisation, created_by: user) |
||||||
|
create(:lettings_log, :setup_completed, :with_illness_without_type, startdate: Time.zone.local(2023, 4, 5), old_id: "old_id_3") |
||||||
|
create(:lettings_log, :setup_completed, illness: 0, startdate: Time.zone.local(2023, 4, 5), owning_organisation: organisation, managing_organisation: organisation, created_by: user, old_id: "old_id_4") |
||||||
|
create(:lettings_log, :setup_completed, illness: 1, illness_type_1: true, startdate: Time.zone.local(2023, 4, 5), owning_organisation: organisation, managing_organisation: organisation, created_by: user, old_id: "old_id_5") |
||||||
|
log = build(:lettings_log, :setup_completed, :with_illness_without_type, startdate: Time.zone.local(2021, 4, 5), owning_organisation: organisation, managing_organisation: organisation, created_by: user, old_id: "old_id_2") |
||||||
|
log.save!(validate: false) |
||||||
|
end |
||||||
|
|
||||||
|
it "returns a csv with relevant logs" do |
||||||
|
illness_csv = replace_entity_ids(lettings_log, File.open("spec/fixtures/files/illness.csv").read) |
||||||
|
csv = service.create_illness_csv |
||||||
|
expect(csv).to eq(illness_csv) |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
context "when the organisation does not have relevant lettings logs" do |
||||||
|
it "returns only headers" do |
||||||
|
illness_headers_only_csv = File.open("spec/fixtures/files/empty_illness.csv").read |
||||||
|
csv = service.create_illness_csv |
||||||
|
expect(csv).to eq(illness_headers_only_csv) |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
Loading…
Reference in new issue