From 6141d34d83e9e69905c39b3a44e8d40fef30cc18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Meny?= Date: Wed, 9 Feb 2022 14:43:45 +0000 Subject: [PATCH] Implement basic case logs export --- Gemfile | 1 + Gemfile.lock | 2 + .../exports/case_log_export_service.rb | 75 ++++ lib/tasks/data_export.rake | 7 + lib/tasks/data_import.rake | 2 - spec/fixtures/exports/case_logs.xml | 325 ++++++++++++++++++ spec/lib/tasks/data_export_spec.rb | 33 ++ spec/lib/tasks/data_import_spec.rb | 2 +- .../exports/case_log_export_service_spec.rb | 42 +++ 9 files changed, 486 insertions(+), 3 deletions(-) create mode 100644 app/services/exports/case_log_export_service.rb create mode 100644 lib/tasks/data_export.rake create mode 100644 spec/fixtures/exports/case_logs.xml create mode 100644 spec/lib/tasks/data_export_spec.rb create mode 100644 spec/services/exports/case_log_export_service_spec.rb diff --git a/Gemfile b/Gemfile index b96ea93e1..b2430519c 100644 --- a/Gemfile +++ b/Gemfile @@ -77,6 +77,7 @@ group :test do gem "rspec-rails", require: false gem "selenium-webdriver", require: false gem "simplecov", require: false + gem "timecop", "~> 0.9.4" gem "webmock", require: false end diff --git a/Gemfile.lock b/Gemfile.lock index 06bea744e..7d21a547b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -411,6 +411,7 @@ GEM railties (>= 6.0.0) strscan (3.0.1) thor (1.2.1) + timecop (0.9.4) timeout (0.2.0) turbo-rails (1.0.1) actionpack (>= 6.0.0) @@ -486,6 +487,7 @@ DEPENDENCIES scss_lint-govuk selenium-webdriver simplecov + timecop (~> 0.9.4) two_factor_authentication! tzinfo-data uk_postcode diff --git a/app/services/exports/case_log_export_service.rb b/app/services/exports/case_log_export_service.rb new file mode 100644 index 000000000..27d790ceb --- /dev/null +++ b/app/services/exports/case_log_export_service.rb @@ -0,0 +1,75 @@ +# Temporary instructions for reference +# (to be updated on feedback and deleted when implemented) +# +# create manifest file (one per run), have to be daily even with no data +# manifest => Manifest_2022_01_30_0001(i).csv +# folder_name / timestamp / full_path +# +# folder => core_year_month_f0001 (use day for now) +# file => dat_core_year_month_f0001_0001(i).xml (increment matches manifest for a given day) +# +# [Manifest generation] +# iterate manifests for the day and determine next increment +# read previous manifest if present (read timestamp / reuse folder name) +# otherwise determine next folder for the month +# hold writing manifest until we checked for data +# +# [Retrieve case logs] +# Find all case logs updates from last run of the day (midnight if none) +# Write manifest +# Determine next increment for the folder (inc = 1 if none) +# Export retrieved case logs into XML file +# +# [Zipping mechanism left for later] + +module Exports + class CaseLogExportService + def initialize(storage_service, logger = Rails.logger) + @storage_service = storage_service + @logger = logger + end + + def export_case_logs + case_logs = retrieve_case_logs + string_io = build_export_string_io(case_logs) + file_path = "#{get_folder_name}/#{get_file_name}.xml" + @storage_service.write_file(file_path, string_io) + end + + private + + def retrieve_case_logs + params = { from: Time.current.beginning_of_day, to: Time.current, status: CaseLog.statuses[:completed] } + CaseLog.where("updated_at >= :from and updated_at <= :to and status = :status", params) + end + + def build_export_string_io(case_logs) + doc = Nokogiri::XML("") + + case_logs.each do |case_log| + form = doc.create_element("form") + doc.at("forms") << form + case_log.attributes.each do |key, value| + form << doc.create_element(key, value) + end + end + doc.write_xml_to(StringIO.new, encoding: "UTF-8") + end + + def get_folder_name + "core_#{day_as_string}" + end + + def get_file_name + "dat_core_#{day_as_string}_#{increment_as_string}" + end + + def day_as_string + Time.current.strftime("%Y_%m_%d") + end + + def increment_as_string(increment = 1) + sprintf("%04d", increment) + end + end +end diff --git a/lib/tasks/data_export.rake b/lib/tasks/data_export.rake new file mode 100644 index 000000000..748a3009e --- /dev/null +++ b/lib/tasks/data_export.rake @@ -0,0 +1,7 @@ +namespace :core do + desc "Export data XMLs for import into Central Data System (CDS)" + task data_export: :environment do + storage_service = StorageService.new(PaasConfigurationService.new, ENV["EXPORT_PAAS_INSTANCE"]) + Exports::CaseLogExportService.new(storage_service).export_case_logs + end +end diff --git a/lib/tasks/data_import.rake b/lib/tasks/data_import.rake index 963782b77..7ac0a7680 100644 --- a/lib/tasks/data_import.rake +++ b/lib/tasks/data_import.rake @@ -1,5 +1,3 @@ -require "nokogiri" - namespace :core do desc "Import data XMLs from Softwire system" task :data_import, %i[type path] => :environment do |_task, args| diff --git a/spec/fixtures/exports/case_logs.xml b/spec/fixtures/exports/case_logs.xml new file mode 100644 index 000000000..1224b71d0 --- /dev/null +++ b/spec/fixtures/exports/case_logs.xml @@ -0,0 +1,325 @@ + + +
+ {id_1} + completed + 2022-02-08 16:52:15 +0000 + 2022-02-08 16:52:15 +0000 + BZ737 + 35 + Female + White: Irish + Estonia + Private sector tenancy + Part-time - Less than 30 hours + 2 + Partner + 32 + Male + Not seeking work + + + + + + + + + + + + + + + + + + + + + + + + + Yes - other homelessness + No + No - they left up to 5 years ago + No + Yes + Yes + No + dummy + BZ757 + No + 5 + Secure (including flexible) + This landlord + SE2 6RT + Tenant abandoned property + House + 3 + 2 + Yes + Weekly + Some + Every 2 weeks + 1 to 2 years + Less than 1 year + NW1 5TY + Yes + dummy + Yes + Yes + No + + Yes + No + No + No + No + No + No + No + Yes + No + No + No + No + No + No + No + No + Yes + No + No + No + No + + + + Yes + No + Test + Test + + 1 + 1 + 2 + 798794 + Permanently decanted from another property owned by this landlord + 123 + Yes + Barnet + Ashford + Housing benefit + Yes + NW1 + 5TY + SE2 + 6RT + No + 2022-02-08 16:52:15 +0000 + 8 + 2 + 2022 + 1 + + + 2022-02-08 16:52:15 +0000 + A current or former regular in the UK Armed Forces (excluding National Service) + + Affordable rent basis + Purpose built + 2019-11-03 00:00:00 +0000 + {owning_org_id_1} + {managing_org_id_1} + + General needs + Affordable Rent General needs LA + Yes + Yes + false + 8 + 2 + 2022 + 0 + 0 + 2 + Weekly + Yes + No + No + No + Yes + 68 + + 200.0 + 50.0 + 40.0 + 35.0 + 325.0 + 12.0 + 7.0 + +
+ {id_2} + completed + 2022-02-08 16:52:15 +0000 + 2022-02-08 16:52:15 +0000 + BZ737 + 35 + Female + White: Irish + Estonia + Private sector tenancy + Part-time - Less than 30 hours + 2 + Partner + 32 + Male + Not seeking work + + + + + + + + + + + + + + + + + + + + + + + + + Yes - other homelessness + No + No - they left up to 5 years ago + No + Yes + Yes + No + dummy + BZ757 + No + 5 + Secure (including flexible) + This landlord + SE2 6RT + Tenant abandoned property + House + 3 + 2 + Yes + Weekly + Some + Every 2 weeks + 1 to 2 years + Less than 1 year + NW1 5TY + Yes + dummy + Yes + Yes + No + + Yes + No + No + No + No + No + No + No + Yes + No + No + No + No + No + No + No + No + Yes + No + No + No + No + + + + Yes + No + Test + Test + + 1 + 1 + 2 + 798794 + Permanently decanted from another property owned by this landlord + 123 + Yes + Barnet + Ashford + Housing benefit + Yes + NW1 + 5TY + SE2 + 6RT + No + 2022-02-08 16:52:15 +0000 + 8 + 2 + 2022 + 1 + + + 2022-02-08 16:52:15 +0000 + A current or former regular in the UK Armed Forces (excluding National Service) + + Affordable rent basis + Purpose built + 2019-11-03 00:00:00 +0000 + {owning_org_id_2} + {managing_org_id_2} + + General needs + Affordable Rent General needs LA + Yes + Yes + false + 8 + 2 + 2022 + 0 + 0 + 2 + Weekly + Yes + No + No + No + Yes + 68 + + 200.0 + 50.0 + 40.0 + 35.0 + 325.0 + 12.0 + 7.0 + +
 diff --git a/spec/lib/tasks/data_export_spec.rb b/spec/lib/tasks/data_export_spec.rb new file mode 100644 index 000000000..bec366c6c --- /dev/null +++ b/spec/lib/tasks/data_export_spec.rb @@ -0,0 +1,33 @@ +require "rails_helper" +require "rake" + +describe "rake core:data_export", type: task do + subject(:task) { Rake::Task["core:data_export"] } + + let(:paas_instance) { "paas_export_instance" } + let(:storage_service) { instance_double(StorageService) } + let(:paas_config_service) { instance_double(PaasConfigurationService) } + let(:export_service) { instance_double(Exports::CaseLogExportService) } + + before do + Rake.application.rake_require("tasks/data_export") + Rake::Task.define_task(:environment) + task.reenable + + allow(StorageService).to receive(:new).and_return(storage_service) + allow(PaasConfigurationService).to receive(:new).and_return(paas_config_service) + allow(Exports::CaseLogExportService).to receive(:new).and_return(export_service) + allow(ENV).to receive(:[]) + allow(ENV).to receive(:[]).with("EXPORT_PAAS_INSTANCE").and_return(paas_instance) + end + + context "when exporting case logs" do + it "starts the export process" do + expect(StorageService).to receive(:new).with(paas_config_service, paas_instance) + expect(Exports::CaseLogExportService).to receive(:new).with(storage_service) + expect(export_service).to receive(:export_case_logs) + + task.invoke + end + end +end diff --git a/spec/lib/tasks/data_import_spec.rb b/spec/lib/tasks/data_import_spec.rb index 22dd7089c..9749ce27f 100644 --- a/spec/lib/tasks/data_import_spec.rb +++ b/spec/lib/tasks/data_import_spec.rb @@ -5,7 +5,7 @@ describe "rake core:data_import", type: :task do subject(:task) { Rake::Task["core:data_import"] } let(:fixture_path) { "spec/fixtures/softwire_imports/organisations" } - let(:instance_name) { "my_instance" } + let(:instance_name) { "paas_import_instance" } let(:organisation_type) { "organisation" } let(:storage_service) { instance_double(StorageService) } diff --git a/spec/services/exports/case_log_export_service_spec.rb b/spec/services/exports/case_log_export_service_spec.rb new file mode 100644 index 000000000..428912c48 --- /dev/null +++ b/spec/services/exports/case_log_export_service_spec.rb @@ -0,0 +1,42 @@ +require "rails_helper" + +RSpec.describe Exports::CaseLogExportService do + let(:storage_service) { instance_double(StorageService) } + let(:export_filepath) { "spec/fixtures/exports/case_logs.xml" } + let(:export_file) { File.open(export_filepath, "r:UTF-8") } + let(:expected_filename) { "core_2022_02_08/dat_core_2022_02_08_0001.xml" } + let(:case_logs) { FactoryBot.create_list(:case_log, 2, :completed) } + + def replace_entity_ids(export_template) + export_template.sub!(/\{id_1\}/, case_logs[0]["id"].to_s) + export_template.sub!(/\{id_2\}/, case_logs[1]["id"].to_s) + export_template.sub!(/\{owning_org_id_1\}/, case_logs[0]["owning_organisation_id"].to_s) + export_template.sub!(/\{owning_org_id_2\}/, case_logs[1]["owning_organisation_id"].to_s) + export_template.sub!(/\{managing_org_id_1\}/, case_logs[0]["managing_organisation_id"].to_s) + export_template.sub!(/\{managing_org_id_2\}/, case_logs[1]["managing_organisation_id"].to_s) + end + + context "when exporting case logs" do + subject(:export_service) { described_class.new(storage_service) } + + before do + Timecop.freeze(Time.new(2022, 2, 8, 16, 52, 15, "+00:00")) + case_logs + end + + it "generate an XML export file with the expected filename" do + actual_filename = nil + allow(storage_service).to receive(:write_file) { |filename, _| actual_filename = filename } + export_service.export_case_logs + expect(actual_filename).to eq(expected_filename) + end + + it "generate an XML export file with the expected content" do + actual_stringio = nil + allow(storage_service).to receive(:write_file) { |_, stringio| actual_stringio = stringio } + actual_content = replace_entity_ids(export_file.read) + export_service.export_case_logs + expect(actual_stringio&.string).to eq(actual_content) + end + end +end