From 76fc8eb5c6d02288a7d5f3ed95dc9f498abf8fb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Meny?= Date: Tue, 5 Apr 2022 17:05:36 +0100 Subject: [PATCH] CLDC-1118: Implement empty master manifest --- app/models/logs_export.rb | 2 + .../exports/case_log_export_service.rb | 66 +++++++++++-------- .../20220401094457_create_logs_export.rb | 9 +++ db/schema.rb | 7 +- .../exports/case_log_export_service_spec.rb | 65 ++++++++++++++---- 5 files changed, 107 insertions(+), 42 deletions(-) create mode 100644 app/models/logs_export.rb create mode 100644 db/migrate/20220401094457_create_logs_export.rb diff --git a/app/models/logs_export.rb b/app/models/logs_export.rb new file mode 100644 index 000000000..0ae2e4179 --- /dev/null +++ b/app/models/logs_export.rb @@ -0,0 +1,2 @@ +class LogsExport < ApplicationRecord +end diff --git a/app/services/exports/case_log_export_service.rb b/app/services/exports/case_log_export_service.rb index 258a3ed8b..c8ad2dd8a 100644 --- a/app/services/exports/case_log_export_service.rb +++ b/app/services/exports/case_log_export_service.rb @@ -1,27 +1,3 @@ -# Temporary instructions for reference -# (to be updated on feedback and deleted when implemented) -# -# create manifest file (one per run), have to be daily even with no data -# manifest => Manifest_2022_01_30_0001(i).csv -# folder_name / timestamp / full_path -# -# folder => core_year_month_f0001 (use day for now) -# file => dat_core_year_month_f0001_0001(i).xml (increment matches manifest for a given day) -# -# [Manifest generation] -# iterate manifests for the day and determine next increment -# read previous manifest if present (read timestamp / reuse folder name) -# otherwise determine next folder for the month -# hold writing manifest until we checked for data -# -# [Retrieve case logs] -# Find all case logs updates from last run of the day (midnight if none) -# Write manifest -# Determine next increment for the folder (inc = 1 if none) -# Export retrieved case logs into XML file -# -# [Zipping mechanism left for later] - module Exports class CaseLogExportService def initialize(storage_service, logger = Rails.logger) @@ -31,9 +7,9 @@ module Exports def export_case_logs case_logs = retrieve_case_logs - string_io = build_export_string_io(case_logs) - file_path = "#{get_folder_name}/#{get_file_name}.xml" - @storage_service.write_file(file_path, string_io) + export = save_export_run + write_master_manifest(export) + write_export_data(case_logs) end def is_omitted_field?(field_name) @@ -44,12 +20,46 @@ module Exports private + def save_export_run + today = Time.zone.today + last_daily_run_number = LogsExport.where(created_at: today.beginning_of_day..today.end_of_day).maximum(:daily_run_number) + last_daily_run_number = 0 if last_daily_run_number.nil? + + export = LogsExport.new + export.daily_run_number = last_daily_run_number + 1 + export.save! + export + end + + def write_master_manifest(export) + today = Time.zone.today + increment_number = export.daily_run_number.to_s.rjust(4, "0") + month = today.month.to_s.rjust(2, "0") + file_path = "Manifest_#{today.year}_#{month}_#{today.day}_#{increment_number}.csv" + string_io = build_manifest_csv_io + @storage_service.write_file(file_path, string_io) + end + + def write_export_data(case_logs) + string_io = build_export_xml_io(case_logs) + file_path = "#{get_folder_name}/#{get_file_name}.xml" + @storage_service.write_file(file_path, string_io) + end + def retrieve_case_logs params = { from: Time.current.beginning_of_day, to: Time.current, status: CaseLog.statuses[:completed] } CaseLog.where("updated_at >= :from and updated_at <= :to and status = :status", params) end - def build_export_string_io(case_logs) + def build_manifest_csv_io + headers = ["zip-name", "date-time zipped folder generated", "zip-file-uri"] + csv_string = CSV.generate do |csv| + csv << headers + end + StringIO.new(csv_string) + end + + def build_export_xml_io(case_logs) doc = Nokogiri::XML("") case_logs.each do |case_log| diff --git a/db/migrate/20220401094457_create_logs_export.rb b/db/migrate/20220401094457_create_logs_export.rb new file mode 100644 index 000000000..b96a5d12e --- /dev/null +++ b/db/migrate/20220401094457_create_logs_export.rb @@ -0,0 +1,9 @@ +class CreateLogsExport < ActiveRecord::Migration[7.0] + def change + create_table :logs_exports do |t| + t.integer :daily_run_number + + t.datetime :created_at, default: -> { "CURRENT_TIMESTAMP" } + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 965fcee3d..4415c98eb 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -264,6 +264,11 @@ ActiveRecord::Schema[7.0].define(version: 202202071123100) do t.index ["start_year", "lettype", "beds", "la"], name: "index_la_rent_ranges_on_start_year_and_lettype_and_beds_and_la", unique: true end + create_table "logs_exports", force: :cascade do |t| + t.integer "daily_run_number" + t.datetime "created_at", default: -> { "CURRENT_TIMESTAMP" } + end + create_table "organisations", force: :cascade do |t| t.string "name" t.string "phone" @@ -313,12 +318,12 @@ ActiveRecord::Schema[7.0].define(version: 202202071123100) do t.string "last_sign_in_ip" t.integer "role" t.string "old_user_id" + t.string "phone" t.integer "failed_attempts", default: 0 t.string "unlock_token" t.datetime "locked_at", precision: nil t.boolean "is_dpo", default: false t.boolean "is_key_contact", default: false - t.string "phone" t.index ["email"], name: "index_users_on_email", unique: true t.index ["organisation_id"], name: "index_users_on_organisation_id" t.index ["reset_password_token"], name: "index_users_on_reset_password_token", unique: true diff --git a/spec/services/exports/case_log_export_service_spec.rb b/spec/services/exports/case_log_export_service_spec.rb index dfea4c1a9..01aab3cb1 100644 --- a/spec/services/exports/case_log_export_service_spec.rb +++ b/spec/services/exports/case_log_export_service_spec.rb @@ -2,9 +2,15 @@ require "rails_helper" RSpec.describe Exports::CaseLogExportService do let(:storage_service) { instance_double(StorageService) } + let(:export_filepath) { "spec/fixtures/exports/case_logs.xml" } let(:export_file) { File.open(export_filepath, "r:UTF-8") } - let(:expected_filename) { "core_2022_02_08/dat_core_2022_02_08_0001.xml" } + + let(:expected_data_filename) { "core_2022_02_08/dat_core_2022_02_08_0001" } + let(:expected_master_manifest_filename) { "Manifest_2022_02_08_0001.csv" } + let(:expected_master_manifest_filename2) { "Manifest_2022_02_08_0002.csv" } + + let(:case_log) { FactoryBot.create(:case_log, :completed) } def replace_entity_ids(export_template) export_template.sub!(/\{id\}/, case_log["id"].to_s) @@ -12,28 +18,61 @@ RSpec.describe Exports::CaseLogExportService do export_template.sub!(/\{managing_org_id\}/, case_log["managing_organisation_id"].to_s) end - context "when exporting case logs" do + context "when exporting daily case logs" do subject(:export_service) { described_class.new(storage_service) } let(:case_log) { FactoryBot.create(:case_log, :completed) } before do Timecop.freeze(case_log.updated_at) + allow(storage_service).to receive(:write_file) end - it "generate an XML export file with the expected filename" do - actual_filename = nil - allow(storage_service).to receive(:write_file) { |filename, _| actual_filename = filename } - export_service.export_case_logs - expect(actual_filename).to eq(expected_filename) + context "and no case logs is available for export" do + it "generates a master manifest with the correct name" do + expect(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) + export_service.export_case_logs + end + + it "generates a master manifest with CSV headers but no data" do + actual_content = nil + expected_content = "zip-name,date-time zipped folder generated,zip-file-uri\n" + allow(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) { |_, arg2| actual_content = arg2&.string } + + export_service.export_case_logs + expect(actual_content).to eq(expected_content) + end end - it "generate an XML export file with the expected content" do - actual_stringio = nil - allow(storage_service).to receive(:write_file) { |_, stringio| actual_stringio = stringio } - actual_content = replace_entity_ids(export_file.read) - export_service.export_case_logs - expect(actual_stringio&.string).to eq(actual_content) + context "and case logs are available for export" do + before do + case_log + end + + it "generates an XML export file with the expected filename" do + expect(storage_service).to receive(:write_file).with(expected_data_filename, any_args) + export_service.export_case_logs + end + + it "generates an XML export file with the expected content" do + actual_content = nil + expected_content = replace_entity_ids(export_file.read) + allow(storage_service).to receive(:write_file).with(expected_data_filename, any_args) { |_, arg2| actual_content = arg2&.string } + + export_service.export_case_logs + expect(actual_content).to eq(expected_content) + end + end + + context "and a previous export has run the same day" do + before do + export_service.export_case_logs + end + + it "increments the master manifest number by 1" do + expect(storage_service).to receive(:write_file).with(expected_master_manifest_filename2, any_args) + export_service.export_case_logs + end end end end