Browse Source
* Refactor manifest creation into a separate job * Add users export service * Call user export service * Rename exports table * Update data export job spec * Update naming * Refactor shared logic into parent class * Update initialize * Allow exporting users individually * Update data export task tests * Move method and update task argument * Add phone extension to the user export * Add static period to filename * Make recent logs export depend on the collection * CLDC-3534 Export organisation data (#2599) * Add organisation export service * Call organisations export and write manifest * Add some additional fields to export * Add period to organisation export filename * Update provider_type and add new fields * Filter exports by the collection * Update tests * Update fields exported in lettings export (#2652) * Add new fields for user ids (#2661) * Undo lettings export field changespull/2669/head
kosiakkatrina
3 months ago
committed by
GitHub
21 changed files with 1266 additions and 236 deletions
@ -1,2 +0,0 @@ |
|||||||
class LogsExport < ApplicationRecord |
|
||||||
end |
|
||||||
@ -0,0 +1,78 @@ |
|||||||
|
module Exports |
||||||
|
class ExportService |
||||||
|
include CollectionTimeHelper |
||||||
|
|
||||||
|
def initialize(storage_service, logger = Rails.logger) |
||||||
|
@storage_service = storage_service |
||||||
|
@logger = logger |
||||||
|
end |
||||||
|
|
||||||
|
def export_xml(full_update: false, collection: nil) |
||||||
|
start_time = Time.zone.now |
||||||
|
daily_run_number = get_daily_run_number |
||||||
|
lettings_archives_for_manifest = {} |
||||||
|
users_archives_for_manifest = {} |
||||||
|
organisations_archives_for_manifest = {} |
||||||
|
|
||||||
|
if collection.present? |
||||||
|
case collection |
||||||
|
when "users" |
||||||
|
users_archives_for_manifest = get_user_archives(start_time, full_update) |
||||||
|
when "organisations" |
||||||
|
organisations_archives_for_manifest = get_organisation_archives(start_time, full_update) |
||||||
|
else |
||||||
|
lettings_archives_for_manifest = get_lettings_archives(start_time, full_update, collection) |
||||||
|
end |
||||||
|
else |
||||||
|
users_archives_for_manifest = get_user_archives(start_time, full_update) |
||||||
|
organisations_archives_for_manifest = get_organisation_archives(start_time, full_update) |
||||||
|
lettings_archives_for_manifest = get_lettings_archives(start_time, full_update, collection) |
||||||
|
end |
||||||
|
|
||||||
|
write_master_manifest(daily_run_number, lettings_archives_for_manifest.merge(users_archives_for_manifest).merge(organisations_archives_for_manifest)) |
||||||
|
end |
||||||
|
|
||||||
|
private |
||||||
|
|
||||||
|
def get_daily_run_number |
||||||
|
today = Time.zone.today |
||||||
|
Export.where(created_at: today.beginning_of_day..today.end_of_day).select(:started_at).distinct.count + 1 |
||||||
|
end |
||||||
|
|
||||||
|
def write_master_manifest(daily_run, archive_datetimes) |
||||||
|
today = Time.zone.today |
||||||
|
increment_number = daily_run.to_s.rjust(4, "0") |
||||||
|
month = today.month.to_s.rjust(2, "0") |
||||||
|
day = today.day.to_s.rjust(2, "0") |
||||||
|
file_path = "Manifest_#{today.year}_#{month}_#{day}_#{increment_number}.csv" |
||||||
|
string_io = build_manifest_csv_io(archive_datetimes) |
||||||
|
@storage_service.write_file(file_path, string_io) |
||||||
|
end |
||||||
|
|
||||||
|
def build_manifest_csv_io(archive_datetimes) |
||||||
|
headers = ["zip-name", "date-time zipped folder generated", "zip-file-uri"] |
||||||
|
csv_string = CSV.generate do |csv| |
||||||
|
csv << headers |
||||||
|
archive_datetimes.each do |(archive, datetime)| |
||||||
|
csv << [archive, datetime, "#{archive}.zip"] |
||||||
|
end |
||||||
|
end |
||||||
|
StringIO.new(csv_string) |
||||||
|
end |
||||||
|
|
||||||
|
def get_user_archives(start_time, full_update) |
||||||
|
users_export_service = Exports::UserExportService.new(@storage_service, start_time) |
||||||
|
users_export_service.export_xml_users(full_update:) |
||||||
|
end |
||||||
|
|
||||||
|
def get_organisation_archives(start_time, full_update) |
||||||
|
organisations_export_service = Exports::OrganisationExportService.new(@storage_service, start_time) |
||||||
|
organisations_export_service.export_xml_organisations(full_update:) |
||||||
|
end |
||||||
|
|
||||||
|
def get_lettings_archives(start_time, full_update, collection) |
||||||
|
lettings_export_service = Exports::LettingsLogExportService.new(@storage_service, start_time) |
||||||
|
lettings_export_service.export_xml_lettings_logs(full_update:, collection_year: collection) |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
@ -0,0 +1,27 @@ |
|||||||
|
module Exports::OrganisationExportConstants |
||||||
|
MAX_XML_RECORDS = 10_000 |
||||||
|
|
||||||
|
EXPORT_FIELDS = Set[ |
||||||
|
"id", |
||||||
|
"name", |
||||||
|
"phone", |
||||||
|
"provider_type", |
||||||
|
"address_line1", |
||||||
|
"address_line2", |
||||||
|
"postcode", |
||||||
|
"holds_own_stock", |
||||||
|
"housing_registration_no", |
||||||
|
"active", |
||||||
|
"old_org_id", |
||||||
|
"old_visible_id", |
||||||
|
"merge_date", |
||||||
|
"absorbing_organisation_id", |
||||||
|
"available_from", |
||||||
|
"deleted_at", |
||||||
|
"dsa_signed", |
||||||
|
"dsa_signed_at", |
||||||
|
"dpo_email", |
||||||
|
"profit_status", |
||||||
|
"group" |
||||||
|
] |
||||||
|
end |
||||||
@ -0,0 +1,72 @@ |
|||||||
|
module Exports |
||||||
|
class OrganisationExportService < Exports::XmlExportService |
||||||
|
include Exports::OrganisationExportConstants |
||||||
|
include CollectionTimeHelper |
||||||
|
|
||||||
|
def export_xml_organisations(full_update: false) |
||||||
|
collection = "organisations" |
||||||
|
recent_export = Export.where(collection:).order("started_at").last |
||||||
|
|
||||||
|
base_number = Export.where(empty_export: false, collection:).maximum(:base_number) || 1 |
||||||
|
export = build_export_run(collection, base_number, full_update) |
||||||
|
archives_for_manifest = write_export_archive(export, collection, recent_export, full_update) |
||||||
|
|
||||||
|
export.empty_export = archives_for_manifest.empty? |
||||||
|
export.save! |
||||||
|
|
||||||
|
archives_for_manifest |
||||||
|
end |
||||||
|
|
||||||
|
private |
||||||
|
|
||||||
|
def get_archive_name(collection, base_number, increment) |
||||||
|
return unless collection |
||||||
|
|
||||||
|
base_number_str = "f#{base_number.to_s.rjust(4, '0')}" |
||||||
|
increment_str = "inc#{increment.to_s.rjust(4, '0')}" |
||||||
|
"#{collection}_2024_2025_apr_mar_#{base_number_str}_#{increment_str}".downcase |
||||||
|
end |
||||||
|
|
||||||
|
def retrieve_resources(recent_export, full_update, _collection) |
||||||
|
if !full_update && recent_export |
||||||
|
params = { from: recent_export.started_at, to: @start_time } |
||||||
|
Organisation.where("(updated_at >= :from AND updated_at <= :to)", params) |
||||||
|
else |
||||||
|
params = { to: @start_time } |
||||||
|
Organisation.where("updated_at <= :to", params) |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
def build_export_xml(organisations) |
||||||
|
doc = Nokogiri::XML("<forms/>") |
||||||
|
|
||||||
|
organisations.each do |organisation| |
||||||
|
attribute_hash = apply_cds_transformation(organisation) |
||||||
|
form = doc.create_element("form") |
||||||
|
doc.at("forms") << form |
||||||
|
attribute_hash.each do |key, value| |
||||||
|
if !EXPORT_FIELDS.include?(key) |
||||||
|
next |
||||||
|
else |
||||||
|
form << doc.create_element(key, value) |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
xml_doc_to_temp_file(doc) |
||||||
|
end |
||||||
|
|
||||||
|
def apply_cds_transformation(organisation) |
||||||
|
attribute_hash = organisation.attributes |
||||||
|
attribute_hash["deleted_at"] = organisation.discarded_at |
||||||
|
attribute_hash["dsa_signed"] = organisation.data_protection_confirmed? |
||||||
|
attribute_hash["dsa_signed_at"] = organisation.data_protection_confirmation&.signed_at |
||||||
|
attribute_hash["dpo_email"] = organisation.data_protection_confirmation&.data_protection_officer_email |
||||||
|
attribute_hash["provider_type"] = organisation.provider_type_before_type_cast |
||||||
|
attribute_hash["profit_status"] = nil # will need update when we add the field to the org |
||||||
|
attribute_hash["group"] = nil # will need update when we add the field to the org |
||||||
|
|
||||||
|
attribute_hash |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
@ -0,0 +1,18 @@ |
|||||||
|
module Exports::UserExportConstants |
||||||
|
MAX_XML_RECORDS = 10_000 |
||||||
|
|
||||||
|
EXPORT_FIELDS = Set[ |
||||||
|
"id", |
||||||
|
"email", |
||||||
|
"name", |
||||||
|
"phone", |
||||||
|
"organisation_id", |
||||||
|
"organisation_name", |
||||||
|
"role", |
||||||
|
"is_dpo", |
||||||
|
"is_key_contact", |
||||||
|
"active", |
||||||
|
"sign_in_count", |
||||||
|
"last_sign_in_at", |
||||||
|
] |
||||||
|
end |
||||||
@ -0,0 +1,68 @@ |
|||||||
|
module Exports |
||||||
|
class UserExportService < Exports::XmlExportService |
||||||
|
include Exports::UserExportConstants |
||||||
|
include CollectionTimeHelper |
||||||
|
|
||||||
|
def export_xml_users(full_update: false) |
||||||
|
collection = "users" |
||||||
|
recent_export = Export.where(collection:).order("started_at").last |
||||||
|
|
||||||
|
base_number = Export.where(empty_export: false, collection:).maximum(:base_number) || 1 |
||||||
|
export = build_export_run(collection, base_number, full_update) |
||||||
|
archives_for_manifest = write_export_archive(export, collection, recent_export, full_update) |
||||||
|
|
||||||
|
export.empty_export = archives_for_manifest.empty? |
||||||
|
export.save! |
||||||
|
|
||||||
|
archives_for_manifest |
||||||
|
end |
||||||
|
|
||||||
|
private |
||||||
|
|
||||||
|
def get_archive_name(collection, base_number, increment) |
||||||
|
return unless collection |
||||||
|
|
||||||
|
base_number_str = "f#{base_number.to_s.rjust(4, '0')}" |
||||||
|
increment_str = "inc#{increment.to_s.rjust(4, '0')}" |
||||||
|
"#{collection}_2024_2025_apr_mar_#{base_number_str}_#{increment_str}".downcase |
||||||
|
end |
||||||
|
|
||||||
|
def retrieve_resources(recent_export, full_update, _collection) |
||||||
|
if !full_update && recent_export |
||||||
|
params = { from: recent_export.started_at, to: @start_time } |
||||||
|
User.where("(updated_at >= :from AND updated_at <= :to)", params) |
||||||
|
else |
||||||
|
params = { to: @start_time } |
||||||
|
User.where("updated_at <= :to", params) |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
def build_export_xml(users) |
||||||
|
doc = Nokogiri::XML("<forms/>") |
||||||
|
|
||||||
|
users.each do |user| |
||||||
|
attribute_hash = apply_cds_transformation(user) |
||||||
|
form = doc.create_element("form") |
||||||
|
doc.at("forms") << form |
||||||
|
attribute_hash.each do |key, value| |
||||||
|
if !EXPORT_FIELDS.include?(key) |
||||||
|
next |
||||||
|
else |
||||||
|
form << doc.create_element(key, value) |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
xml_doc_to_temp_file(doc) |
||||||
|
end |
||||||
|
|
||||||
|
def apply_cds_transformation(user) |
||||||
|
attribute_hash = user.attributes_before_type_cast |
||||||
|
attribute_hash["role"] = user.role |
||||||
|
attribute_hash["organisation_name"] = user.organisation.name |
||||||
|
attribute_hash["active"] = user.active? |
||||||
|
attribute_hash["phone"] = [user.phone, user.phone_extension].compact.join(" ") |
||||||
|
attribute_hash |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
@ -0,0 +1,97 @@ |
|||||||
|
module Exports |
||||||
|
class XmlExportService |
||||||
|
include Exports::LettingsLogExportConstants |
||||||
|
include CollectionTimeHelper |
||||||
|
|
||||||
|
def initialize(storage_service, start_time, logger = Rails.logger) |
||||||
|
@storage_service = storage_service |
||||||
|
@logger = logger |
||||||
|
@start_time = start_time |
||||||
|
end |
||||||
|
|
||||||
|
private |
||||||
|
|
||||||
|
def build_export_run(collection, base_number, full_update) |
||||||
|
@logger.info("Building export run for #{collection}") |
||||||
|
previous_exports_with_data = Export.where(collection:, empty_export: false) |
||||||
|
|
||||||
|
increment_number = previous_exports_with_data.where(base_number:).maximum(:increment_number) || 1 |
||||||
|
|
||||||
|
if full_update |
||||||
|
base_number += 1 if Export.any? # Only increment when it's not the first run |
||||||
|
increment_number = 1 |
||||||
|
else |
||||||
|
increment_number += 1 |
||||||
|
end |
||||||
|
|
||||||
|
if previous_exports_with_data.empty? |
||||||
|
return Export.new(collection:, base_number:, started_at: @start_time) |
||||||
|
end |
||||||
|
|
||||||
|
Export.new(collection:, started_at: @start_time, base_number:, increment_number:) |
||||||
|
end |
||||||
|
|
||||||
|
def write_export_archive(export, collection, recent_export, full_update) |
||||||
|
archive = get_archive_name(collection, export.base_number, export.increment_number) # archive name would be the same for all logs because they're already filtered by year (?) |
||||||
|
|
||||||
|
initial_count = retrieve_resources(recent_export, full_update, collection).count |
||||||
|
@logger.info("Creating #{archive} - #{initial_count} resources") |
||||||
|
return {} if initial_count.zero? |
||||||
|
|
||||||
|
zip_file = Zip::File.open_buffer(StringIO.new) |
||||||
|
|
||||||
|
part_number = 1 |
||||||
|
last_processed_marker = nil |
||||||
|
count_after_export = 0 |
||||||
|
|
||||||
|
loop do |
||||||
|
slice = if last_processed_marker.present? |
||||||
|
retrieve_resources(recent_export, full_update, collection) |
||||||
|
.where("created_at > ?", last_processed_marker) |
||||||
|
.order(:created_at) |
||||||
|
.limit(MAX_XML_RECORDS).to_a |
||||||
|
else |
||||||
|
retrieve_resources(recent_export, full_update, collection) |
||||||
|
.order(:created_at) |
||||||
|
.limit(MAX_XML_RECORDS).to_a |
||||||
|
end |
||||||
|
|
||||||
|
break if slice.empty? |
||||||
|
|
||||||
|
data_xml = build_export_xml(slice) |
||||||
|
part_number_str = "pt#{part_number.to_s.rjust(3, '0')}" |
||||||
|
zip_file.add("#{archive}_#{part_number_str}.xml", data_xml) |
||||||
|
part_number += 1 |
||||||
|
last_processed_marker = slice.last.created_at |
||||||
|
count_after_export += slice.count |
||||||
|
@logger.info("Added #{archive}_#{part_number_str}.xml") |
||||||
|
end |
||||||
|
|
||||||
|
manifest_xml = build_manifest_xml(count_after_export) |
||||||
|
zip_file.add("manifest.xml", manifest_xml) |
||||||
|
|
||||||
|
# Required by S3 to avoid Aws::S3::Errors::BadDigest |
||||||
|
zip_io = zip_file.write_buffer |
||||||
|
zip_io.rewind |
||||||
|
@logger.info("Writing #{archive}.zip") |
||||||
|
@storage_service.write_file("#{archive}.zip", zip_io) |
||||||
|
{ archive => Time.zone.now } |
||||||
|
end |
||||||
|
|
||||||
|
def xml_doc_to_temp_file(xml_doc) |
||||||
|
file = Tempfile.new |
||||||
|
xml_doc.write_xml_to(file, encoding: "UTF-8") |
||||||
|
file.rewind |
||||||
|
file |
||||||
|
end |
||||||
|
|
||||||
|
def build_manifest_xml(record_number) |
||||||
|
doc = Nokogiri::XML("<report/>") |
||||||
|
doc.at("report") << doc.create_element("form-data-summary") |
||||||
|
doc.at("form-data-summary") << doc.create_element("records") |
||||||
|
doc.at("records") << doc.create_element("count-of-records", record_number) |
||||||
|
|
||||||
|
xml_doc_to_temp_file(doc) |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
@ -0,0 +1,5 @@ |
|||||||
|
class RenameExportTable < ActiveRecord::Migration[7.0] |
||||||
|
def change |
||||||
|
rename_table :logs_exports, :exports |
||||||
|
end |
||||||
|
end |
||||||
@ -0,0 +1,26 @@ |
|||||||
|
<?xml version="1.0" encoding="UTF-8"?> |
||||||
|
<forms> |
||||||
|
<form> |
||||||
|
<id>{id}</id> |
||||||
|
<name>MHCLG</name> |
||||||
|
<phone/> |
||||||
|
<provider_type>1</provider_type> |
||||||
|
<address_line1>2 Marsham Street</address_line1> |
||||||
|
<address_line2>London</address_line2> |
||||||
|
<postcode>SW1P 4DF</postcode> |
||||||
|
<holds_own_stock>true</holds_own_stock> |
||||||
|
<active>true</active> |
||||||
|
<housing_registration_no>1234</housing_registration_no> |
||||||
|
<old_org_id/> |
||||||
|
<old_visible_id/> |
||||||
|
<merge_date/> |
||||||
|
<absorbing_organisation_id/> |
||||||
|
<available_from/> |
||||||
|
<deleted_at/> |
||||||
|
<dsa_signed>true</dsa_signed> |
||||||
|
<dsa_signed_at>{dsa_signed_at}</dsa_signed_at> |
||||||
|
<dpo_email>{dpo_email}</dpo_email> |
||||||
|
<profit_status/> |
||||||
|
<group/> |
||||||
|
</form> |
||||||
|
</forms> |
||||||
@ -0,0 +1,17 @@ |
|||||||
|
<?xml version="1.0" encoding="UTF-8"?> |
||||||
|
<forms> |
||||||
|
<form> |
||||||
|
<id>{id}</id> |
||||||
|
<email>{email}</email> |
||||||
|
<name>Danny Rojas</name> |
||||||
|
<organisation_id>{organisation_id}</organisation_id> |
||||||
|
<sign_in_count>5</sign_in_count> |
||||||
|
<last_sign_in_at/> |
||||||
|
<role>data_provider</role> |
||||||
|
<phone>1234512345123 123</phone> |
||||||
|
<is_dpo>false</is_dpo> |
||||||
|
<is_key_contact>false</is_key_contact> |
||||||
|
<active>true</active> |
||||||
|
<organisation_name>MHCLG</organisation_name> |
||||||
|
</form> |
||||||
|
</forms> |
||||||
@ -0,0 +1,333 @@ |
|||||||
|
require "rails_helper" |
||||||
|
|
||||||
|
RSpec.describe Exports::ExportService do |
||||||
|
subject(:export_service) { described_class.new(storage_service) } |
||||||
|
|
||||||
|
let(:storage_service) { instance_double(Storage::S3Service) } |
||||||
|
let(:expected_master_manifest_filename) { "Manifest_2022_05_01_0001.csv" } |
||||||
|
let(:start_time) { Time.zone.local(2022, 5, 1) } |
||||||
|
let(:user) { FactoryBot.create(:user, email: "test1@example.com") } |
||||||
|
let(:organisations_export_service) { instance_double("Exports::OrganisationExportService", export_xml_organisations: {}) } |
||||||
|
let(:users_export_service) { instance_double("Exports::UserExportService", export_xml_users: {}) } |
||||||
|
|
||||||
|
before do |
||||||
|
Timecop.freeze(start_time) |
||||||
|
Singleton.__init__(FormHandler) |
||||||
|
allow(storage_service).to receive(:write_file) |
||||||
|
allow(Exports::LettingsLogExportService).to receive(:new).and_return(lettings_logs_export_service) |
||||||
|
allow(Exports::UserExportService).to receive(:new).and_return(users_export_service) |
||||||
|
allow(Exports::OrganisationExportService).to receive(:new).and_return(organisations_export_service) |
||||||
|
end |
||||||
|
|
||||||
|
after do |
||||||
|
Timecop.return |
||||||
|
end |
||||||
|
|
||||||
|
context "when exporting daily XMLs" do |
||||||
|
context "and no lettings archives get created in lettings logs export" do |
||||||
|
let(:lettings_logs_export_service) { instance_double("Exports::LettingsLogExportService", export_xml_lettings_logs: {}) } |
||||||
|
|
||||||
|
context "and no user or organisation archives get created in user export" do |
||||||
|
it "generates a master manifest with the correct name" do |
||||||
|
expect(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) |
||||||
|
export_service.export_xml |
||||||
|
end |
||||||
|
|
||||||
|
it "generates a master manifest with CSV headers but no data" do |
||||||
|
actual_content = nil |
||||||
|
expected_content = "zip-name,date-time zipped folder generated,zip-file-uri\n" |
||||||
|
allow(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) { |_, arg2| actual_content = arg2&.string } |
||||||
|
|
||||||
|
export_service.export_xml |
||||||
|
expect(actual_content).to eq(expected_content) |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
context "and one user archive gets created in user export" do |
||||||
|
let(:users_export_service) { instance_double("Exports::UserExportService", export_xml_users: { "some_user_file_base_name" => start_time }) } |
||||||
|
|
||||||
|
it "generates a master manifest with the correct name" do |
||||||
|
expect(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) |
||||||
|
export_service.export_xml |
||||||
|
end |
||||||
|
|
||||||
|
it "generates a master manifest with CSV headers and correct data" do |
||||||
|
actual_content = nil |
||||||
|
expected_content = "zip-name,date-time zipped folder generated,zip-file-uri\nsome_user_file_base_name,2022-05-01 00:00:00 +0100,some_user_file_base_name.zip\n" |
||||||
|
allow(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) { |_, arg2| actual_content = arg2&.string } |
||||||
|
|
||||||
|
export_service.export_xml |
||||||
|
expect(actual_content).to eq(expected_content) |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
context "and one organisation archive gets created in organisation export" do |
||||||
|
let(:organisations_export_service) { instance_double("Exports::OrganisationExportService", export_xml_organisations: { "some_organisation_file_base_name" => start_time }) } |
||||||
|
|
||||||
|
it "generates a master manifest with the correct name" do |
||||||
|
expect(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) |
||||||
|
export_service.export_xml |
||||||
|
end |
||||||
|
|
||||||
|
it "generates a master manifest with CSV headers and correct data" do |
||||||
|
actual_content = nil |
||||||
|
expected_content = "zip-name,date-time zipped folder generated,zip-file-uri\nsome_organisation_file_base_name,2022-05-01 00:00:00 +0100,some_organisation_file_base_name.zip\n" |
||||||
|
allow(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) { |_, arg2| actual_content = arg2&.string } |
||||||
|
|
||||||
|
export_service.export_xml |
||||||
|
expect(actual_content).to eq(expected_content) |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
context "and user and organisation archive gets created in organisation export" do |
||||||
|
let(:organisations_export_service) { instance_double("Exports::OrganisationExportService", export_xml_organisations: { "some_organisation_file_base_name" => start_time }) } |
||||||
|
let(:users_export_service) { instance_double("Exports::UserExportService", export_xml_users: { "some_user_file_base_name" => start_time }) } |
||||||
|
|
||||||
|
it "generates a master manifest with the correct name" do |
||||||
|
expect(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) |
||||||
|
export_service.export_xml |
||||||
|
end |
||||||
|
|
||||||
|
it "generates a master manifest with CSV headers and correct data" do |
||||||
|
actual_content = nil |
||||||
|
expected_content = "zip-name,date-time zipped folder generated,zip-file-uri\nsome_user_file_base_name,2022-05-01 00:00:00 +0100,some_user_file_base_name.zip\nsome_organisation_file_base_name,2022-05-01 00:00:00 +0100,some_organisation_file_base_name.zip\n" |
||||||
|
allow(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) { |_, arg2| actual_content = arg2&.string } |
||||||
|
|
||||||
|
export_service.export_xml |
||||||
|
expect(actual_content).to eq(expected_content) |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
context "and one lettings archive gets created in lettings logs export" do |
||||||
|
let(:lettings_logs_export_service) { instance_double("Exports::LettingsLogExportService", export_xml_lettings_logs: { "some_file_base_name" => start_time }) } |
||||||
|
|
||||||
|
context "and no user archives get created in user export" do |
||||||
|
it "generates a master manifest with the correct name" do |
||||||
|
expect(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) |
||||||
|
export_service.export_xml |
||||||
|
end |
||||||
|
|
||||||
|
it "generates a master manifest with CSV headers and correct data" do |
||||||
|
actual_content = nil |
||||||
|
expected_content = "zip-name,date-time zipped folder generated,zip-file-uri\nsome_file_base_name,2022-05-01 00:00:00 +0100,some_file_base_name.zip\n" |
||||||
|
allow(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) { |_, arg2| actual_content = arg2&.string } |
||||||
|
|
||||||
|
export_service.export_xml |
||||||
|
expect(actual_content).to eq(expected_content) |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
context "and one user archive gets created in user export" do |
||||||
|
let(:users_export_service) { instance_double("Exports::UserExportService", export_xml_users: { "some_user_file_base_name" => start_time }) } |
||||||
|
|
||||||
|
it "generates a master manifest with the correct name" do |
||||||
|
expect(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) |
||||||
|
export_service.export_xml |
||||||
|
end |
||||||
|
|
||||||
|
it "generates a master manifest with CSV headers and correct data" do |
||||||
|
actual_content = nil |
||||||
|
expected_content = "zip-name,date-time zipped folder generated,zip-file-uri\nsome_file_base_name,2022-05-01 00:00:00 +0100,some_file_base_name.zip\nsome_user_file_base_name,2022-05-01 00:00:00 +0100,some_user_file_base_name.zip\n" |
||||||
|
allow(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) { |_, arg2| actual_content = arg2&.string } |
||||||
|
|
||||||
|
export_service.export_xml |
||||||
|
expect(actual_content).to eq(expected_content) |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
context "and multiple lettings archives get created in lettings logs export" do |
||||||
|
let(:lettings_logs_export_service) { instance_double("Exports::LettingsLogExportService", export_xml_lettings_logs: { "some_file_base_name" => start_time, "second_file_base_name" => start_time }) } |
||||||
|
|
||||||
|
context "and no user archives get created in user export" do |
||||||
|
it "generates a master manifest with the correct name" do |
||||||
|
expect(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) |
||||||
|
export_service.export_xml |
||||||
|
end |
||||||
|
|
||||||
|
it "generates a master manifest with CSV headers and correct data" do |
||||||
|
actual_content = nil |
||||||
|
expected_content = "zip-name,date-time zipped folder generated,zip-file-uri\nsome_file_base_name,2022-05-01 00:00:00 +0100,some_file_base_name.zip\nsecond_file_base_name,2022-05-01 00:00:00 +0100,second_file_base_name.zip\n" |
||||||
|
allow(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) { |_, arg2| actual_content = arg2&.string } |
||||||
|
|
||||||
|
export_service.export_xml |
||||||
|
expect(actual_content).to eq(expected_content) |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
context "and multiple user archive gets created in user export" do |
||||||
|
let(:users_export_service) { instance_double("Exports::UserExportService", export_xml_users: { "some_user_file_base_name" => start_time, "second_user_file_base_name" => start_time }) } |
||||||
|
|
||||||
|
it "generates a master manifest with the correct name" do |
||||||
|
expect(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) |
||||||
|
export_service.export_xml |
||||||
|
end |
||||||
|
|
||||||
|
it "generates a master manifest with CSV headers and correct data" do |
||||||
|
actual_content = nil |
||||||
|
expected_content = "zip-name,date-time zipped folder generated,zip-file-uri\nsome_file_base_name,2022-05-01 00:00:00 +0100,some_file_base_name.zip\nsecond_file_base_name,2022-05-01 00:00:00 +0100,second_file_base_name.zip\nsome_user_file_base_name,2022-05-01 00:00:00 +0100,some_user_file_base_name.zip\nsecond_user_file_base_name,2022-05-01 00:00:00 +0100,second_user_file_base_name.zip\n" |
||||||
|
allow(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) { |_, arg2| actual_content = arg2&.string } |
||||||
|
|
||||||
|
export_service.export_xml |
||||||
|
expect(actual_content).to eq(expected_content) |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
context "and multiple user and organisation archives gets created in user export" do |
||||||
|
let(:users_export_service) { instance_double("Exports::UserExportService", export_xml_users: { "some_user_file_base_name" => start_time, "second_user_file_base_name" => start_time }) } |
||||||
|
let(:organisations_export_service) { instance_double("Exports::OrganisationExportService", export_xml_organisations: { "some_organisation_file_base_name" => start_time, "second_organisation_file_base_name" => start_time }) } |
||||||
|
|
||||||
|
it "generates a master manifest with the correct name" do |
||||||
|
expect(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) |
||||||
|
export_service.export_xml |
||||||
|
end |
||||||
|
|
||||||
|
it "generates a master manifest with CSV headers and correct data" do |
||||||
|
actual_content = nil |
||||||
|
expected_content = "zip-name,date-time zipped folder generated,zip-file-uri\nsome_file_base_name,2022-05-01 00:00:00 +0100,some_file_base_name.zip\nsecond_file_base_name,2022-05-01 00:00:00 +0100,second_file_base_name.zip\nsome_user_file_base_name,2022-05-01 00:00:00 +0100,some_user_file_base_name.zip\nsecond_user_file_base_name,2022-05-01 00:00:00 +0100,second_user_file_base_name.zip\nsome_organisation_file_base_name,2022-05-01 00:00:00 +0100,some_organisation_file_base_name.zip\nsecond_organisation_file_base_name,2022-05-01 00:00:00 +0100,second_organisation_file_base_name.zip\n" |
||||||
|
allow(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) { |_, arg2| actual_content = arg2&.string } |
||||||
|
|
||||||
|
export_service.export_xml |
||||||
|
expect(actual_content).to eq(expected_content) |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
context "when exporting specific lettings log collection" do |
||||||
|
context "and no lettings archives get created in lettings logs export" do |
||||||
|
let(:lettings_logs_export_service) { instance_double("Exports::LettingsLogExportService", export_xml_lettings_logs: {}) } |
||||||
|
|
||||||
|
context "and user archive gets created in user export" do |
||||||
|
let(:users_export_service) { instance_double("Exports::UserExportService", export_xml_users: { "some_user_file_base_name" => start_time }) } |
||||||
|
|
||||||
|
it "generates a master manifest with the correct name" do |
||||||
|
expect(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) |
||||||
|
export_service.export_xml(full_update: true, collection: "2022") |
||||||
|
end |
||||||
|
|
||||||
|
it "does not write user data" do |
||||||
|
actual_content = nil |
||||||
|
expected_content = "zip-name,date-time zipped folder generated,zip-file-uri\n" |
||||||
|
allow(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) { |_, arg2| actual_content = arg2&.string } |
||||||
|
|
||||||
|
export_service.export_xml(full_update: true, collection: "2022") |
||||||
|
expect(actual_content).to eq(expected_content) |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
context "and lettings archive gets created in lettings logs export" do |
||||||
|
let(:lettings_logs_export_service) { instance_double("Exports::LettingsLogExportService", export_xml_lettings_logs: { "some_file_base_name" => start_time }) } |
||||||
|
|
||||||
|
context "and user archive gets created in user export" do |
||||||
|
let(:users_export_service) { instance_double("Exports::UserExportService", export_xml_users: { "some_user_file_base_name" => start_time }) } |
||||||
|
|
||||||
|
it "generates a master manifest with the correct name" do |
||||||
|
expect(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) |
||||||
|
export_service.export_xml(full_update: true, collection: "2023") |
||||||
|
end |
||||||
|
|
||||||
|
it "does not write user data" do |
||||||
|
actual_content = nil |
||||||
|
expected_content = "zip-name,date-time zipped folder generated,zip-file-uri\nsome_file_base_name,2022-05-01 00:00:00 +0100,some_file_base_name.zip\n" |
||||||
|
allow(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) { |_, arg2| actual_content = arg2&.string } |
||||||
|
|
||||||
|
export_service.export_xml(full_update: true, collection: "2023") |
||||||
|
expect(actual_content).to eq(expected_content) |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
context "when exporting user collection" do |
||||||
|
context "and no user archives get created in users export" do |
||||||
|
context "and lettings log archive gets created in lettings logs export" do |
||||||
|
let(:lettings_logs_export_service) { instance_double("Exports::LettingsLogExportService", export_xml_lettings_logs: { "some_file_base_name" => start_time }) } |
||||||
|
|
||||||
|
it "generates a master manifest with the correct name" do |
||||||
|
expect(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) |
||||||
|
export_service.export_xml(full_update: true, collection: "users") |
||||||
|
end |
||||||
|
|
||||||
|
it "does not write lettings log data" do |
||||||
|
actual_content = nil |
||||||
|
expected_content = "zip-name,date-time zipped folder generated,zip-file-uri\n" |
||||||
|
allow(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) { |_, arg2| actual_content = arg2&.string } |
||||||
|
|
||||||
|
export_service.export_xml(full_update: true, collection: "users") |
||||||
|
expect(actual_content).to eq(expected_content) |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
context "and users archive gets created in users export" do |
||||||
|
let(:lettings_logs_export_service) { instance_double("Exports::LettingsLogExportService", export_xml_lettings_logs: { "some_file_base_name" => start_time }) } |
||||||
|
|
||||||
|
context "and lettings log archive gets created in lettings log export" do |
||||||
|
let(:users_export_service) { instance_double("Exports::UserExportService", export_xml_users: { "some_user_file_base_name" => start_time }) } |
||||||
|
|
||||||
|
it "generates a master manifest with the correct name" do |
||||||
|
expect(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) |
||||||
|
export_service.export_xml(full_update: true, collection: "users") |
||||||
|
end |
||||||
|
|
||||||
|
it "does not write lettings log data" do |
||||||
|
actual_content = nil |
||||||
|
expected_content = "zip-name,date-time zipped folder generated,zip-file-uri\nsome_user_file_base_name,2022-05-01 00:00:00 +0100,some_user_file_base_name.zip\n" |
||||||
|
allow(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) { |_, arg2| actual_content = arg2&.string } |
||||||
|
|
||||||
|
export_service.export_xml(full_update: true, collection: "users") |
||||||
|
expect(actual_content).to eq(expected_content) |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
context "when exporting organisation collection" do |
||||||
|
context "and no organisation archives get created in organisations export" do |
||||||
|
let(:organisations_export_service) { instance_double("Exports::OrganisationExportService", export_xml_organisations: {}) } |
||||||
|
|
||||||
|
context "and lettings log archive gets created in lettings logs export" do |
||||||
|
let(:lettings_logs_export_service) { instance_double("Exports::LettingsLogExportService", export_xml_lettings_logs: { "some_file_base_name" => start_time }) } |
||||||
|
|
||||||
|
it "generates a master manifest with the correct name" do |
||||||
|
expect(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) |
||||||
|
export_service.export_xml(full_update: true, collection: "organisations") |
||||||
|
end |
||||||
|
|
||||||
|
it "does not write lettings log data" do |
||||||
|
actual_content = nil |
||||||
|
expected_content = "zip-name,date-time zipped folder generated,zip-file-uri\n" |
||||||
|
allow(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) { |_, arg2| actual_content = arg2&.string } |
||||||
|
|
||||||
|
export_service.export_xml(full_update: true, collection: "organisations") |
||||||
|
expect(actual_content).to eq(expected_content) |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
context "and organisations archive gets created in organisations export" do |
||||||
|
let(:lettings_logs_export_service) { instance_double("Exports::LettingsLogExportService", export_xml_lettings_logs: { "some_file_base_name" => start_time }) } |
||||||
|
|
||||||
|
context "and lettings log archive gets created in lettings log export" do |
||||||
|
let(:organisations_export_service) { instance_double("Exports::OrganisationExportService", export_xml_organisations: { "some_organisation_file_base_name" => start_time }) } |
||||||
|
|
||||||
|
it "generates a master manifest with the correct name" do |
||||||
|
expect(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) |
||||||
|
export_service.export_xml(full_update: true, collection: "organisations") |
||||||
|
end |
||||||
|
|
||||||
|
it "does not write lettings log data" do |
||||||
|
actual_content = nil |
||||||
|
expected_content = "zip-name,date-time zipped folder generated,zip-file-uri\nsome_organisation_file_base_name,2022-05-01 00:00:00 +0100,some_organisation_file_base_name.zip\n" |
||||||
|
allow(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) { |_, arg2| actual_content = arg2&.string } |
||||||
|
|
||||||
|
export_service.export_xml(full_update: true, collection: "organisations") |
||||||
|
expect(actual_content).to eq(expected_content) |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
@ -0,0 +1,219 @@ |
|||||||
|
require "rails_helper" |
||||||
|
|
||||||
|
RSpec.describe Exports::OrganisationExportService do |
||||||
|
subject(:export_service) { described_class.new(storage_service, start_time) } |
||||||
|
|
||||||
|
let(:storage_service) { instance_double(Storage::S3Service) } |
||||||
|
|
||||||
|
let(:xml_export_file) { File.open("spec/fixtures/exports/organisation.xml", "r:UTF-8") } |
||||||
|
let(:local_manifest_file) { File.open("spec/fixtures/exports/manifest.xml", "r:UTF-8") } |
||||||
|
|
||||||
|
let(:expected_zip_filename) { "organisations_2024_2025_apr_mar_f0001_inc0001.zip" } |
||||||
|
let(:expected_data_filename) { "organisations_2024_2025_apr_mar_f0001_inc0001_pt001.xml" } |
||||||
|
let(:expected_manifest_filename) { "manifest.xml" } |
||||||
|
let(:start_time) { Time.zone.local(2022, 5, 1) } |
||||||
|
let(:organisation) { create(:organisation, with_dsa: false) } |
||||||
|
|
||||||
|
def replace_entity_ids(organisation, export_template) |
||||||
|
export_template.sub!(/\{id\}/, organisation["id"].to_s) |
||||||
|
export_template.sub!(/\{dsa_signed_at\}/, organisation.data_protection_confirmation&.signed_at.to_s) |
||||||
|
export_template.sub!(/\{dpo_email\}/, organisation.data_protection_confirmation&.data_protection_officer_email) |
||||||
|
end |
||||||
|
|
||||||
|
def replace_record_number(export_template, record_number) |
||||||
|
export_template.sub!(/\{recno\}/, record_number.to_s) |
||||||
|
end |
||||||
|
|
||||||
|
before do |
||||||
|
Timecop.freeze(start_time) |
||||||
|
Singleton.__init__(FormHandler) |
||||||
|
allow(storage_service).to receive(:write_file) |
||||||
|
end |
||||||
|
|
||||||
|
after do |
||||||
|
Timecop.return |
||||||
|
end |
||||||
|
|
||||||
|
context "when exporting daily organisations in XML" do |
||||||
|
context "and no organisations are available for export" do |
||||||
|
it "returns an empty archives list" do |
||||||
|
expect(export_service.export_xml_organisations).to eq({}) |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
context "and one organisation is available for export" do |
||||||
|
let!(:organisation) { create(:organisation) } |
||||||
|
|
||||||
|
it "generates a ZIP export file with the expected filename" do |
||||||
|
expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) |
||||||
|
export_service.export_xml_organisations |
||||||
|
end |
||||||
|
|
||||||
|
it "generates an XML export file with the expected filename within the ZIP file" do |
||||||
|
expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) do |_, content| |
||||||
|
entry = Zip::File.open_buffer(content).find_entry(expected_data_filename) |
||||||
|
expect(entry).not_to be_nil |
||||||
|
expect(entry.name).to eq(expected_data_filename) |
||||||
|
end |
||||||
|
export_service.export_xml_organisations |
||||||
|
end |
||||||
|
|
||||||
|
it "generates an XML manifest file with the expected content within the ZIP file" do |
||||||
|
expected_content = replace_record_number(local_manifest_file.read, 1) |
||||||
|
expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) do |_, content| |
||||||
|
entry = Zip::File.open_buffer(content).find_entry(expected_manifest_filename) |
||||||
|
expect(entry).not_to be_nil |
||||||
|
expect(entry.get_input_stream.read).to eq(expected_content) |
||||||
|
end |
||||||
|
|
||||||
|
export_service.export_xml_organisations |
||||||
|
end |
||||||
|
|
||||||
|
it "generates an XML export file with the expected content within the ZIP file" do |
||||||
|
expected_content = replace_entity_ids(organisation, xml_export_file.read) |
||||||
|
expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) do |_, content| |
||||||
|
entry = Zip::File.open_buffer(content).find_entry(expected_data_filename) |
||||||
|
expect(entry).not_to be_nil |
||||||
|
expect(entry.get_input_stream.read).to eq(expected_content) |
||||||
|
end |
||||||
|
|
||||||
|
export_service.export_xml_organisations |
||||||
|
end |
||||||
|
|
||||||
|
it "returns the list with correct archive" do |
||||||
|
expect(export_service.export_xml_organisations).to eq({ expected_zip_filename.gsub(".zip", "") => start_time }) |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
context "and multiple organisations are available for export" do |
||||||
|
before do |
||||||
|
create(:organisation) |
||||||
|
create(:organisation) |
||||||
|
end |
||||||
|
|
||||||
|
it "generates an XML manifest file with the expected content within the ZIP file" do |
||||||
|
expected_content = replace_record_number(local_manifest_file.read, 2) |
||||||
|
expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) do |_, content| |
||||||
|
entry = Zip::File.open_buffer(content).find_entry(expected_manifest_filename) |
||||||
|
expect(entry).not_to be_nil |
||||||
|
expect(entry.get_input_stream.read).to eq(expected_content) |
||||||
|
end |
||||||
|
|
||||||
|
export_service.export_xml_organisations |
||||||
|
end |
||||||
|
|
||||||
|
it "creates an export record in a database with correct time" do |
||||||
|
expect { export_service.export_xml_organisations } |
||||||
|
.to change(Export, :count).by(1) |
||||||
|
expect(Export.last.started_at).to be_within(2.seconds).of(start_time) |
||||||
|
end |
||||||
|
|
||||||
|
context "when this is the first export (full)" do |
||||||
|
it "returns a ZIP archive for the master manifest" do |
||||||
|
expect(export_service.export_xml_organisations).to eq({ expected_zip_filename.gsub(".zip", "").gsub(".zip", "") => start_time }) |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
context "and underlying data changes between getting the organisations and writting the manifest" do |
||||||
|
def remove_organisations(organisations) |
||||||
|
organisations.each(&:destroy) |
||||||
|
file = Tempfile.new |
||||||
|
doc = Nokogiri::XML("<forms/>") |
||||||
|
doc.write_xml_to(file, encoding: "UTF-8") |
||||||
|
file.rewind |
||||||
|
file |
||||||
|
end |
||||||
|
|
||||||
|
def create_fake_maifest |
||||||
|
file = Tempfile.new |
||||||
|
doc = Nokogiri::XML("<forms/>") |
||||||
|
doc.write_xml_to(file, encoding: "UTF-8") |
||||||
|
file.rewind |
||||||
|
file |
||||||
|
end |
||||||
|
|
||||||
|
it "maintains the same record number" do |
||||||
|
# rubocop:disable RSpec/SubjectStub |
||||||
|
allow(export_service).to receive(:build_export_xml) do |organisations| |
||||||
|
remove_organisations(organisations) |
||||||
|
end |
||||||
|
allow(export_service).to receive(:build_manifest_xml) do |
||||||
|
create_fake_maifest |
||||||
|
end |
||||||
|
|
||||||
|
expect(export_service).to receive(:build_manifest_xml).with(2) |
||||||
|
# rubocop:enable RSpec/SubjectStub |
||||||
|
export_service.export_xml_organisations |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
context "when this is a second export (partial)" do |
||||||
|
before do |
||||||
|
start_time = Time.zone.local(2022, 6, 1) |
||||||
|
Export.new(started_at: start_time, collection: "organisations").save! # this should be organisation export |
||||||
|
end |
||||||
|
|
||||||
|
it "does not add any entry for the master manifest (no organisations)" do |
||||||
|
expect(export_service.export_xml_organisations).to eq({}) |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
context "and a previous export has run the same day having organisations" do |
||||||
|
before do |
||||||
|
create(:organisation) |
||||||
|
export_service.export_xml_organisations |
||||||
|
end |
||||||
|
|
||||||
|
context "and we trigger another full update" do |
||||||
|
it "increments the base number" do |
||||||
|
export_service.export_xml_organisations(full_update: true) |
||||||
|
expect(Export.last.base_number).to eq(2) |
||||||
|
end |
||||||
|
|
||||||
|
it "resets the increment number" do |
||||||
|
export_service.export_xml_organisations(full_update: true) |
||||||
|
expect(Export.last.increment_number).to eq(1) |
||||||
|
end |
||||||
|
|
||||||
|
it "returns a correct archives list for manifest file" do |
||||||
|
expect(export_service.export_xml_organisations(full_update: true)).to eq({ "organisations_2024_2025_apr_mar_f0002_inc0001" => start_time }) |
||||||
|
end |
||||||
|
|
||||||
|
it "generates a ZIP export file with the expected filename" do |
||||||
|
expect(storage_service).to receive(:write_file).with("organisations_2024_2025_apr_mar_f0002_inc0001.zip", any_args) |
||||||
|
export_service.export_xml_organisations(full_update: true) |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
context "and a previous export has run having no organisations" do |
||||||
|
before { export_service.export_xml_organisations } |
||||||
|
|
||||||
|
it "doesn't increment the manifest number by 1" do |
||||||
|
export_service.export_xml_organisations |
||||||
|
|
||||||
|
expect(Export.last.increment_number).to eq(1) |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
context "and an organisation has been migrated since the previous partial export" do |
||||||
|
before do |
||||||
|
create(:organisation, updated_at: Time.zone.local(2022, 4, 27)) |
||||||
|
create(:organisation, updated_at: Time.zone.local(2022, 4, 27)) |
||||||
|
Export.create!(started_at: Time.zone.local(2022, 4, 26), base_number: 1, increment_number: 1) |
||||||
|
end |
||||||
|
|
||||||
|
it "generates an XML manifest file with the expected content within the ZIP file" do |
||||||
|
expected_content = replace_record_number(local_manifest_file.read, 2) |
||||||
|
expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) do |_, content| |
||||||
|
entry = Zip::File.open_buffer(content).find_entry(expected_manifest_filename) |
||||||
|
expect(entry).not_to be_nil |
||||||
|
expect(entry.get_input_stream.read).to eq(expected_content) |
||||||
|
end |
||||||
|
|
||||||
|
expect(export_service.export_xml_organisations).to eq({ expected_zip_filename.gsub(".zip", "") => start_time }) |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
@ -0,0 +1,219 @@ |
|||||||
|
require "rails_helper" |
||||||
|
|
||||||
|
RSpec.describe Exports::UserExportService do |
||||||
|
subject(:export_service) { described_class.new(storage_service, start_time) } |
||||||
|
|
||||||
|
let(:storage_service) { instance_double(Storage::S3Service) } |
||||||
|
|
||||||
|
let(:xml_export_file) { File.open("spec/fixtures/exports/user.xml", "r:UTF-8") } |
||||||
|
let(:local_manifest_file) { File.open("spec/fixtures/exports/manifest.xml", "r:UTF-8") } |
||||||
|
|
||||||
|
let(:expected_zip_filename) { "users_2024_2025_apr_mar_f0001_inc0001.zip" } |
||||||
|
let(:expected_data_filename) { "users_2024_2025_apr_mar_f0001_inc0001_pt001.xml" } |
||||||
|
let(:expected_manifest_filename) { "manifest.xml" } |
||||||
|
let(:start_time) { Time.zone.local(2022, 5, 1) } |
||||||
|
let(:organisation) { create(:organisation, with_dsa: false) } |
||||||
|
|
||||||
|
def replace_entity_ids(user, export_template) |
||||||
|
export_template.sub!(/\{id\}/, user["id"].to_s) |
||||||
|
export_template.sub!(/\{organisation_id\}/, user["organisation_id"].to_s) |
||||||
|
export_template.sub!(/\{email\}/, user["email"].to_s) |
||||||
|
end |
||||||
|
|
||||||
|
def replace_record_number(export_template, record_number) |
||||||
|
export_template.sub!(/\{recno\}/, record_number.to_s) |
||||||
|
end |
||||||
|
|
||||||
|
before do |
||||||
|
Timecop.freeze(start_time) |
||||||
|
Singleton.__init__(FormHandler) |
||||||
|
allow(storage_service).to receive(:write_file) |
||||||
|
end |
||||||
|
|
||||||
|
after do |
||||||
|
Timecop.return |
||||||
|
end |
||||||
|
|
||||||
|
context "when exporting daily users in XML" do |
||||||
|
context "and no users are available for export" do |
||||||
|
it "returns an empty archives list" do |
||||||
|
expect(export_service.export_xml_users).to eq({}) |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
context "and one user is available for export" do |
||||||
|
let!(:user) { create(:user, organisation:, phone_extension: "123") } |
||||||
|
|
||||||
|
it "generates a ZIP export file with the expected filename" do |
||||||
|
expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) |
||||||
|
export_service.export_xml_users |
||||||
|
end |
||||||
|
|
||||||
|
it "generates an XML export file with the expected filename within the ZIP file" do |
||||||
|
expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) do |_, content| |
||||||
|
entry = Zip::File.open_buffer(content).find_entry(expected_data_filename) |
||||||
|
expect(entry).not_to be_nil |
||||||
|
expect(entry.name).to eq(expected_data_filename) |
||||||
|
end |
||||||
|
export_service.export_xml_users |
||||||
|
end |
||||||
|
|
||||||
|
it "generates an XML manifest file with the expected content within the ZIP file" do |
||||||
|
expected_content = replace_record_number(local_manifest_file.read, 1) |
||||||
|
expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) do |_, content| |
||||||
|
entry = Zip::File.open_buffer(content).find_entry(expected_manifest_filename) |
||||||
|
expect(entry).not_to be_nil |
||||||
|
expect(entry.get_input_stream.read).to eq(expected_content) |
||||||
|
end |
||||||
|
|
||||||
|
export_service.export_xml_users |
||||||
|
end |
||||||
|
|
||||||
|
it "generates an XML export file with the expected content within the ZIP file" do |
||||||
|
expected_content = replace_entity_ids(user, xml_export_file.read) |
||||||
|
expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) do |_, content| |
||||||
|
entry = Zip::File.open_buffer(content).find_entry(expected_data_filename) |
||||||
|
expect(entry).not_to be_nil |
||||||
|
expect(entry.get_input_stream.read).to eq(expected_content) |
||||||
|
end |
||||||
|
|
||||||
|
export_service.export_xml_users |
||||||
|
end |
||||||
|
|
||||||
|
it "returns the list with correct archive" do |
||||||
|
expect(export_service.export_xml_users).to eq({ expected_zip_filename.gsub(".zip", "") => start_time }) |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
context "and multiple users are available for export" do |
||||||
|
before do |
||||||
|
create(:user, organisation:) |
||||||
|
create(:user, organisation:) |
||||||
|
end |
||||||
|
|
||||||
|
it "generates an XML manifest file with the expected content within the ZIP file" do |
||||||
|
expected_content = replace_record_number(local_manifest_file.read, 2) |
||||||
|
expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) do |_, content| |
||||||
|
entry = Zip::File.open_buffer(content).find_entry(expected_manifest_filename) |
||||||
|
expect(entry).not_to be_nil |
||||||
|
expect(entry.get_input_stream.read).to eq(expected_content) |
||||||
|
end |
||||||
|
|
||||||
|
export_service.export_xml_users |
||||||
|
end |
||||||
|
|
||||||
|
it "creates an export record in a database with correct time" do |
||||||
|
expect { export_service.export_xml_users } |
||||||
|
.to change(Export, :count).by(1) |
||||||
|
expect(Export.last.started_at).to be_within(2.seconds).of(start_time) |
||||||
|
end |
||||||
|
|
||||||
|
context "when this is the first export (full)" do |
||||||
|
it "returns a ZIP archive for the master manifest (existing lettings logs)" do |
||||||
|
expect(export_service.export_xml_users).to eq({ expected_zip_filename.gsub(".zip", "").gsub(".zip", "") => start_time }) |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
context "and underlying data changes between getting the users and writting the manifest" do |
||||||
|
def remove_users(users) |
||||||
|
users.each(&:destroy) |
||||||
|
file = Tempfile.new |
||||||
|
doc = Nokogiri::XML("<forms/>") |
||||||
|
doc.write_xml_to(file, encoding: "UTF-8") |
||||||
|
file.rewind |
||||||
|
file |
||||||
|
end |
||||||
|
|
||||||
|
def create_fake_maifest |
||||||
|
file = Tempfile.new |
||||||
|
doc = Nokogiri::XML("<forms/>") |
||||||
|
doc.write_xml_to(file, encoding: "UTF-8") |
||||||
|
file.rewind |
||||||
|
file |
||||||
|
end |
||||||
|
|
||||||
|
it "maintains the same record number" do |
||||||
|
# rubocop:disable RSpec/SubjectStub |
||||||
|
allow(export_service).to receive(:build_export_xml) do |users| |
||||||
|
remove_users(users) |
||||||
|
end |
||||||
|
allow(export_service).to receive(:build_manifest_xml) do |
||||||
|
create_fake_maifest |
||||||
|
end |
||||||
|
|
||||||
|
expect(export_service).to receive(:build_manifest_xml).with(2) |
||||||
|
# rubocop:enable RSpec/SubjectStub |
||||||
|
export_service.export_xml_users |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
context "when this is a second export (partial)" do |
||||||
|
before do |
||||||
|
start_time = Time.zone.local(2022, 6, 1) |
||||||
|
Export.new(started_at: start_time, collection: "users").save! # this should be user export |
||||||
|
end |
||||||
|
|
||||||
|
it "does not add any entry for the master manifest (no users)" do |
||||||
|
expect(export_service.export_xml_users).to eq({}) |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
context "and a previous export has run the same day having users" do |
||||||
|
before do |
||||||
|
create(:user, organisation:) |
||||||
|
export_service.export_xml_users |
||||||
|
end |
||||||
|
|
||||||
|
context "and we trigger another full update" do |
||||||
|
it "increments the base number" do |
||||||
|
export_service.export_xml_users(full_update: true) |
||||||
|
expect(Export.last.base_number).to eq(2) |
||||||
|
end |
||||||
|
|
||||||
|
it "resets the increment number" do |
||||||
|
export_service.export_xml_users(full_update: true) |
||||||
|
expect(Export.last.increment_number).to eq(1) |
||||||
|
end |
||||||
|
|
||||||
|
it "returns a correct archives list for manifest file" do |
||||||
|
expect(export_service.export_xml_users(full_update: true)).to eq({ "users_2024_2025_apr_mar_f0002_inc0001" => start_time }) |
||||||
|
end |
||||||
|
|
||||||
|
it "generates a ZIP export file with the expected filename" do |
||||||
|
expect(storage_service).to receive(:write_file).with("users_2024_2025_apr_mar_f0002_inc0001.zip", any_args) |
||||||
|
export_service.export_xml_users(full_update: true) |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
context "and a previous export has run having no users" do |
||||||
|
before { export_service.export_xml_users } |
||||||
|
|
||||||
|
it "doesn't increment the manifest number by 1" do |
||||||
|
export_service.export_xml_users |
||||||
|
|
||||||
|
expect(Export.last.increment_number).to eq(1) |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
context "and a user has been migrated since the previous partial export" do |
||||||
|
before do |
||||||
|
create(:user, updated_at: Time.zone.local(2022, 4, 27), organisation:) |
||||||
|
create(:user, updated_at: Time.zone.local(2022, 4, 27), organisation:) |
||||||
|
Export.create!(started_at: Time.zone.local(2022, 4, 26), base_number: 1, increment_number: 1) |
||||||
|
end |
||||||
|
|
||||||
|
it "generates an XML manifest file with the expected content within the ZIP file" do |
||||||
|
expected_content = replace_record_number(local_manifest_file.read, 2) |
||||||
|
expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) do |_, content| |
||||||
|
entry = Zip::File.open_buffer(content).find_entry(expected_manifest_filename) |
||||||
|
expect(entry).not_to be_nil |
||||||
|
expect(entry.get_input_stream.read).to eq(expected_content) |
||||||
|
end |
||||||
|
|
||||||
|
expect(export_service.export_xml_users).to eq({ expected_zip_filename.gsub(".zip", "") => start_time }) |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
Loading…
Reference in new issue