baarkerlounger
/
submit-social-housing-lettings-and-sales-data
mirror of https://github.com/communitiesuk/submit-social-housing-lettings-and-sales-data.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Update sales export

CLDC-3788-export-sales-logs
Kat 3 weeks ago
parent
9acbc45699
commit
9cd6579b3c
7 changed files with 1149 additions and 1 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      app/models/export.rb
  2. 194
      app/services/exports/sales_log_export_constants.rb
  3. 74
      app/services/exports/sales_log_export_service.rb
  4. 104
      spec/factories/sales_log.rb
  5. 207
      spec/fixtures/exports/sales_log.xml
  6. 207
      spec/fixtures/exports/sales_log_2024.xml
  7. 363
      spec/services/exports/sales_log_export_service_spec.rb

1
app/models/export.rb
Unescape View File

@ -1,5 +1,6 @@
class Export < ApplicationRecord
scope :lettings, -> { where(collection: "lettings") }
scope :sales, -> { where(collection: "sales") }
scope :organisations, -> { where(collection: "organisations") }
scope :users, -> { where(collection: "users") }
end

194
app/services/exports/sales_log_export_constants.rb
Unescape View File

@ -0,0 +1,194 @@
module Exports::SalesLogExportConstants
MAX_XML_RECORDS = 10_000
LOG_ID_OFFSET = 300_000_000_000
EXPORT_MODE = {
xml: 1,
csv: 2,
}.freeze
EXPORT_FIELDS = Set["id",
"status",
"saledate",
"created_at",
"updated_at",
"owning_organisation_id",
"assigned_to_id",
"purchid",
"type",
"ownershipsch",
"othtype",
"jointmore",
"jointpur",
"beds",
"companybuy",
"ethnic",
"ethnic_group",
"buy1livein",
"buylivein",
"builtype",
"proptype",
"noint",
"buy2livein",
"privacynotice",
"wheel",
"hholdcount",
"la",
"la_known",
"income1",
"income1nk",
"details_known_2",
"details_known_3",
"details_known_4",
"inc1mort",
"income2",
"income2nk",
"savingsnk",
"savings",
"prevown",
"updated_by_id",
"income1_value_check",
"mortgage",
"inc2mort",
"mortgage_value_check",
"hb",
"savings_value_check",
"deposit_value_check",
"frombeds",
"staircase",
"stairbought",
"stairowned",
"mrent",
"exdate",
"exday",
"exmonth",
"exyear",
"resale",
"deposit",
"cashdis",
"disabled",
"lanomagr",
"wheel_value_check",
"soctenant",
"value",
"equity",
"discount",
"grant",
"pregyrha",
"pregla",
"pregghb",
"pregother",
"ppostcode_full",
"is_previous_la_inferred",
"ppcodenk",
"ppostc1",
"ppostc2",
"prevloc",
"previous_la_known",
"hhregres",
"hhregresstill",
"proplen",
"has_mscharge",
"mscharge",
"prevten",
"mortgageused",
"wchair",
"income2_value_check",
"armedforcesspouse",
"hodate",
"hoday",
"homonth",
"hoyear",
"fromprop",
"socprevten",
"mortgagelender",
"mortgagelenderother",
"mortlen",
"extrabor",
"hhmemb",
"totadult",
"totchild",
"hhtype",
"pcode1",
"pcode2",
"pcodenk",
"postcode_full",
"is_la_inferred",
"bulk_upload_id",
"retirement_value_check",
"hodate_check",
"extrabor_value_check",
"deposit_and_mortgage_value_check",
"shared_ownership_deposit_value_check",
"grant_value_check",
"value_value_check",
"old_persons_shared_ownership_value_check",
"staircase_bought_value_check",
"monthly_charges_value_check",
"details_known_5",
"details_known_6",
"saledate_check",
"prevshared",
"staircasesale",
"ethnic_group2",
"ethnicbuy2",
"proplen_asked",
"old_id",
"buy2living",
"prevtenbuy2",
"pregblank",
"uprn",
"uprn_known",
"uprn_confirmed",
"address_line1",
"address_line2",
"town_or_city",
"county",
"nationalbuy2",
"discounted_sale_value_check",
"student_not_child_value_check",
"percentage_discount_value_check",
"combined_income_value_check",
"buyer_livein_value_check",
"discarded_at",
"stairowned_value_check",
"creation_method",
"old_form_id",
"managing_organisation_id",
"duplicate_set_id",
"nationality_all",
"nationality_all_group",
"nationality_all_buyer2",
"nationality_all_buyer2_group",
"address_line1_input",
"postcode_full_input",
"address_search_value_check",
"uprn_selection",
"address_line1_as_entered",
"address_line2_as_entered",
"town_or_city_as_entered",
"county_as_entered",
"postcode_full_as_entered",
"la_as_entered",
"partner_under_16_value_check",
"multiple_partners_value_check",
"created_by_id",
"has_management_fee",
"management_fee",
"firststair",
"numstair",
"mrentprestaircasing",
"lasttransaction",
"initialpurchase"]
(1..6).each do |index|
EXPORT_FIELDS << "age#{index}"
EXPORT_FIELDS << "age#{index}_known"
EXPORT_FIELDS << "ecstat#{index}"
EXPORT_FIELDS << "sex#{index}"
end
(2..6).each do |index|
EXPORT_FIELDS << "relat#{index}"
end
end

74
app/services/exports/sales_log_export_service.rb
Unescape View File

@ -1,5 +1,77 @@
module Exports
class SalesLogExportService < Exports::XmlExportService
def export_xml_sales_logs(full_update: false, collection_year: nil); end
include Exports::SalesLogExportConstants
include CollectionTimeHelper
def export_xml_sales_logs(full_update: false, collection_year: nil)
archives_for_manifest = {}
collection_years_to_export(collection_year).each do |year|
recent_export = Export.sales.where(year:).order("started_at").last
base_number = Export.sales.where(empty_export: false, year:).maximum(:base_number) || 1
export = build_export_run("sales", base_number, full_update, year)
archives = write_export_archive(export, year, recent_export, full_update)
archives_for_manifest.merge!(archives)
export.empty_export = archives.empty?
export.save!
end
archives_for_manifest
end
private
def get_archive_name(year, base_number, increment)
return unless year
base_number_str = "f#{base_number.to_s.rjust(4, '0')}"
increment_str = "inc#{increment.to_s.rjust(4, '0')}"
"core_sales_#{year}_#{year + 1}_apr_mar_#{base_number_str}_#{increment_str}".downcase
end
def retrieve_resources(recent_export, full_update, year)
if !full_update && recent_export
params = { from: recent_export.started_at, to: @start_time }
SalesLog.exportable.where("(updated_at >= :from AND updated_at <= :to) OR (values_updated_at IS NOT NULL AND values_updated_at >= :from AND values_updated_at <= :to)", params).filter_by_year(year)
else
params = { to: @start_time }
SalesLog.exportable.where("updated_at <= :to", params).filter_by_year(year)
end
end
def apply_cds_transformation(sales_log, _export_mode)
sales_log.attributes_before_type_cast
# attribute_hash["formid"] = attribute_hash["old_form_id"] || (attribute_hash["id"] + LOG_ID_OFFSET)
end
def is_omitted_field?(field_name, _sales_log)
!EXPORT_FIELDS.include?(field_name)
end
def build_export_xml(sales_logs)
doc = Nokogiri::XML("<forms/>")
sales_logs.each do |sales_log|
attribute_hash = apply_cds_transformation(sales_log, EXPORT_MODE[:xml])
form = doc.create_element("form")
doc.at("forms") << form
attribute_hash.each do |key, value|
if is_omitted_field?(key, sales_log)
next
else
form << doc.create_element(key, value)
end
end
end
xml_doc_to_temp_file(doc)
end
def collection_years_to_export(collection_year)
return [collection_year] if collection_year.present?
FormHandler.instance.sales_forms.values.map { |f| f.start_date.year }.uniq.select { |year| year > 2024 }
end
end
end

104
spec/factories/sales_log.rb
Unescape View File

@ -200,5 +200,109 @@ FactoryBot.define do
instance.save!(validate: false)
end
end
trait :export do
purchid { "123" }
ownershipsch { 2 }
type { 8 }
saledate_today
jointpur { 1 }
beds { 2 }
jointmore { 1 }
noint { 2 }
privacynotice { 1 }
age1_known { 0 }
age1 { 27 }
sex1 { "F" }
national { 18 }
buy1livein { 1 }
relat2 { "P" }
proptype { 1 }
age2_known { 0 }
age2 { 33 }
builtype { 1 }
ethnic { 3 }
ethnic_group { 17 }
sex2 { "X" }
buy2livein { "1" }
ecstat1 { "1" }
ecstat2 { "1" }
hholdcount { "4" }
wheel { 1 }
details_known_3 { 1 }
age3_known { 0 }
age3 { 14 }
details_known_4 { 1 }
age4_known { 0 }
age4 { 18 }
details_known_5 { 1 }
age5_known { 0 }
age5 { 40 }
details_known_6 { 1 }
age6_known { 0 }
age6 { 40 }
income1nk { 0 }
income1 { 10_000 }
inc1mort { 1 }
income2nk { 0 }
income2 { 10_000 }
inc2mort { 1 }
uprn_known { 0 }
address_line1 { "Address line 1" }
town_or_city { "City" }
la_known { 1 }
la { "E09000003" }
savingsnk { 1 }
prevown { 1 }
prevshared { 2 }
sex3 { "F" }
sex4 { "X" }
sex5 { "M" }
sex6 { "X" }
mortgage { 20_000 }
ecstat3 { 9 }
ecstat4 { 3 }
ecstat5 { 2 }
ecstat6 { 1 }
disabled { 1 }
deposit { 80_000 }
value { 110_000 }
value_value_check { 0 }
grant { 10_000 }
proplen { 10 }
pregyrha { 1 }
pregla { 1 }
pregother { 1 }
pregghb { 1 }
hhregres { 7 }
ppcodenk { 1 }
prevten { 1 }
previous_la_known { 0 }
relat3 { "X" }
relat4 { "X" }
relat5 { "R" }
relat6 { "R" }
hb { 4 }
mortgageused { 1 }
wchair { 1 }
armedforcesspouse { 5 }
has_mscharge { 1 }
mscharge { 100 }
mortlen { 10 }
pcodenk { 0 }
postcode_full { "SW1A 1AA" }
is_la_inferred { false }
mortgagelender { 5 }
extrabor { 1 }
ethnic_group2 { 17 }
nationalbuy2 { 13 }
buy2living { 3 }
proplen_asked { 1 }
address_line1_input { "Address line 1" }
postcode_full_input { "SW1A 1AA" }
nationality_all_group { 826 }
nationality_all_buyer2_group { 826 }
uprn { "10033558653" }
uprn_selection { 1 }
end
end
end

207
spec/fixtures/exports/sales_log.xml vendored
Unescape View File

@ -0,0 +1,207 @@
<?xml version="1.0" encoding="UTF-8"?>
<forms>
<form>
<id>{id}</id>
<status>1</status>
<saledate>2026-03-01 00:00:00 UTC</saledate>
<created_at>2026-03-01 00:00:00 UTC</created_at>
<updated_at>2026-03-01 00:00:00 UTC</updated_at>
<owning_organisation_id>{owning_org_id}</owning_organisation_id>
<assigned_to_id>{assigned_to_id}</assigned_to_id>
<purchid>123</purchid>
<type>8</type>
<ownershipsch>2</ownershipsch>
<othtype/>
<jointmore>1</jointmore>
<jointpur>1</jointpur>
<beds>2</beds>
<companybuy/>
<age1>27</age1>
<age1_known>0</age1_known>
<sex1>F</sex1>
<ethnic>17</ethnic>
<ethnic_group>17</ethnic_group>
<buy1livein>1</buy1livein>
<buylivein/>
<builtype>1</builtype>
<proptype>1</proptype>
<age2>33</age2>
<age2_known>0</age2_known>
<relat2>P</relat2>
<sex2>X</sex2>
<noint>2</noint>
<buy2livein>1</buy2livein>
<ecstat2>1</ecstat2>
<privacynotice>1</privacynotice>
<ecstat1>1</ecstat1>
<wheel>1</wheel>
<hholdcount>4</hholdcount>
<age3>14</age3>
<age3_known>0</age3_known>
<la>E09000033</la>
<la_known>1</la_known>
<income1>10000</income1>
<income1nk>0</income1nk>
<details_known_2/>
<details_known_3>1</details_known_3>
<details_known_4>1</details_known_4>
<age4>18</age4>
<age4_known>0</age4_known>
<age5>40</age5>
<age5_known>0</age5_known>
<age6>40</age6>
<age6_known>0</age6_known>
<inc1mort>1</inc1mort>
<income2>10000</income2>
<income2nk>0</income2nk>
<savingsnk>1</savingsnk>
<savings/>
<prevown>1</prevown>
<sex3>F</sex3>
<updated_by_id/>
<income1_value_check/>
<mortgage>20000.0</mortgage>
<inc2mort>1</inc2mort>
<mortgage_value_check/>
<ecstat3>9</ecstat3>
<ecstat4>3</ecstat4>
<ecstat5>2</ecstat5>
<ecstat6>1</ecstat6>
<relat3>X</relat3>
<relat4>X</relat4>
<relat5>R</relat5>
<relat6>R</relat6>
<hb>4</hb>
<sex4>X</sex4>
<sex5>M</sex5>
<sex6>X</sex6>
<savings_value_check/>
<deposit_value_check/>
<frombeds/>
<staircase/>
<stairbought/>
<stairowned/>
<mrent/>
<exdate/>
<exday/>
<exmonth/>
<exyear/>
<resale/>
<deposit>80000.0</deposit>
<cashdis/>
<disabled>1</disabled>
<lanomagr/>
<wheel_value_check/>
<soctenant/>
<value>110000.0</value>
<equity/>
<discount/>
<grant>10000.0</grant>
<pregyrha>1</pregyrha>
<pregla>1</pregla>
<pregghb>1</pregghb>
<pregother>1</pregother>
<ppostcode_full>SW1A 1AA</ppostcode_full>
<is_previous_la_inferred>true</is_previous_la_inferred>
<ppcodenk>0</ppcodenk>
<ppostc1>SW1A</ppostc1>
<ppostc2>1AA</ppostc2>
<prevloc>E09000033</prevloc>
<previous_la_known>1</previous_la_known>
<hhregres>7</hhregres>
<hhregresstill/>
<proplen/>
<has_mscharge>1</has_mscharge>
<mscharge>100.0</mscharge>
<prevten>1</prevten>
<mortgageused>1</mortgageused>
<wchair>1</wchair>
<income2_value_check/>
<armedforcesspouse>5</armedforcesspouse>
<hodate/>
<hoday/>
<homonth/>
<hoyear/>
<fromprop/>
<socprevten/>
<mortgagelender>5</mortgagelender>
<mortgagelenderother/>
<mortlen>10</mortlen>
<extrabor>1</extrabor>
<hhmemb>6</hhmemb>
<totadult>5</totadult>
<totchild>1</totchild>
<hhtype>6</hhtype>
<pcode1>SW1A</pcode1>
<pcode2>1AA</pcode2>
<pcodenk>0</pcodenk>
<postcode_full>AA1 1AA</postcode_full>
<is_la_inferred>true</is_la_inferred>
<bulk_upload_id/>
<retirement_value_check/>
<hodate_check/>
<extrabor_value_check/>
<deposit_and_mortgage_value_check/>
<shared_ownership_deposit_value_check/>
<grant_value_check/>
<value_value_check/>
<old_persons_shared_ownership_value_check/>
<staircase_bought_value_check/>
<monthly_charges_value_check/>
<details_known_5>1</details_known_5>
<details_known_6>1</details_known_6>
<saledate_check/>
<prevshared>2</prevshared>
<staircasesale/>
<ethnic_group2>17</ethnic_group2>
<ethnicbuy2/>
<proplen_asked/>
<old_id/>
<buy2living>3</buy2living>
<prevtenbuy2/>
<pregblank/>
<uprn>1</uprn>
<uprn_known>1</uprn_known>
<uprn_confirmed>1</uprn_confirmed>
<address_line1>1, Test Street</address_line1>
<address_line2/>
<town_or_city>Test Town</town_or_city>
<county/>
<nationalbuy2>13</nationalbuy2>
<discounted_sale_value_check/>
<student_not_child_value_check/>
<percentage_discount_value_check/>
<combined_income_value_check/>
<buyer_livein_value_check/>
<discarded_at/>
<stairowned_value_check/>
<creation_method>1</creation_method>
<old_form_id/>
<managing_organisation_id>{managing_org_id}</managing_organisation_id>
<duplicate_set_id/>
<nationality_all>826</nationality_all>
<nationality_all_group>826</nationality_all_group>
<nationality_all_buyer2>826</nationality_all_buyer2>
<nationality_all_buyer2_group>826</nationality_all_buyer2_group>
<address_line1_input>Address line 1</address_line1_input>
<postcode_full_input>SW1A 1AA</postcode_full_input>
<address_search_value_check/>
<uprn_selection/>
<address_line1_as_entered/>
<address_line2_as_entered/>
<town_or_city_as_entered/>
<county_as_entered/>
<postcode_full_as_entered/>
<la_as_entered/>
<partner_under_16_value_check/>
<multiple_partners_value_check/>
<created_by_id>{created_by_id}</created_by_id>
<has_management_fee/>
<management_fee/>
<firststair/>
<numstair/>
<mrentprestaircasing/>
<lasttransaction/>
<initialpurchase/>
</form>
</forms>

207
spec/fixtures/exports/sales_log_2024.xml vendored
Unescape View File

@ -0,0 +1,207 @@
<?xml version="1.0" encoding="UTF-8"?>
<forms>
<form>
<id>{id}</id>
<status>2</status>
<saledate>2024-04-02 23:00:00 UTC</saledate>
<created_at>2024-04-02 23:00:00 UTC</created_at>
<updated_at>2024-04-02 23:00:00 UTC</updated_at>
<owning_organisation_id>{owning_org_id}</owning_organisation_id>
<assigned_to_id>{assigned_to_id}</assigned_to_id>
<purchid>123</purchid>
<type>8</type>
<ownershipsch>2</ownershipsch>
<othtype/>
<jointmore>1</jointmore>
<jointpur>1</jointpur>
<beds>2</beds>
<companybuy/>
<age1>27</age1>
<age1_known>0</age1_known>
<sex1>F</sex1>
<ethnic>17</ethnic>
<ethnic_group>17</ethnic_group>
<buy1livein>1</buy1livein>
<buylivein/>
<builtype>1</builtype>
<proptype>1</proptype>
<age2>33</age2>
<age2_known>0</age2_known>
<relat2>P</relat2>
<sex2>X</sex2>
<noint>2</noint>
<buy2livein>1</buy2livein>
<ecstat2>1</ecstat2>
<privacynotice>1</privacynotice>
<ecstat1>1</ecstat1>
<wheel>1</wheel>
<hholdcount>4</hholdcount>
<age3>14</age3>
<age3_known>0</age3_known>
<la>E09000033</la>
<la_known>1</la_known>
<income1>10000</income1>
<income1nk>0</income1nk>
<details_known_2/>
<details_known_3>1</details_known_3>
<details_known_4>1</details_known_4>
<age4>18</age4>
<age4_known>0</age4_known>
<age5>40</age5>
<age5_known>0</age5_known>
<age6>40</age6>
<age6_known>0</age6_known>
<inc1mort>1</inc1mort>
<income2>10000</income2>
<income2nk>0</income2nk>
<savingsnk>1</savingsnk>
<savings/>
<prevown>1</prevown>
<sex3>F</sex3>
<updated_by_id/>
<income1_value_check/>
<mortgage>20000.0</mortgage>
<inc2mort>1</inc2mort>
<mortgage_value_check/>
<ecstat3>9</ecstat3>
<ecstat4>3</ecstat4>
<ecstat5>2</ecstat5>
<ecstat6>1</ecstat6>
<relat3>X</relat3>
<relat4>X</relat4>
<relat5>R</relat5>
<relat6>R</relat6>
<hb>4</hb>
<sex4>X</sex4>
<sex5>M</sex5>
<sex6>X</sex6>
<savings_value_check/>
<deposit_value_check/>
<frombeds/>
<staircase/>
<stairbought/>
<stairowned/>
<mrent/>
<exdate/>
<exday/>
<exmonth/>
<exyear/>
<resale/>
<deposit>80000.0</deposit>
<cashdis/>
<disabled>1</disabled>
<lanomagr/>
<wheel_value_check/>
<soctenant/>
<value>110000.0</value>
<equity/>
<discount/>
<grant>10000.0</grant>
<pregyrha>1</pregyrha>
<pregla>1</pregla>
<pregghb>1</pregghb>
<pregother>1</pregother>
<ppostcode_full>SW1A 1AA</ppostcode_full>
<is_previous_la_inferred>true</is_previous_la_inferred>
<ppcodenk>0</ppcodenk>
<ppostc1>SW1A</ppostc1>
<ppostc2>1AA</ppostc2>
<prevloc>E09000033</prevloc>
<previous_la_known>1</previous_la_known>
<hhregres>7</hhregres>
<hhregresstill/>
<proplen>10</proplen>
<has_mscharge>1</has_mscharge>
<mscharge>100.0</mscharge>
<prevten>1</prevten>
<mortgageused>1</mortgageused>
<wchair>1</wchair>
<income2_value_check/>
<armedforcesspouse>5</armedforcesspouse>
<hodate/>
<hoday/>
<homonth/>
<hoyear/>
<fromprop/>
<socprevten/>
<mortgagelender>5</mortgagelender>
<mortgagelenderother/>
<mortlen>10</mortlen>
<extrabor>1</extrabor>
<hhmemb>6</hhmemb>
<totadult>5</totadult>
<totchild>1</totchild>
<hhtype>6</hhtype>
<pcode1>SW1A</pcode1>
<pcode2>1AA</pcode2>
<pcodenk>0</pcodenk>
<postcode_full>AA1 1AA</postcode_full>
<is_la_inferred>true</is_la_inferred>
<bulk_upload_id/>
<retirement_value_check/>
<hodate_check/>
<extrabor_value_check/>
<deposit_and_mortgage_value_check/>
<shared_ownership_deposit_value_check/>
<grant_value_check/>
<value_value_check/>
<old_persons_shared_ownership_value_check/>
<staircase_bought_value_check/>
<monthly_charges_value_check/>
<details_known_5>1</details_known_5>
<details_known_6>1</details_known_6>
<saledate_check/>
<prevshared>2</prevshared>
<staircasesale/>
<ethnic_group2>17</ethnic_group2>
<ethnicbuy2/>
<proplen_asked>1</proplen_asked>
<old_id/>
<buy2living>3</buy2living>
<prevtenbuy2/>
<pregblank/>
<uprn>1</uprn>
<uprn_known>1</uprn_known>
<uprn_confirmed>1</uprn_confirmed>
<address_line1>1, Test Street</address_line1>
<address_line2/>
<town_or_city>Test Town</town_or_city>
<county/>
<nationalbuy2>13</nationalbuy2>
<discounted_sale_value_check/>
<student_not_child_value_check/>
<percentage_discount_value_check/>
<combined_income_value_check/>
<buyer_livein_value_check/>
<discarded_at/>
<stairowned_value_check/>
<creation_method>1</creation_method>
<old_form_id/>
<managing_organisation_id>{managing_org_id}</managing_organisation_id>
<duplicate_set_id/>
<nationality_all>826</nationality_all>
<nationality_all_group>826</nationality_all_group>
<nationality_all_buyer2>826</nationality_all_buyer2>
<nationality_all_buyer2_group>826</nationality_all_buyer2_group>
<address_line1_input>Address line 1</address_line1_input>
<postcode_full_input>SW1A 1AA</postcode_full_input>
<address_search_value_check/>
<uprn_selection/>
<address_line1_as_entered/>
<address_line2_as_entered/>
<town_or_city_as_entered/>
<county_as_entered/>
<postcode_full_as_entered/>
<la_as_entered/>
<partner_under_16_value_check/>
<multiple_partners_value_check/>
<created_by_id>{created_by_id}</created_by_id>
<has_management_fee/>
<management_fee/>
<firststair/>
<numstair/>
<mrentprestaircasing/>
<lasttransaction/>
<initialpurchase/>
</form>
</forms>

363
spec/services/exports/sales_log_export_service_spec.rb
Unescape View File

@ -0,0 +1,363 @@
require "rails_helper"
RSpec.describe Exports::SalesLogExportService do
subject(:export_service) { described_class.new(storage_service, start_time) }
let(:storage_service) { instance_double(Storage::S3Service) }
let(:xml_export_file) { File.open("spec/fixtures/exports/sales_log.xml", "r:UTF-8") }
let(:local_manifest_file) { File.open("spec/fixtures/exports/manifest.xml", "r:UTF-8") }
let(:expected_zip_filename) { "core_sales_2025_2026_apr_mar_f0001_inc0001.zip" }
let(:expected_data_filename) { "core_sales_2025_2026_apr_mar_f0001_inc0001_pt001.xml" }
let(:expected_manifest_filename) { "manifest.xml" }
let(:start_time) { Time.zone.local(2026, 3, 1) }
let(:organisation) { create(:organisation, name: "MHCLG", housing_registration_no: 1234) }
let(:user) { FactoryBot.create(:user, email: "test1@example.com", organisation:) }
def replace_entity_ids(sales_log, export_template)
# export_template.sub!(/\{id\}/, (sales_log["id"] + Exports::SalesLogExportService::LOG_ID_OFFSET).to_s)
export_template.sub!(/\{owning_org_id\}/, sales_log["owning_organisation_id"].to_s)
export_template.sub!(/\{managing_org_id\}/, sales_log["managing_organisation_id"].to_s)
export_template.sub!(/\{assigned_to_id\}/, sales_log["assigned_to_id"].to_s)
export_template.sub!(/\{created_by_id\}/, sales_log["created_by_id"].to_s)
export_template.sub!(/\{id\}/, sales_log["id"].to_s)
end
def replace_record_number(export_template, record_number)
export_template.sub!(/\{recno\}/, record_number.to_s)
end
before do
Timecop.freeze(start_time)
Singleton.__init__(FormHandler)
allow(storage_service).to receive(:write_file)
end
after do
Timecop.return
end
context "when exporting daily sales logs in XML" do
context "and no sales logs are available for export" do
it "returns an empty archives list" do
expect(storage_service).not_to receive(:write_file)
expect(export_service.export_xml_sales_logs).to eq({})
end
end
context "when one pending sales log exists" do
before do
FactoryBot.create(
:sales_log,
:export,
status: "pending",
skip_update_status: true,
)
end
it "returns empty archives list for archives manifest" do
expect(storage_service).not_to receive(:write_file)
expect(export_service.export_xml_sales_logs).to eq({})
end
end
context "and one sales log is available for export" do
let!(:sales_log) { FactoryBot.create(:sales_log, :export, assigned_to: user) }
it "generates a ZIP export file with the expected filename" do
expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args)
export_service.export_xml_sales_logs
end
it "generates an XML export file with the expected filename within the ZIP file" do
expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) do |_, content|
entry = Zip::File.open_buffer(content).find_entry(expected_data_filename)
expect(entry).not_to be_nil
expect(entry.name).to eq(expected_data_filename)
end
export_service.export_xml_sales_logs
end
it "generates an XML manifest file with the expected content within the ZIP file" do
expected_content = replace_record_number(local_manifest_file.read, 1)
expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) do |_, content|
entry = Zip::File.open_buffer(content).find_entry(expected_manifest_filename)
expect(entry).not_to be_nil
expect(entry.get_input_stream.read).to eq(expected_content)
end
export_service.export_xml_sales_logs
end
it "generates an XML export file with the expected content within the ZIP file" do
expected_content = replace_entity_ids(sales_log, xml_export_file.read)
expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) do |_, content|
entry = Zip::File.open_buffer(content).find_entry(expected_data_filename)
expect(entry).not_to be_nil
expect(entry.get_input_stream.read).to eq(expected_content)
end
export_service.export_xml_sales_logs
end
it "returns the list with correct archive" do
expect(export_service.export_xml_sales_logs).to eq({ expected_zip_filename.gsub(".zip", "") => start_time })
end
end
context "and multiple sales logs are available for export on different periods" do
let(:previous_zip_filename) { "core_sales_2024_2025_apr_mar_f0001_inc0001.zip" }
let(:next_zip_filename) { "core_sales_2026_2027_apr_mar_f0001_inc0001.zip" }
before do
FactoryBot.create(:sales_log, :ignore_validation_errors, saledate: Time.zone.local(2024, 5, 1))
FactoryBot.create(:sales_log, saledate: Time.zone.local(2025, 5, 1))
FactoryBot.create(:sales_log, :ignore_validation_errors, saledate: Time.zone.local(2026, 4, 1))
end
context "when sales logs are across multiple years" do
it "generates multiple ZIP export files with the expected filenames" do
expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args)
expect(storage_service).not_to receive(:write_file).with(previous_zip_filename, any_args)
expect(storage_service).to receive(:write_file).with(next_zip_filename, any_args)
expect(Rails.logger).to receive(:info).with("Building export run for sales 2025")
expect(Rails.logger).to receive(:info).with("Creating core_sales_2025_2026_apr_mar_f0001_inc0001 - 1 resources")
expect(Rails.logger).to receive(:info).with("Added core_sales_2025_2026_apr_mar_f0001_inc0001_pt001.xml")
expect(Rails.logger).to receive(:info).with("Writing core_sales_2025_2026_apr_mar_f0001_inc0001.zip")
expect(Rails.logger).to receive(:info).with("Building export run for sales 2026")
expect(Rails.logger).to receive(:info).with("Creating core_sales_2026_2027_apr_mar_f0001_inc0001 - 1 resources")
expect(Rails.logger).to receive(:info).with("Added core_sales_2026_2027_apr_mar_f0001_inc0001_pt001.xml")
expect(Rails.logger).to receive(:info).with("Writing core_sales_2026_2027_apr_mar_f0001_inc0001.zip")
export_service.export_xml_sales_logs
end
it "generates zip export files only for specified year" do
expect(storage_service).to receive(:write_file).with(next_zip_filename, any_args)
expect(Rails.logger).to receive(:info).with("Building export run for sales 2026")
expect(Rails.logger).to receive(:info).with("Creating core_sales_2026_2027_apr_mar_f0001_inc0001 - 1 resources")
expect(Rails.logger).to receive(:info).with("Added core_sales_2026_2027_apr_mar_f0001_inc0001_pt001.xml")
expect(Rails.logger).to receive(:info).with("Writing core_sales_2026_2027_apr_mar_f0001_inc0001.zip")
export_service.export_xml_sales_logs(collection_year: 2026)
end
context "and previous full exports are different for previous years" do
let(:expected_zip_filename) { "core_sales_2025_2026_apr_mar_f0007_inc0004.zip" }
let(:next_zip_filename) { "core_sales_2026_2027_apr_mar_f0001_inc0001.zip" }
before do
Export.new(started_at: Time.zone.yesterday, base_number: 7, increment_number: 3, collection: "sales", year: 2025).save!
end
it "generates multiple ZIP export files with different base numbers in the filenames" do
expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args)
expect(storage_service).to receive(:write_file).with(next_zip_filename, any_args)
expect(Rails.logger).to receive(:info).with("Building export run for sales 2025")
expect(Rails.logger).to receive(:info).with("Creating core_sales_2025_2026_apr_mar_f0007_inc0004 - 1 resources")
expect(Rails.logger).to receive(:info).with("Added core_sales_2025_2026_apr_mar_f0007_inc0004_pt001.xml")
expect(Rails.logger).to receive(:info).with("Writing core_sales_2025_2026_apr_mar_f0007_inc0004.zip")
expect(Rails.logger).to receive(:info).with("Building export run for sales 2026")
expect(Rails.logger).to receive(:info).with("Creating core_sales_2026_2027_apr_mar_f0001_inc0001 - 1 resources")
expect(Rails.logger).to receive(:info).with("Added core_sales_2026_2027_apr_mar_f0001_inc0001_pt001.xml")
expect(Rails.logger).to receive(:info).with("Writing core_sales_2026_2027_apr_mar_f0001_inc0001.zip")
export_service.export_xml_sales_logs
end
end
end
end
context "and multiple sales logs are available for export on same quarter" do
before do
FactoryBot.create(:sales_log, saledate: Time.zone.local(2025, 4, 1))
FactoryBot.create(:sales_log, saledate: Time.zone.local(2025, 4, 20))
end
it "generates an XML manifest file with the expected content within the ZIP file" do
expected_content = replace_record_number(local_manifest_file.read, 2)
expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) do |_, content|
entry = Zip::File.open_buffer(content).find_entry(expected_manifest_filename)
expect(entry).not_to be_nil
expect(entry.get_input_stream.read).to eq(expected_content)
end
export_service.export_xml_sales_logs
end
it "creates a logs export record in a database with correct time" do
expect { export_service.export_xml_sales_logs }
.to change(Export, :count).by(2)
expect(Export.last.started_at).to be_within(2.seconds).of(start_time)
end
context "when this is the first export (full)" do
it "returns a ZIP archive for the master manifest (existing sales logs)" do
expect(export_service.export_xml_sales_logs).to eq({ expected_zip_filename.gsub(".zip", "").gsub(".zip", "") => start_time })
end
end
context "and underlying data changes between getting the logs and writting the manifest" do
before do
FactoryBot.create(:sales_log, saledate: Time.zone.local(2026, 2, 1))
FactoryBot.create(:sales_log, :ignore_validation_errors, saledate: Time.zone.local(2026, 4, 1))
end
def remove_logs(logs)
logs.each(&:destroy)
file = Tempfile.new
doc = Nokogiri::XML("<forms/>")
doc.write_xml_to(file, encoding: "UTF-8")
file.rewind
file
end
def create_fake_maifest
file = Tempfile.new
doc = Nokogiri::XML("<forms/>")
doc.write_xml_to(file, encoding: "UTF-8")
file.rewind
file
end
it "maintains the same record number" do
# rubocop:disable RSpec/SubjectStub
allow(export_service).to receive(:build_export_xml) do |logs|
remove_logs(logs)
end
allow(export_service).to receive(:build_manifest_xml) do
create_fake_maifest
end
expect(export_service).to receive(:build_manifest_xml).with(1)
# rubocop:enable RSpec/SubjectStub
export_service.export_xml_sales_logs
end
end
context "when this is a second export (partial)" do
before do
start_time = Time.zone.local(2026, 6, 1)
Export.new(started_at: start_time, collection: "sales", year: 2025).save!
end
it "does not add any entry for the master manifest (no sales logs)" do
expect(storage_service).not_to receive(:write_file)
expect(export_service.export_xml_sales_logs).to eq({})
end
end
end
context "and a previous export has run the same day having sales logs" do
before do
FactoryBot.create(:sales_log, saledate: Time.zone.local(2025, 5, 1))
export_service.export_xml_sales_logs
end
context "and we trigger another full update" do
it "increments the base number" do
export_service.export_xml_sales_logs(full_update: true)
expect(Export.last.base_number).to eq(2)
end
it "resets the increment number" do
export_service.export_xml_sales_logs(full_update: true)
expect(Export.last.increment_number).to eq(1)
end
it "returns a correct archives list for manifest file" do
expect(export_service.export_xml_sales_logs(full_update: true)).to eq({ "core_sales_2025_2026_apr_mar_f0002_inc0001" => start_time })
end
it "generates a ZIP export file with the expected filename" do
expect(storage_service).to receive(:write_file).with("core_sales_2025_2026_apr_mar_f0002_inc0001.zip", any_args)
export_service.export_xml_sales_logs(full_update: true)
end
end
end
context "and a previous export has run having no sales logs" do
before { export_service.export_xml_sales_logs }
it "doesn't increment the manifest number by 1" do
export_service.export_xml_sales_logs
expect(Export.last.increment_number).to eq(1)
end
end
context "and a log has been manually updated since the previous partial export" do
let(:expected_zip_filename) { "core_sales_2025_2026_apr_mar_f0001_inc0002.zip" }
before do
FactoryBot.create(:sales_log, saledate: Time.zone.local(2026, 2, 1), updated_at: Time.zone.local(2026, 2, 27), values_updated_at: Time.zone.local(2026, 2, 29))
FactoryBot.create(:sales_log, saledate: Time.zone.local(2026, 2, 1), updated_at: Time.zone.local(2026, 2, 27), values_updated_at: Time.zone.local(2026, 2, 29))
Export.create!(started_at: Time.zone.local(2026, 2, 28), base_number: 1, increment_number: 1, collection: "sales", year: 2025)
end
it "generates an XML manifest file with the expected content within the ZIP file" do
expected_content = replace_record_number(local_manifest_file.read, 2)
expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) do |_, content|
entry = Zip::File.open_buffer(content).find_entry(expected_manifest_filename)
expect(entry).not_to be_nil
expect(entry.get_input_stream.read).to eq(expected_content)
end
expect(export_service.export_xml_sales_logs).to eq({ expected_zip_filename.gsub(".zip", "") => start_time })
end
end
context "and one sales log with duplicate reference is available for export" do
let!(:sales_log) { FactoryBot.create(:sales_log, :export, duplicate_set_id: 123) }
def replace_duplicate_set_id(export_file)
export_file.sub!("<duplicate_set_id/>", "<duplicate_set_id>123</duplicate_set_id>")
end
it "generates an XML export file with the expected content within the ZIP file" do
expected_content = replace_entity_ids(sales_log, xml_export_file.read)
expected_content = replace_duplicate_set_id(expected_content)
expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) do |_, content|
entry = Zip::File.open_buffer(content).find_entry(expected_data_filename)
expect(entry).not_to be_nil
expect(entry.get_input_stream.read).to eq(expected_content)
end
export_service.export_xml_sales_logs
end
end
context "when exporting only 24/25 collection period" do
let(:start_time) { Time.zone.local(2024, 4, 3) }
before do
Timecop.freeze(start_time)
Singleton.__init__(FormHandler)
end
after do
Timecop.unfreeze
Singleton.__init__(FormHandler)
end
context "and one sales log is available for export" do
let!(:sales_log) { FactoryBot.create(:sales_log, :export) }
let(:expected_zip_filename) { "core_sales_2024_2025_apr_mar_f0001_inc0001.zip" }
let(:expected_data_filename) { "core_sales_2024_2025_apr_mar_f0001_inc0001_pt001.xml" }
let(:xml_export_file) { File.open("spec/fixtures/exports/sales_log_2024.xml", "r:UTF-8") }
it "generates an XML export file with the expected content within the ZIP file" do
expected_content = replace_entity_ids(sales_log, xml_export_file.read)
expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) do |_, content|
entry = Zip::File.open_buffer(content).find_entry(expected_data_filename)
expect(entry).not_to be_nil
expect(entry.get_input_stream.read).to eq(expected_content)
end
export_service.export_xml_sales_logs(full_update: true, collection_year: 2024)
end
end
end
end
end
Write Preview
Loading…
Cancel
Save