Browse Source

Merge branch 'main' into dependabot/npm_and_yarn/babel/helpers-7.26.10

pull/3007/head
kosiakkatrina 1 month ago committed by GitHub
parent
commit
252ee3f450
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
  1. 1
      docs/Gemfile
  2. 3
      docs/Gemfile.lock
  3. 3
      docs/_config.yml
  4. 111
      docs/diagrams.md
  5. 177
      lib/tasks/update_manual_address_entry_selected_prexisting_logs.rake
  6. 102
      spec/lib/tasks/update_manual_address_entry_selected_prexisting_logs_spec.rb

1
docs/Gemfile

@ -2,7 +2,6 @@ source "https://rubygems.org"
group :jekyll_plugins do
gem "github-pages"
gem "jekyll-plantuml"
end
group :development do

3
docs/Gemfile.lock

@ -130,8 +130,6 @@ GEM
jekyll-optional-front-matter (0.3.2)
jekyll (>= 3.0, < 5.0)
jekyll-paginate (1.1.0)
jekyll-plantuml (1.4.2)
jekyll (> 2.0)
jekyll-readme-index (0.3.0)
jekyll (>= 3.0, < 5.0)
jekyll-redirect-from (0.16.0)
@ -263,7 +261,6 @@ PLATFORMS
DEPENDENCIES
github-pages
jekyll-plantuml
webrick
BUNDLED WITH

3
docs/_config.yml

@ -10,6 +10,3 @@ aux_links:
- https://core-design-history.herokuapp.com
"GitHub":
- https://github.com/communitiesuk/submit-social-housing-lettings-and-sales-data
gems: ["jekyll-plantuml"]
plugins:
- jekyll-diagrams

111
docs/diagrams.md

@ -1,111 +0,0 @@
---
title: PlanUML source for diagrams
---
{% plantuml %}
!define AWSPuml https://raw.githubusercontent.com/awslabs/aws-icons-for-plantuml/v19.0/dist
!include AWSPuml/AWSCommon.puml
!include AWSPuml/Groups/VPC.puml
!include AWSPuml/Groups/GenericOrange.puml
!include AWSPuml/Storage/SimpleStorageService.puml
!include AWSPuml/AWSSimplified.puml
!include AWSPuml/Groups/PublicSubnet.puml
!include AWSPuml/Groups/PrivateSubnet.puml
!include AWSPuml/Containers/ElasticContainerService.puml
!include AWSPuml/Database/RDS.puml
!include AWSPuml/Database/ElastiCacheElastiCacheforRedis.puml
!include AWSPuml/NetworkingContentDelivery/CloudFront.puml
!include AWSPuml/NetworkingContentDelivery/ElasticLoadBalancingApplicationLoadBalancer.puml
!include AWSPuml/Containers/ElasticContainerRegistry.puml
!include AWSPuml/Groups/AWSAccount.puml
' External Systems
actor User
' Main System: Meta Environment
AWSAccountGroup(meta, "Meta Environment") {
ElasticContainerRegistry(ecr, "ECR (Elastic Container Registry) - Shared between environments", "ECR (Elastic Container Registry)","")
}
VPCGroup(vpc) {
PublicSubnetGroup(public_subnet, "Public subnet") {
ElasticLoadBalancingApplicationLoadBalancer(loadBalancer, "Load Balancer", "Load Balancer", "")
}
PrivateSubnetGroup(private_subnet, "Private subnet") {
GenericOrangeGroup(fargateTasks, "ECS Fargate Tasks") {
ElasticContainerService(App, "App", "Container", "")
ElasticContainerService(Sidekiq, "Sidekiq", "Container", "")
ElasticContainerService(AdHocTasks, "Ad-hoc Tasks", "Container", "")
}
RDS(rdsDatabase, "RDS", "RDS","")
ElastiCacheElastiCacheforRedis(redis, "ElastiCache for Redis", "Redis","")
}
}
' Networking
SimpleStorageService(bulkUpload, "Bulk Upload", "Bulk upload bucket", "")
SimpleStorageService(cdsExport, "CDS Export", "CDS export bucket", "")
SimpleStorageService(collectionResources, "Collection resources", "Collection resources bucket", "")
CloudFront(cloudFront, "CloudFront", "CloudFront", "")
' Relationships
User --> [cloudFront] : Interacts with
[loadBalancer] --> [fargateTasks] : Routes traffic to
[ecr] --> [fargateTasks] : Pulls Docker images from
[cloudFront] --> [loadBalancer] : Routes traffic to
[fargateTasks] --> [bulkUpload] : Uploads data to
[fargateTasks] --> [cdsExport] : Exports data to
[fargateTasks] --> [collectionResources] : Reads/writes data to
[fargateTasks] --> [rdsDatabase] : Writes data to
[fargateTasks] --> [redis] : Caches data in
{% endplantuml %}
---
{% plantuml %}
!define AWSPuml https://raw.githubusercontent.com/awslabs/aws-icons-for-plantuml/v19.0/dist
!include AWSPuml/AWSCommon.puml
!include AWSPuml/Storage/SimpleStorageService.puml
!include AWSPuml/Groups/GenericOrange.puml
!define SPRITESURL https://raw.githubusercontent.com/plantuml-stdlib/gilbarbara-plantuml-sprites/v1.1/sprites
!define IMAGESSURL https://raw.githubusercontent.com/plantuml-stdlib/gilbarbara-plantuml-sprites/v1.1/pngs
!includeurl SPRITESURL/sentry.puml
!includeurl SPRITESURL/google-analytics.puml
!include AWSPuml/Containers/ElasticContainerService.puml
!include AWSPuml/AWSSimplified.puml
skinparam actorPadding 15
skinparam packagePadding 20
skinparam componentPadding 20
skinparam rectanglePadding 20
rectangle "<img:IMAGESSURL/google-analytics.png>" as ga
rectangle "<img:IMAGESSURL/sentry.png>" as sentry
rectangle "GOV.UK Notify" as notify
rectangle "OS Places API" as osapi
rectangle "CDS Ingest Pipeline" as cds
actor "End Users" as Users
actor "Support Users" as SupportUsers
' Main System: Application Package
package "Application" {
ElasticContainerService(App, "App", "Container", "")
SimpleStorageService(cdsExport, "CDS Export", "CDS export bucket", "")
}
' Relationships and interactions
Users --> [App] : Provides data, downloads CSVs
SupportUsers --> [App] : Support tasks, file downloads
[App] --> [notify] : Email notifications
[App] --> [sentry] : Alerts
[App] --> [osapi] : Address lookup
[cds] --> [cdsExport] : Downloads data for processing
[App] --> [ga] : Sends analytics data to Google Analytics
{% endplantuml %}

177
lib/tasks/update_manual_address_entry_selected_prexisting_logs.rake

@ -2,7 +2,21 @@ namespace :bulk_update do
desc "Update logs with specific criteria and set manual_address_entry_selected to true"
task update_manual_address_entry_selected: :environment do
updated_lettings_logs_count = 0
lettings_postcode_fixed_count = 0
lettings_postcode_fixed_status_changed_count = 0
lettings_postcode_not_fixed_status_changed_count = 0
lettings_postcode_fixed_status_changed_ids = []
lettings_postcode_not_fixed_status_changed_ids = []
lettings_updated_without_issue = 0
updated_sales_logs_count = 0
sales_postcode_fixed_count = 0
sales_postcode_fixed_status_changed_count = 0
sales_postcode_not_fixed_status_changed_count = 0
sales_postcode_fixed_status_changed_ids = []
sales_postcode_not_fixed_status_changed_ids = []
sales_updated_without_issue = 0
lettings_logs = LettingsLog.filter_by_year(2024)
.where(status: %w[in_progress completed])
.where(needstype: 1, manual_address_entry_selected: false, uprn: nil)
@ -13,16 +27,47 @@ namespace :bulk_update do
log.manual_address_entry_selected = true
if log.save
updated_lettings_logs_count += 1
Rails.logger.info "manual_address_entry_selected updated for lettings log #{log.id}"
else
Rails.logger.info "Could not save changes to lettings log #{log.id}"
Rails.logger.info "Could not save manual_address_entry_selected changes to lettings log #{log.id} : #{log.errors.full_messages.join(', ')}"
end
postcode_fixed = false
if log.postcode_full.nil? && log.address_line1 == log.address_line1_input
log.postcode_full = log.postcode_full_input
if log.save
lettings_postcode_fixed_count += 1
Rails.logger.info "postcode_full updated by address_line1_input for lettings log #{log.id}"
postcode_fixed = true
else
Rails.logger.info "Could not save postcode_full changes to lettings log #{log.id} : #{log.errors.full_messages.join(', ')}"
end
status_post_change = log.status
unless status_pre_change == status_post_change
Rails.logger.info "Status changed from #{status_pre_change} to #{status_post_change} for lettings log #{log.id}"
end
if log.postcode_full.nil? && log.creation_method == "bulk upload" && log.address_line1 == log.address_line1_as_entered
log.postcode_full = log.postcode_full_as_entered
if log.save
lettings_postcode_fixed_count += 1
Rails.logger.info "postcode_full updated by address_line1_as_entered for lettings log #{log.id}"
postcode_fixed = true
else
Rails.logger.info "Could not save postcode_full changes to lettings log #{log.id} : #{log.errors.full_messages.join(', ')}"
end
end
puts "#{updated_lettings_logs_count} lettings logs updated."
status_post_change = log.status
if status_pre_change != status_post_change
if postcode_fixed
lettings_postcode_fixed_status_changed_count += 1
lettings_postcode_fixed_status_changed_ids << log.id
else
lettings_postcode_not_fixed_status_changed_count += 1
lettings_postcode_not_fixed_status_changed_ids << log.id
end
else
lettings_updated_without_issue += 1
end
end
sales_logs = SalesLog.filter_by_year(2024)
.where(status: %w[in_progress completed])
@ -34,15 +79,129 @@ namespace :bulk_update do
log.manual_address_entry_selected = true
if log.save
updated_sales_logs_count += 1
Rails.logger.info "manual_address_entry_selected updated for sales log #{log.id}"
else
Rails.logger.info "Could not save manual_address_entry_selected changes to sales log #{log.id} : #{log.errors.full_messages.join(', ')}"
end
postcode_fixed = false
if log.postcode_full.nil? && log.address_line1 == log.address_line1_input
log.postcode_full = log.postcode_full_input
if log.save
sales_postcode_fixed_count += 1
Rails.logger.info "postcode_full updated by address_line1_input for sales log #{log.id}"
postcode_fixed = true
else
Rails.logger.info "Could not save changes to sales log #{log.id}"
Rails.logger.info "Could not save postcode_full changes to sales log #{log.id} : #{log.errors.full_messages.join(', ')}"
end
end
if log.postcode_full.nil? && log.creation_method == "bulk upload" && log.address_line1 == log.address_line1_as_entered
log.postcode_full = log.postcode_full_as_entered
if log.save
sales_postcode_fixed_count += 1
Rails.logger.info "postcode_full updated by address_line1_as_entered for sales log #{log.id}"
postcode_fixed = true
else
Rails.logger.info "Could not save postcode_full changes to sales log #{log.id} : #{log.errors.full_messages.join(', ')}"
end
end
status_post_change = log.status
unless status_pre_change == status_post_change
Rails.logger.info "Status changed from #{status_pre_change} to #{status_post_change} for sales log #{log.id}"
if status_pre_change != status_post_change
if postcode_fixed
sales_postcode_fixed_status_changed_count += 1
sales_postcode_fixed_status_changed_ids << log.id
else
sales_postcode_not_fixed_status_changed_count += 1
sales_postcode_not_fixed_status_changed_ids << log.id
end
else
sales_updated_without_issue += 1
end
end
puts "#{updated_lettings_logs_count} lettings logs were updated."
puts "#{lettings_updated_without_issue} lettings logs were updated without issue."
puts "#{lettings_postcode_fixed_count} lettings logs where postcode fix was applied."
puts "#{lettings_postcode_fixed_status_changed_count} lettings logs with postcode fix and status changed."
puts "#{lettings_postcode_not_fixed_status_changed_count} lettings logs without postcode fix and status changed."
puts "IDs of lettings logs with postcode fix and status changed: [#{lettings_postcode_fixed_status_changed_ids.join(', ')}]"
puts "IDs of lettings logs without postcode fix and status changed: [#{lettings_postcode_not_fixed_status_changed_ids.join(', ')}]"
lettings_postcode_fixed_org_counts = LettingsLog.where(id: lettings_postcode_fixed_status_changed_ids).group(:owning_organisation_id).count
lettings_postcode_fixed_org_counts.each do |org_id, count|
puts "Org #{org_id}: #{count} logs with postcode fix and status changed."
end
lettings_postcode_not_fixed_org_counts = LettingsLog.where(id: lettings_postcode_not_fixed_status_changed_ids).group(:owning_organisation_id).count
lettings_postcode_not_fixed_org_counts.each do |org_id, count|
puts "Org #{org_id}: #{count} logs without postcode fix and status changed."
end
puts "#{updated_sales_logs_count} sales logs were updated."
puts "#{sales_updated_without_issue} sales logs were updated without issue."
puts "#{sales_postcode_fixed_count} sales logs where postcode fix was applied."
puts "#{sales_postcode_fixed_status_changed_count} sales logs with postcode fix and status changed."
puts "#{sales_postcode_not_fixed_status_changed_count} sales logs without postcode fix and status changed."
puts "IDs of sales logs with postcode fix and status changed: [#{sales_postcode_fixed_status_changed_ids.join(', ')}]"
puts "IDs of sales logs without postcode fix and status changed: [#{sales_postcode_not_fixed_status_changed_ids.join(', ')}]"
sales_postcode_fixed_org_counts = SalesLog.where(id: sales_postcode_fixed_status_changed_ids).group(:owning_organisation_id).count
sales_postcode_fixed_org_counts.each do |org_id, count|
puts "Org #{org_id}: #{count} logs with postcode fix and status changed."
end
sales_postcode_not_fixed_org_counts = SalesLog.where(id: sales_postcode_not_fixed_status_changed_ids).group(:owning_organisation_id).count
sales_postcode_not_fixed_org_counts.each do |org_id, count|
puts "Org #{org_id}: #{count} logs without postcode fix and status changed."
end
end
desc "Find logs to fix and update postcode_full if conditions are met"
task update_postcode_full_preexisting_manual_entry_logs: :environment do
updated_count = 0
fixed_count = 0
not_updated_count = 0
not_updated_ids = []
updated_but_not_fixed_ids = []
logs_to_fix = LettingsLog.filter_by_year(2024).where(manual_address_entry_selected: true, uprn: nil, status: "in_progress", postcode_full: nil, updated_at: Time.zone.parse("2025-03-19 16:00:00")..Time.zone.parse("2025-03-19 17:00:00"))
logs_to_fix.find_each do |log|
previous_version = log.versions[-2]
previous_status = previous_version&.reify&.status
if log.address_line1 == log.address_line1_input
log.postcode_full = log.postcode_full_input
elsif log.creation_method == "bulk upload" && log.address_line1 == log.address_line1_as_entered
log.postcode_full = log.postcode_full_as_entered
end
if log.postcode_full.present?
if log.save
Rails.logger.info "Updated postcode_full for lettings log #{log.id}"
updated_count += 1
if log.status == previous_status
fixed_count += 1
else
updated_but_not_fixed_ids << log.id
end
else
Rails.logger.info "Could not save changes to lettings log #{log.id}: #{log.errors.full_messages.join(', ')}"
not_updated_count += 1
not_updated_ids << log.id
end
else
not_updated_count += 1
not_updated_ids << log.id
end
end
puts "#{updated_sales_logs_count} sales logs updated."
puts "#{updated_count} logs updated."
puts "#{fixed_count} logs fixed."
puts "#{not_updated_count} logs not updated."
puts "IDs of logs not updated: [#{not_updated_ids.join(', ')}]"
puts "IDs of logs updated but not fixed: [#{updated_but_not_fixed_ids.join(', ')}]"
end
end

102
spec/lib/tasks/update_manual_address_entry_selected_prexisting_logs_spec.rb

@ -1,7 +1,14 @@
require "rails_helper"
require "rake"
RSpec.describe "bulk_update:update_manual_address_entry_selected", type: :task do
RSpec.describe "update_manual_address_entry_selected_preexisting_logs_spec", type: :task do
before do
Rake.application.rake_require("tasks/update_manual_address_entry_selected_prexisting_logs")
Rake::Task.define_task(:environment)
task.reenable
end
describe "bulk_update:update_manual_address_entry_selected" do
let(:task) { Rake::Task["bulk_update:update_manual_address_entry_selected"] }
let(:lettings_log_uprn_entered) do
@ -36,17 +43,13 @@ RSpec.describe "bulk_update:update_manual_address_entry_selected", type: :task d
build(:sales_log, :completed_without_uprn, saledate: Time.zone.local(2024, 12, 30))
end
before do
Rake.application.rake_require("tasks/update_manual_address_entry_selected_prexisting_logs")
Rake::Task.define_task(:environment)
end
context "when running the task" do
context "when logs do not meet the criteria" do
before do
lettings_log_uprn_found.save!(validate: false)
lettings_log_uprn_entered.save!(validate: false)
lettings_log_address_fields_not_entered.save!(validate: false)
sales_log_uprn_found.save!(validate: false)
sales_log_uprn_entered.save!(validate: false)
sales_log_address_fields_not_entered.save!(validate: false)
@ -54,8 +57,10 @@ RSpec.describe "bulk_update:update_manual_address_entry_selected", type: :task d
it "does not update logs with a UPRN entered" do
task.invoke
lettings_log_uprn_entered.reload
sales_log_uprn_entered.reload
expect(lettings_log_uprn_entered.manual_address_entry_selected).to be false
expect(lettings_log_uprn_entered.uprn).to eq("10033558653")
expect(sales_log_uprn_entered.manual_address_entry_selected).to be false
@ -64,8 +69,10 @@ RSpec.describe "bulk_update:update_manual_address_entry_selected", type: :task d
it "does not update logs with a UPRN found" do
task.invoke
lettings_log_uprn_found.reload
sales_log_uprn_found.reload
expect(lettings_log_uprn_found.manual_address_entry_selected).to be false
expect(lettings_log_uprn_found.uprn).to eq("10033558653")
expect(sales_log_uprn_found.manual_address_entry_selected).to be false
@ -74,8 +81,10 @@ RSpec.describe "bulk_update:update_manual_address_entry_selected", type: :task d
it "does not update logs with no UPRN or address fields entered" do
task.invoke
lettings_log_address_fields_not_entered.reload
sales_log_address_fields_not_entered.reload
expect(lettings_log_address_fields_not_entered.manual_address_entry_selected).to be false
expect(sales_log_address_fields_not_entered.manual_address_entry_selected).to be false
end
@ -85,6 +94,7 @@ RSpec.describe "bulk_update:update_manual_address_entry_selected", type: :task d
before do
lettings_log_address_manually_entered.manual_address_entry_selected = false
lettings_log_address_manually_entered.save!(validate: false)
sales_log_address_manually_entered.manual_address_entry_selected = false
sales_log_address_manually_entered.save!(validate: false)
end
@ -95,6 +105,7 @@ RSpec.describe "bulk_update:update_manual_address_entry_selected", type: :task d
expect(lettings_log_address_manually_entered.address_line2).to eq("Testville")
expect(lettings_log_address_manually_entered.town_or_city).to eq("Testford")
expect(lettings_log_address_manually_entered.postcode_full).to eq("SW1 1AA")
expect(sales_log_address_manually_entered.manual_address_entry_selected).to be false
expect(sales_log_address_manually_entered.address_line1).to eq("1 Test Street")
expect(sales_log_address_manually_entered.address_line2).to eq("Testville")
@ -102,6 +113,7 @@ RSpec.describe "bulk_update:update_manual_address_entry_selected", type: :task d
expect(sales_log_address_manually_entered.postcode_full).to eq("SW1 1AA")
task.invoke
lettings_log_address_manually_entered.reload
sales_log_address_manually_entered.reload
@ -110,6 +122,7 @@ RSpec.describe "bulk_update:update_manual_address_entry_selected", type: :task d
expect(lettings_log_address_manually_entered.address_line2).to eq("Testville")
expect(lettings_log_address_manually_entered.town_or_city).to eq("Testford")
expect(lettings_log_address_manually_entered.postcode_full).to eq("SW1 1AA")
expect(sales_log_address_manually_entered.manual_address_entry_selected).to be true
expect(sales_log_address_manually_entered.address_line1).to eq("1 Test Street")
expect(sales_log_address_manually_entered.address_line2).to eq("Testville")
@ -118,4 +131,81 @@ RSpec.describe "bulk_update:update_manual_address_entry_selected", type: :task d
end
end
end
end
describe "bulk_update:update_postcode_full_preexisting_manual_entry_logs" do
let(:task) { Rake::Task["bulk_update:update_postcode_full_preexisting_manual_entry_logs"] }
let(:lettings_log_to_fix) do
build(:lettings_log, :inprogress_without_address_fields, startdate: Time.zone.local(2024, 6, 1), updated_at: Time.zone.parse("2025-03-19 16:30:00"))
end
let(:bu_lettings_log_to_fix) do
build(:lettings_log, :inprogress_without_address_fields, startdate: Time.zone.local(2024, 6, 1), creation_method: "bulk upload", updated_at: Time.zone.parse("2025-03-19 16:30:00"))
end
let(:lettings_log_not_to_fix) do
build(:lettings_log, :inprogress_without_address_fields, startdate: Time.zone.local(2024, 6, 1), updated_at: Time.zone.parse("2025-03-19 15:30:00"))
end
before do
lettings_log_to_fix.manual_address_entry_selected = true
lettings_log_to_fix.address_line1 = "1 Test Street"
lettings_log_to_fix.address_line2 = "Testville"
lettings_log_to_fix.town_or_city = "Testford"
lettings_log_to_fix.postcode_full = nil
lettings_log_to_fix.address_line1_input = "1 Test Street"
lettings_log_to_fix.postcode_full_input = "SW1 2BB"
lettings_log_to_fix.save!(validate: false)
bu_lettings_log_to_fix.manual_address_entry_selected = true
bu_lettings_log_to_fix.address_line1 = "1 Test Street"
bu_lettings_log_to_fix.address_line2 = "Testville"
bu_lettings_log_to_fix.town_or_city = "Testford"
bu_lettings_log_to_fix.postcode_full = nil
bu_lettings_log_to_fix.address_line1_as_entered = "1 Test Street"
bu_lettings_log_to_fix.postcode_full_as_entered = "SW1 2BB"
bu_lettings_log_to_fix.save!(validate: false)
lettings_log_not_to_fix.postcode_full = nil
lettings_log_not_to_fix.save!(validate: false)
end
context "when running the task" do
it "updates logs that meet the criteria" do
expect(lettings_log_to_fix.postcode_full).to be_nil
expect(lettings_log_to_fix.address_line1).to eq("1 Test Street")
expect(lettings_log_to_fix.address_line2).to eq("Testville")
expect(lettings_log_to_fix.town_or_city).to eq("Testford")
expect(lettings_log_to_fix.address_line1_input).to eq("1 Test Street")
expect(lettings_log_to_fix.postcode_full_input).to eq("SW1 2BB")
expect(bu_lettings_log_to_fix.postcode_full).to be_nil
expect(bu_lettings_log_to_fix.address_line1_input).to be_nil
expect(bu_lettings_log_to_fix.address_line1).to eq("1 Test Street")
expect(bu_lettings_log_to_fix.address_line2).to eq("Testville")
expect(bu_lettings_log_to_fix.town_or_city).to eq("Testford")
expect(bu_lettings_log_to_fix.address_line1_as_entered).to eq("1 Test Street")
expect(bu_lettings_log_to_fix.postcode_full_as_entered).to eq("SW1 2BB")
task.invoke
lettings_log_to_fix.reload
bu_lettings_log_to_fix.reload
expect(lettings_log_to_fix.postcode_full).to eq(lettings_log_to_fix.postcode_full_input)
expect(lettings_log_to_fix.postcode_full).to eq("SW1 2BB")
expect(bu_lettings_log_to_fix.postcode_full).to eq(bu_lettings_log_to_fix.postcode_full_as_entered)
expect(bu_lettings_log_to_fix.postcode_full).to eq("SW1 2BB")
end
it "does not update logs that do not meet the criteria" do
task.invoke
lettings_log_not_to_fix.reload
expect(lettings_log_not_to_fix.postcode_full).to be_nil
end
end
end
end

Loading…
Cancel
Save