Rachael Booth
1 year ago
56 changed files with 866 additions and 303 deletions
@ -0,0 +1,6 @@
|
||||
class AddMigratedOnFields < ActiveRecord::Migration[7.0] |
||||
def change |
||||
add_column :lettings_logs, :values_updated_at, :datetime |
||||
add_column :sales_logs, :values_updated_at, :datetime |
||||
end |
||||
end |
||||
@ -0,0 +1,6 @@
|
||||
desc "Alter ppcodenk values for non imported lettings logs in the database" |
||||
task correct_ppcodenk_values: :environment do |
||||
LettingsLog.where.not(ppcodenk: nil).find_each do |log| |
||||
log.update_columns(ppcodenk: log.ppcodenk == 1 ? 0 : 1) |
||||
end |
||||
end |
||||
@ -0,0 +1,42 @@
|
||||
namespace :data_import do |
||||
desc "Import address data from a csv file" |
||||
task :import_address_from_csv, %i[file_name] => :environment do |_task, args| |
||||
file_name = args[:file_name] |
||||
|
||||
raise "Usage: rake data_import:import_address_from_csv['csv_file_name']" if file_name.blank? |
||||
|
||||
s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"]) |
||||
addresses_csv = CSV.parse(s3_service.get_file_io(file_name), headers: true) |
||||
|
||||
addresses_csv.each do |row| |
||||
lettings_log_id = row[0] |
||||
|
||||
if lettings_log_id.blank? |
||||
Rails.logger.info("Lettings log ID not provided for address: #{[row[1], row[2], row[3], row[4]].join(', ')}") |
||||
next |
||||
end |
||||
|
||||
lettings_log = LettingsLog.find_by(id: lettings_log_id) |
||||
if lettings_log.blank? |
||||
Rails.logger.info("Could not find a lettings log with id #{lettings_log_id}") |
||||
next |
||||
end |
||||
|
||||
lettings_log.uprn_known = 0 |
||||
lettings_log.uprn = nil |
||||
lettings_log.uprn_confirmed = nil |
||||
lettings_log.address_line1 = row[1] |
||||
lettings_log.address_line2 = row[2] |
||||
lettings_log.town_or_city = row[3] |
||||
lettings_log.postcode_full = row[4] |
||||
lettings_log.postcode_known = lettings_log.postcode_full.present? ? 1 : nil |
||||
lettings_log.county = nil |
||||
lettings_log.is_la_inferred = nil |
||||
lettings_log.la = nil |
||||
lettings_log.values_updated_at = Time.zone.now |
||||
|
||||
lettings_log.save! |
||||
Rails.logger.info("Updated lettings log #{lettings_log_id}, with address: #{[lettings_log.address_line1, lettings_log.address_line2, lettings_log.town_or_city, lettings_log.postcode_full].join(', ')}") |
||||
end |
||||
end |
||||
end |
||||
|
|
|
@ -0,0 +1,23 @@
|
||||
<institution xmlns="dclg:institution"> |
||||
<id>8c5bd5fb549c09z2c55d9cb90d7ba84927e64618</id> |
||||
<name>HA Ltd</name> |
||||
<institution-type>HOUSING-ASSOCIATION</institution-type> |
||||
<active>true</active> |
||||
<almsyn>false</almsyn> |
||||
<old-association-type>CHHA</old-association-type> |
||||
<telephone-number>xxxxxxxx</telephone-number> |
||||
<software-supplier-id>false</software-supplier-id> |
||||
<housing-management-system/> |
||||
<choice-based-letting>false</choice-based-letting> |
||||
<common-housing-register>false</common-housing-register> |
||||
<choice-allocation-policy>false</choice-allocation-policy> |
||||
<cbl-proportion-percentage/> |
||||
<enter-affordable-logs>true</enter-affordable-logs> |
||||
<owns-affordable-rent>true</owns-affordable-rent> |
||||
<housing-registration-no>LH9999</housing-registration-no> |
||||
<visible-id>1034</visible-id> |
||||
<holds-stock>true</holds-stock> |
||||
<general-needs-units>1104</general-needs-units> |
||||
<supported-housing-units>217</supported-housing-units> |
||||
<unspecified-units>0</unspecified-units> |
||||
</institution> |
||||
@ -0,0 +1,154 @@
|
||||
require "rails_helper" |
||||
require "rake" |
||||
|
||||
RSpec.describe "data_import" do |
||||
def replace_entity_ids(lettings_log, second_lettings_log, third_lettings_log, export_template) |
||||
export_template.sub!(/\{id\}/, lettings_log.id.to_s) |
||||
export_template.sub!(/\{id2\}/, second_lettings_log.id.to_s) |
||||
export_template.sub!(/\{id3\}/, third_lettings_log.id.to_s) |
||||
end |
||||
|
||||
describe ":import_address_from_csv", type: :task do |
||||
subject(:task) { Rake::Task["data_import:import_address_from_csv"] } |
||||
|
||||
let(:instance_name) { "paas_import_instance" } |
||||
let(:storage_service) { instance_double(Storage::S3Service) } |
||||
let(:env_config_service) { instance_double(Configuration::EnvConfigurationService) } |
||||
let(:paas_config_service) { instance_double(Configuration::PaasConfigurationService) } |
||||
|
||||
before do |
||||
allow(Storage::S3Service).to receive(:new).and_return(storage_service) |
||||
allow(Configuration::EnvConfigurationService).to receive(:new).and_return(env_config_service) |
||||
allow(Configuration::PaasConfigurationService).to receive(:new).and_return(paas_config_service) |
||||
allow(ENV).to receive(:[]) |
||||
allow(ENV).to receive(:[]).with("IMPORT_PAAS_INSTANCE").and_return(instance_name) |
||||
allow(ENV).to receive(:[]).with("VCAP_SERVICES").and_return("dummy") |
||||
|
||||
Rake.application.rake_require("tasks/import_address_from_csv") |
||||
Rake::Task.define_task(:environment) |
||||
task.reenable |
||||
|
||||
WebMock.stub_request(:get, /api.postcodes.io/) |
||||
.to_return(status: 200, body: "{\"status\":404,\"error\":\"Postcode not found\"}", headers: {}) |
||||
WebMock.stub_request(:get, /api.postcodes.io\/postcodes\/B11BB/) |
||||
.to_return(status: 200, body: '{"status":200,"result":{"admin_district":"Westminster","codes":{"admin_district":"E08000035"}}}', headers: {}) |
||||
end |
||||
|
||||
context "when the rake task is run" do |
||||
let(:addresses_csv_path) { "addresses_reimport_123.csv" } |
||||
let(:wrong_file_path) { "/test/no_csv_here.csv" } |
||||
let!(:lettings_log) do |
||||
create(:lettings_log, |
||||
uprn_known: nil, |
||||
uprn: nil, |
||||
uprn_confirmed: nil, |
||||
address_line1: nil, |
||||
address_line2: nil, |
||||
town_or_city: nil, |
||||
county: nil, |
||||
postcode_known: 1, |
||||
postcode_full: "A1 1AA", |
||||
la: "E06000064", |
||||
is_la_inferred: true) |
||||
end |
||||
|
||||
let!(:second_lettings_log) do |
||||
create(:lettings_log, |
||||
uprn_known: 1, |
||||
uprn: "1", |
||||
uprn_confirmed: nil, |
||||
address_line1: "wrong address line1", |
||||
address_line2: "wrong address 2", |
||||
town_or_city: "wrong town", |
||||
county: "wrong city", |
||||
postcode_known: 1, |
||||
postcode_full: "A1 1AA", |
||||
la: "E06000064", |
||||
is_la_inferred: true) |
||||
end |
||||
|
||||
let!(:third_lettings_log) do |
||||
create(:lettings_log, |
||||
uprn_known: 1, |
||||
uprn: "1", |
||||
uprn_confirmed: nil, |
||||
address_line1: "wrong address line1", |
||||
address_line2: "wrong address 2", |
||||
town_or_city: "wrong town", |
||||
county: "wrong city", |
||||
postcode_known: 1, |
||||
postcode_full: "A1 1AA", |
||||
la: "E06000064", |
||||
is_la_inferred: true) |
||||
end |
||||
|
||||
before do |
||||
allow(storage_service).to receive(:get_file_io) |
||||
.with("addresses_reimport_123.csv") |
||||
.and_return(replace_entity_ids(lettings_log, second_lettings_log, third_lettings_log, File.open("./spec/fixtures/files/addresses_reimport.csv").read)) |
||||
end |
||||
|
||||
it "updates the log address when old address was not given" do |
||||
task.invoke(addresses_csv_path) |
||||
lettings_log.reload |
||||
expect(lettings_log.uprn_known).to eq(0) |
||||
expect(lettings_log.uprn).to eq(nil) |
||||
expect(lettings_log.uprn_confirmed).to eq(nil) |
||||
expect(lettings_log.address_line1).to eq("address 1") |
||||
expect(lettings_log.address_line2).to eq("address 2") |
||||
expect(lettings_log.town_or_city).to eq("town") |
||||
expect(lettings_log.county).to eq(nil) |
||||
expect(lettings_log.postcode_known).to eq(1) |
||||
expect(lettings_log.postcode_full).to eq("B1 1BB") |
||||
expect(lettings_log.la).to eq("E08000035") |
||||
expect(lettings_log.is_la_inferred).to eq(true) |
||||
end |
||||
|
||||
it "updates the log address when old address was given" do |
||||
task.invoke(addresses_csv_path) |
||||
second_lettings_log.reload |
||||
expect(second_lettings_log.uprn_known).to eq(0) |
||||
expect(second_lettings_log.uprn).to eq(nil) |
||||
expect(second_lettings_log.uprn_confirmed).to eq(nil) |
||||
expect(second_lettings_log.address_line1).to eq("address 3") |
||||
expect(second_lettings_log.address_line2).to eq("address 4") |
||||
expect(second_lettings_log.town_or_city).to eq("city") |
||||
expect(second_lettings_log.county).to eq(nil) |
||||
expect(second_lettings_log.postcode_known).to eq(1) |
||||
expect(second_lettings_log.postcode_full).to eq("B1 1BB") |
||||
expect(second_lettings_log.la).to eq("E08000035") |
||||
expect(second_lettings_log.is_la_inferred).to eq(true) |
||||
end |
||||
|
||||
it "updates the log address when new address is not given" do |
||||
task.invoke(addresses_csv_path) |
||||
third_lettings_log.reload |
||||
expect(third_lettings_log.uprn_known).to eq(0) |
||||
expect(third_lettings_log.uprn).to eq(nil) |
||||
expect(third_lettings_log.uprn_confirmed).to eq(nil) |
||||
expect(third_lettings_log.address_line1).to eq(nil) |
||||
expect(third_lettings_log.address_line2).to eq(nil) |
||||
expect(third_lettings_log.town_or_city).to eq(nil) |
||||
expect(third_lettings_log.county).to eq(nil) |
||||
expect(third_lettings_log.postcode_known).to eq(1) |
||||
expect(third_lettings_log.postcode_full).to eq("C11CC") |
||||
expect(third_lettings_log.la).to eq(nil) |
||||
expect(third_lettings_log.is_la_inferred).to eq(false) |
||||
end |
||||
|
||||
it "logs the progress of the update" do |
||||
expect(Rails.logger).to receive(:info).with("Updated lettings log #{lettings_log.id}, with address: address 1, address 2, town, B1 1BB") |
||||
expect(Rails.logger).to receive(:info).with("Updated lettings log #{second_lettings_log.id}, with address: address 3, address 4, city, B1 1BB") |
||||
expect(Rails.logger).to receive(:info).with("Updated lettings log #{third_lettings_log.id}, with address: , , , C11CC") |
||||
expect(Rails.logger).to receive(:info).with("Lettings log ID not provided for address: address 5, address 6, city, D1 1DD") |
||||
expect(Rails.logger).to receive(:info).with("Could not find a lettings log with id fake_id") |
||||
|
||||
task.invoke(addresses_csv_path) |
||||
end |
||||
|
||||
it "raises an error when no path is given" do |
||||
expect { task.invoke(nil) }.to raise_error(RuntimeError, "Usage: rake data_import:import_address_from_csv['csv_file_name']") |
||||
end |
||||
end |
||||
end |
||||
end |
||||
@ -0,0 +1,67 @@
|
||||
require "rails_helper" |
||||
require "rake" |
||||
|
||||
RSpec.describe "correct_ppcodenk_values" do |
||||
describe ":correct_ppcodenk_values", type: :task do |
||||
subject(:task) { Rake::Task["correct_ppcodenk_values"] } |
||||
|
||||
before do |
||||
Rake.application.rake_require("tasks/correct_ppcodenk_values") |
||||
Rake::Task.define_task(:environment) |
||||
task.reenable |
||||
end |
||||
|
||||
context "when the rake task is run" do |
||||
let!(:lettings_log) { create(:lettings_log, :completed) } |
||||
|
||||
it "updates lettings logs with ppcodenk 0 to have ppcodenk 1" do |
||||
lettings_log.update!(ppcodenk: 0) |
||||
task.invoke |
||||
expect(lettings_log.reload.ppcodenk).to eq(1) |
||||
end |
||||
|
||||
it "updates lettings logs with ppcodenk 1 to have ppcodenk 0" do |
||||
lettings_log.update!(ppcodenk: 1) |
||||
task.invoke |
||||
expect(lettings_log.reload.ppcodenk).to eq(0) |
||||
end |
||||
|
||||
it "does not update lettings logs with ppcodenk nil" do |
||||
lettings_log.update!(ppcodenk: nil) |
||||
task.invoke |
||||
expect(lettings_log.reload.ppcodenk).to eq(nil) |
||||
end |
||||
|
||||
context "with multiple lettings logs" do |
||||
let(:lettings_log_2) { create(:lettings_log, :completed) } |
||||
let(:lettings_log_3) { create(:lettings_log, :completed) } |
||||
|
||||
it "only updates each log once" do |
||||
lettings_log.update!(ppcodenk: nil) |
||||
lettings_log_2.update!(ppcodenk: 0) |
||||
lettings_log_3.update!(ppcodenk: 1) |
||||
task.invoke |
||||
expect(lettings_log.reload.ppcodenk).to eq(nil) |
||||
expect(lettings_log_2.reload.ppcodenk).to eq(1) |
||||
expect(lettings_log_3.reload.ppcodenk).to eq(0) |
||||
end |
||||
end |
||||
|
||||
it "does not update updated_at value" do |
||||
lettings_log.updated_at = Time.zone.local(2021, 3, 3) |
||||
lettings_log.save!(validate: false) |
||||
expect(lettings_log.updated_at.to_date).to eq(Time.zone.local(2021, 3, 3)) |
||||
task.invoke |
||||
expect(lettings_log.updated_at.to_date).to eq(Time.zone.local(2021, 3, 3)) |
||||
end |
||||
|
||||
it "skips validations for previous years" do |
||||
lettings_log.update!(ppcodenk: 1) |
||||
lettings_log.startdate = Time.zone.local(2021, 3, 3) |
||||
lettings_log.save!(validate: false) |
||||
task.invoke |
||||
expect(lettings_log.reload.ppcodenk).to eq(0) |
||||
end |
||||
end |
||||
end |
||||
end |
||||
Loading…
Reference in new issue