diff --git a/.github/workflows/production_pipeline.yml b/.github/workflows/production_pipeline.yml index 09020a74b..d2fb26fd9 100644 --- a/.github/workflows/production_pipeline.yml +++ b/.github/workflows/production_pipeline.yml @@ -180,4 +180,5 @@ jobs: cf set-env $APP_NAME IMPORT_PAAS_INSTANCE $IMPORT_PAAS_INSTANCE cf set-env $APP_NAME EXPORT_PAAS_INSTANCE $EXPORT_PAAS_INSTANCE cf set-env $APP_NAME SENTRY_DSN $SENTRY_DSN + cf set-env $APP_NAME RAILS_LOG_TO_STDOUT true cf push $APP_NAME --strategy rolling diff --git a/.github/workflows/staging_pipeline.yml b/.github/workflows/staging_pipeline.yml index 9ba8eee7f..785df00a4 100644 --- a/.github/workflows/staging_pipeline.yml +++ b/.github/workflows/staging_pipeline.yml @@ -148,4 +148,5 @@ jobs: cf set-env $APP_NAME IMPORT_PAAS_INSTANCE $IMPORT_PAAS_INSTANCE cf set-env $APP_NAME EXPORT_PAAS_INSTANCE $EXPORT_PAAS_INSTANCE cf set-env $APP_NAME SENTRY_DSN $SENTRY_DSN + cf set-env $APP_NAME RAILS_LOG_TO_STDOUT true cf push $APP_NAME --strategy rolling diff --git a/app/services/imports/case_logs_import_service.rb b/app/services/imports/case_logs_import_service.rb index 806f55856..fe719065d 100644 --- a/app/services/imports/case_logs_import_service.rb +++ b/app/services/imports/case_logs_import_service.rb @@ -65,6 +65,7 @@ module Imports end (2..8).each do |index| attributes["relat#{index}"] = relat(xml_doc, index) + attributes["details_known_#{index}"] = details_known(index, attributes) end attributes["ethnic"] = unsafe_string_as_integer(xml_doc, "P1Eth") attributes["ethnic_group"] = ethnic_group(attributes["ethnic"]) @@ -140,17 +141,31 @@ module Imports attributes["postcode_full"] = compose_postcode(xml_doc, "POSTCODE", "POSTCOD2") attributes["postcode_known"] = attributes["postcode_full"].nil? ? 0 : 1 - # Not specific to our form but required for CDS and can't be inferred + # Not specific to our form but required for consistency (present in import) attributes["old_form_id"] = Integer(field_value(xml_doc, "xmlns", "FORM")) + attributes["created_at"] = Time.zone.parse(field_value(xml_doc, "meta", "created-date")) + attributes["updated_at"] = Time.zone.parse(field_value(xml_doc, "meta", "modified-date")) - # Specific to us + # Required for our form invalidated questions (not present in import) attributes["previous_la_known"] = 1 # Defaulting to Yes (Required) attributes["la_known"] = 1 # Defaulting to Yes (Required) - attributes["created_at"] = Date.parse(field_value(xml_doc, "meta", "created-date")) - attributes["updated_at"] = Date.parse(field_value(xml_doc, "meta", "modified-date")) + attributes["is_la_inferred"] = false # Always keep the given LA + attributes["first_time_property_let_as_social_housing"] = first_time_let(attributes["rsnvac"]) case_log = CaseLog.new(attributes) case_log.save! + compute_differences(case_log, attributes) + end + + def compute_differences(case_log, attributes) + differences = [] + attributes.each do |key, value| + case_log_value = case_log.send(key.to_sym) + if value != case_log_value + differences.push("#{key.to_s} #{value.to_s} #{case_log_value.to_s}") + end + end + raise "Differences found when saving log #{case_log.id}: #{differences}" unless differences.empty? end # Safe: A string that represents only an integer (or empty/nil) @@ -162,7 +177,11 @@ module Imports # Safe: A string that represents only a decimal (or empty/nil) def safe_string_as_decimal(xml_doc, attribute) str = field_value(xml_doc, "xmlns", attribute) - BigDecimal(str, exception: false) + if str.blank? + BigDecimal("0.0") + else + BigDecimal(str, exception: false) + end end # Unsafe: A string that has more than just the integer value @@ -182,7 +201,7 @@ module Imports if day.nil? || month.nil? || year.nil? nil else - Date.new(year, month, day) + Time.zone.local(year, month, day) end end @@ -290,6 +309,19 @@ module Imports end end + def details_known(index, attributes) + if index > attributes["hhmemb"] + nil + elsif attributes["age#{index}_known"] == 1 && + attributes["sex#{index}"] == "R" && + attributes["relat#{index}"] == "R" && + attributes["ecstat#{index}"] == 10 + 1 # No + else + 0 # Yes + end + end + def previous_postcode_known(xml_doc) previous_postcode_known = field_value(xml_doc, "xmlns", "Q12bnot") if previous_postcode_known == "Temporary or Unknown" @@ -305,7 +337,7 @@ module Imports if outcode_value.blank? || incode_value.blank? nil else - "#{outcode_value} #{incode_value}" + "#{outcode_value}#{incode_value}" end end @@ -388,5 +420,13 @@ module Imports 0 end end + + def first_time_let(rsnvac) + if rsnvac == 15 || rsnvac == 16 || rsnvac == 17 + 1 + else + 0 + end + end end end diff --git a/app/services/imports/import_service.rb b/app/services/imports/import_service.rb index 8ac8a035d..5d0193d93 100644 --- a/app/services/imports/import_service.rb +++ b/app/services/imports/import_service.rb @@ -13,6 +13,8 @@ module Imports file_io = @storage_service.get_file_io(filename) xml_document = Nokogiri::XML(file_io) send(create_method, xml_document) + rescue StandardError => e + @logger.error "#{e.class} #{e.message}" end end diff --git a/config/forms/2021_2022.json b/config/forms/2021_2022.json index 71ab5ad47..dbba6b37e 100644 --- a/config/forms/2021_2022.json +++ b/config/forms/2021_2022.json @@ -849,6 +849,38 @@ } }, "depends_on": [ + { + "renewal": 0, + "rsnvac": 5 + }, + { + "renewal": 0, + "rsnvac": 6 + }, + { + "renewal": 0, + "rsnvac": 8 + }, + { + "renewal": 0, + "rsnvac": 9 + }, + { + "renewal": 0, + "rsnvac": 10 + }, + { + "renewal": 0, + "rsnvac": 11 + }, + { + "renewal": 0, + "rsnvac": 12 + }, + { + "renewal": 0, + "rsnvac": 13 + }, { "renewal": 0, "rsnvac": 16 @@ -856,6 +888,14 @@ { "renewal": 0, "rsnvac": 17 + }, + { + "renewal": 0, + "rsnvac": 18 + }, + { + "renewal": 0, + "rsnvac": 19 } ] }, @@ -1032,7 +1072,7 @@ }, "depends_on": [ { - "startertenancy": 0 + "startertenancy": 2 } ] }, diff --git a/spec/services/imports/case_logs_import_service_spec.rb b/spec/services/imports/case_logs_import_service_spec.rb index 55b6cadca..111106f31 100644 --- a/spec/services/imports/case_logs_import_service_spec.rb +++ b/spec/services/imports/case_logs_import_service_spec.rb @@ -9,9 +9,10 @@ RSpec.describe Imports::CaseLogsImportService do let(:case_log_file2) { File.open("#{fixture_directory}/#{case_log_id2}.xml") } let(:storage_service) { instance_double(StorageService) } let(:real_2021_2022_form) { Form.new("config/forms/2021_2022.json", "2021_2022") } + let(:logger) { instance_double(ActiveSupport::Logger) } context "when importing users" do - subject(:case_log_service) { described_class.new(storage_service) } + subject(:case_log_service) { described_class.new(storage_service, logger) } before do # Stub the S3 file listing and download @@ -27,8 +28,10 @@ RSpec.describe Imports::CaseLogsImportService do allow(FormHandler.instance).to receive(:get_form).with(anything).and_return(real_2021_2022_form) end - it "successfully create a case log with the expected data" do - case_log_service.create_logs(remote_folder) + it "successfully create all case logs" do + expect(logger).not_to receive(:error) + expect { case_log_service.create_logs(remote_folder) } + .to change(CaseLog, :count).by(2) end end end diff --git a/spec/services/imports/data_protection_confirmation_import_service_spec.rb b/spec/services/imports/data_protection_confirmation_import_service_spec.rb index eddfca6a2..12f260618 100644 --- a/spec/services/imports/data_protection_confirmation_import_service_spec.rb +++ b/spec/services/imports/data_protection_confirmation_import_service_spec.rb @@ -6,9 +6,10 @@ RSpec.describe Imports::DataProtectionConfirmationImportService do let(:old_id) { old_org_id } let(:import_file) { File.open("#{fixture_directory}/#{old_id}.xml") } let(:storage_service) { instance_double(StorageService) } + let(:logger) { instance_double(ActiveSupport::Logger) } context "when importing data protection confirmations" do - subject(:import_service) { described_class.new(storage_service) } + subject(:import_service) { described_class.new(storage_service, logger) } before do allow(storage_service) @@ -22,8 +23,8 @@ RSpec.describe Imports::DataProtectionConfirmationImportService do context "when the organisation in the import file doesn't exist in the system" do it "does not create a data protection confirmation" do - expect { import_service.create_data_protection_confirmations("data_protection_directory") } - .to raise_error(ActiveRecord::RecordInvalid, /Organisation must exist/) + expect(logger).to receive(:error).with(/Organisation must exist/) + import_service.create_data_protection_confirmations("data_protection_directory") end end @@ -73,7 +74,7 @@ RSpec.describe Imports::DataProtectionConfirmationImportService do end it "logs that the record already exists" do - expect(Rails.logger).to receive(:warn) + expect(logger).to receive(:warn) import_service.create_data_protection_confirmations("data_protection_directory") end end diff --git a/spec/services/imports/user_import_service_spec.rb b/spec/services/imports/user_import_service_spec.rb index 70552f41a..91487b593 100644 --- a/spec/services/imports/user_import_service_spec.rb +++ b/spec/services/imports/user_import_service_spec.rb @@ -6,9 +6,10 @@ RSpec.describe Imports::UserImportService do let(:old_org_id) { "7c5bd5fb549c09a2c55d7cb90d7ba84927e64618" } let(:user_file) { File.open("#{fixture_directory}/#{old_user_id}.xml") } let(:storage_service) { instance_double(StorageService) } + let(:logger) { instance_double(ActiveSupport::Logger) } context "when importing users" do - subject(:import_service) { described_class.new(storage_service) } + subject(:import_service) { described_class.new(storage_service, logger) } before do allow(storage_service).to receive(:list_files) @@ -33,8 +34,8 @@ RSpec.describe Imports::UserImportService do end it "refuses to create a user belonging to a non existing organisation" do - expect { import_service.create_users("user_directory") } - .to raise_error(ActiveRecord::RecordInvalid, /Organisation must exist/) + expect(logger).to receive(:error).with(/Organisation must exist/) + import_service.create_users("user_directory") end context "when the user is a data coordinator" do @@ -100,7 +101,7 @@ RSpec.describe Imports::UserImportService do end it "logs that the user already exists" do - expect(Rails.logger).to receive(:warn) + expect(logger).to receive(:warn) import_service.create_users("user_directory") end end