Reduce differences between imported values and saved values

.github/workflows/production_pipeline.yml

@@ -180,4 +180,5 @@ jobs:
         cf set-env $APP_NAME IMPORT_PAAS_INSTANCE $IMPORT_PAAS_INSTANCE
         cf set-env $APP_NAME EXPORT_PAAS_INSTANCE $EXPORT_PAAS_INSTANCE
         cf set-env $APP_NAME SENTRY_DSN $SENTRY_DSN
+        cf set-env $APP_NAME RAILS_LOG_TO_STDOUT true
         cf push $APP_NAME --strategy rolling

.github/workflows/staging_pipeline.yml

@@ -148,4 +148,5 @@ jobs:
         cf set-env $APP_NAME IMPORT_PAAS_INSTANCE $IMPORT_PAAS_INSTANCE
         cf set-env $APP_NAME EXPORT_PAAS_INSTANCE $EXPORT_PAAS_INSTANCE
         cf set-env $APP_NAME SENTRY_DSN $SENTRY_DSN
+        cf set-env $APP_NAME RAILS_LOG_TO_STDOUT true
         cf push $APP_NAME --strategy rolling

app/services/imports/case_logs_import_service.rb

@@ -65,6 +65,7 @@ module Imports
       end
       (2..8).each do |index|
         attributes["relat#{index}"] = relat(xml_doc, index)
+        attributes["details_known_#{index}"] = details_known(index, attributes)
       end
       attributes["ethnic"] = unsafe_string_as_integer(xml_doc, "P1Eth")
       attributes["ethnic_group"] = ethnic_group(attributes["ethnic"])
@@ -140,17 +141,31 @@ module Imports
       attributes["postcode_full"] = compose_postcode(xml_doc, "POSTCODE", "POSTCOD2")
       attributes["postcode_known"] = attributes["postcode_full"].nil? ? 0 : 1
 
-      # Not specific to our form but required for CDS and can't be inferred
+      # Not specific to our form but required for consistency (present in import)
       attributes["old_form_id"] = Integer(field_value(xml_doc, "xmlns", "FORM"))
+      attributes["created_at"] = Time.zone.parse(field_value(xml_doc, "meta", "created-date"))
+      attributes["updated_at"] = Time.zone.parse(field_value(xml_doc, "meta", "modified-date"))
 
-      # Specific to us
+      # Required for our form invalidated questions (not present in import)
       attributes["previous_la_known"] = 1 # Defaulting to Yes (Required)
       attributes["la_known"] = 1 # Defaulting to Yes (Required)
-      attributes["created_at"] = Date.parse(field_value(xml_doc, "meta", "created-date"))
-      attributes["updated_at"] = Date.parse(field_value(xml_doc, "meta", "modified-date"))
+      attributes["is_la_inferred"] = false # Always keep the given LA
+      attributes["first_time_property_let_as_social_housing"] = first_time_let(attributes["rsnvac"])
 
       case_log = CaseLog.new(attributes)
       case_log.save!
+      compute_differences(case_log, attributes)
+    end
+
+    def compute_differences(case_log, attributes)
+      differences = []
+      attributes.each do |key, value|
+        case_log_value = case_log.send(key.to_sym)
+        if value != case_log_value
+          differences.push("#{key.to_s} #{value.to_s} #{case_log_value.to_s}")
+        end
+      end
+      raise "Differences found when saving log #{case_log.id}: #{differences}" unless differences.empty?
     end
 
     # Safe: A string that represents only an integer (or empty/nil)
@@ -162,7 +177,11 @@ module Imports
     # Safe: A string that represents only a decimal (or empty/nil)
     def safe_string_as_decimal(xml_doc, attribute)
       str = field_value(xml_doc, "xmlns", attribute)
-      BigDecimal(str, exception: false)
+      if str.blank?
+        BigDecimal("0.0")
+      else
+        BigDecimal(str, exception: false)
+      end
     end
 
     # Unsafe: A string that has more than just the integer value
@@ -182,7 +201,7 @@ module Imports
       if day.nil? || month.nil? || year.nil?
         nil
       else
-        Date.new(year, month, day)
+        Time.zone.local(year, month, day)
       end
     end
 
@@ -290,6 +309,19 @@ module Imports
       end
     end
 
+    def details_known(index, attributes)
+      if index > attributes["hhmemb"]
+        nil
+      elsif attributes["age#{index}_known"] == 1 &&
+        attributes["sex#{index}"] == "R" &&
+        attributes["relat#{index}"] == "R" &&
+        attributes["ecstat#{index}"] == 10
+        1 # No
+      else
+        0 # Yes
+      end
+    end
+
     def previous_postcode_known(xml_doc)
       previous_postcode_known = field_value(xml_doc, "xmlns", "Q12bnot")
       if previous_postcode_known == "Temporary or Unknown"
@@ -305,7 +337,7 @@ module Imports
       if outcode_value.blank? || incode_value.blank?
         nil
       else
-        "#{outcode_value} #{incode_value}"
+        "#{outcode_value}#{incode_value}"
       end
     end
 
@@ -388,5 +420,13 @@ module Imports
         0
       end
     end
+
+    def first_time_let(rsnvac)
+      if rsnvac == 15 || rsnvac == 16 || rsnvac == 17
+        1
+      else
+        0
+      end
+    end
   end
 end

app/services/imports/import_service.rb

@@ -13,6 +13,8 @@ module Imports
         file_io = @storage_service.get_file_io(filename)
         xml_document = Nokogiri::XML(file_io)
         send(create_method, xml_document)
+      rescue StandardError => e
+        @logger.error "#{e.class} #{e.message}"
       end
     end
 

config/forms/2021_2022.json

@@ -849,6 +849,38 @@
                 }
               },
               "depends_on": [
+                {
+                  "renewal": 0,
+                  "rsnvac": 5
+                },
+                {
+                  "renewal": 0,
+                  "rsnvac": 6
+                },
+                {
+                  "renewal": 0,
+                  "rsnvac": 8
+                },
+                {
+                  "renewal": 0,
+                  "rsnvac": 9
+                },
+                {
+                  "renewal": 0,
+                  "rsnvac": 10
+                },
+                {
+                  "renewal": 0,
+                  "rsnvac": 11
+                },
+                {
+                  "renewal": 0,
+                  "rsnvac": 12
+                },
+                {
+                  "renewal": 0,
+                  "rsnvac": 13
+                },
                 {
                   "renewal": 0,
                   "rsnvac": 16
@@ -856,6 +888,14 @@
                 {
                   "renewal": 0,
                   "rsnvac": 17
+                },
+                {
+                  "renewal": 0,
+                  "rsnvac": 18
+                },
+                {
+                  "renewal": 0,
+                  "rsnvac": 19
                 }
               ]
             },
@@ -1032,7 +1072,7 @@
               },
               "depends_on": [
                 {
-                  "startertenancy": 0
+                  "startertenancy": 2
                 }
               ]
             },

spec/services/imports/case_logs_import_service_spec.rb

@@ -9,9 +9,10 @@ RSpec.describe Imports::CaseLogsImportService do
   let(:case_log_file2) { File.open("#{fixture_directory}/#{case_log_id2}.xml") }
   let(:storage_service) { instance_double(StorageService) }
   let(:real_2021_2022_form) { Form.new("config/forms/2021_2022.json", "2021_2022") }
+  let(:logger) { instance_double(ActiveSupport::Logger) }
 
   context "when importing users" do
-    subject(:case_log_service) { described_class.new(storage_service) }
+    subject(:case_log_service) { described_class.new(storage_service, logger) }
 
     before do
       # Stub the S3 file listing and download
@@ -27,8 +28,10 @@ RSpec.describe Imports::CaseLogsImportService do
       allow(FormHandler.instance).to receive(:get_form).with(anything).and_return(real_2021_2022_form)
     end
 
-    it "successfully create a case log with the expected data" do
-      case_log_service.create_logs(remote_folder)
+    it "successfully create all case logs" do
+      expect(logger).not_to receive(:error)
+      expect { case_log_service.create_logs(remote_folder) }
+        .to change(CaseLog, :count).by(2)
     end
   end
 end

spec/services/imports/data_protection_confirmation_import_service_spec.rb

@@ -6,9 +6,10 @@ RSpec.describe Imports::DataProtectionConfirmationImportService do
   let(:old_id) { old_org_id }
   let(:import_file) { File.open("#{fixture_directory}/#{old_id}.xml") }
   let(:storage_service) { instance_double(StorageService) }
+  let(:logger) { instance_double(ActiveSupport::Logger) }
 
   context "when importing data protection confirmations" do
-    subject(:import_service) { described_class.new(storage_service) }
+    subject(:import_service) { described_class.new(storage_service, logger) }
 
     before do
       allow(storage_service)
@@ -22,8 +23,8 @@ RSpec.describe Imports::DataProtectionConfirmationImportService do
 
     context "when the organisation in the import file doesn't exist in the system" do
       it "does not create a data protection confirmation" do
-        expect { import_service.create_data_protection_confirmations("data_protection_directory") }
-          .to raise_error(ActiveRecord::RecordInvalid, /Organisation must exist/)
+        expect(logger).to receive(:error).with(/Organisation must exist/)
+        import_service.create_data_protection_confirmations("data_protection_directory")
       end
     end
 
@@ -73,7 +74,7 @@ RSpec.describe Imports::DataProtectionConfirmationImportService do
           end
 
           it "logs that the record already exists" do
-            expect(Rails.logger).to receive(:warn)
+            expect(logger).to receive(:warn)
             import_service.create_data_protection_confirmations("data_protection_directory")
           end
         end

spec/services/imports/user_import_service_spec.rb

@@ -6,9 +6,10 @@ RSpec.describe Imports::UserImportService do
   let(:old_org_id) { "7c5bd5fb549c09a2c55d7cb90d7ba84927e64618" }
   let(:user_file) { File.open("#{fixture_directory}/#{old_user_id}.xml") }
   let(:storage_service) { instance_double(StorageService) }
+  let(:logger) { instance_double(ActiveSupport::Logger) }
 
   context "when importing users" do
-    subject(:import_service) { described_class.new(storage_service) }
+    subject(:import_service) { described_class.new(storage_service, logger) }
 
     before do
       allow(storage_service).to receive(:list_files)
@@ -33,8 +34,8 @@ RSpec.describe Imports::UserImportService do
     end
 
     it "refuses to create a user belonging to a non existing organisation" do
-      expect { import_service.create_users("user_directory") }
-        .to raise_error(ActiveRecord::RecordInvalid, /Organisation must exist/)
+      expect(logger).to receive(:error).with(/Organisation must exist/)
+      import_service.create_users("user_directory")
     end
 
     context "when the user is a data coordinator" do
@@ -100,7 +101,7 @@ RSpec.describe Imports::UserImportService do
       end
 
       it "logs that the user already exists" do
-        expect(Rails.logger).to receive(:warn)
+        expect(logger).to receive(:warn)
         import_service.create_users("user_directory")
       end
     end