CLDC-2315 Add bulk upload inferences (#1592)

* feat: bulk upload complete => redirect to all logs regardless of filter memory

* feat: derive variables if buyer not interviewed

* feat: remove redundant derivation that is overrided by other derivation

* feat: update url for lettings

* tests: tests inferences

* tests: update

* feat: update links

* feat: update urls to use url helper

* feat: update urls to use url helper

* feat: update logic

* test: update

* test: update

* test: update

* feat: infer ecstat1 as 10 not 0

* feat: update tests

app/mailers/bulk_upload_mailer.rb

@@ -27,9 +27,9 @@ class BulkUploadMailer < NotifyMailer
 
   def send_bulk_upload_complete_mail(user:, bulk_upload:)
     url = if bulk_upload.lettings?
-            lettings_logs_url
+            lettings_logs_url("[years][]" => "", "[status][]" => "", user: :all, organisation_select: :all)
           else
-            sales_logs_url
+            sales_logs_url("[years][]" => "", "[status][]" => "", user: :all)
           end
 
     n_logs = pluralize(bulk_upload.logs.count, "log")

app/services/bulk_upload/sales/year2022/row_parser.rb

@@ -339,6 +339,10 @@ class BulkUpload::Sales::Year2022::RowParser
 
 private
 
+  def buyer_not_interviewed?
+    field_6 == 1
+  end
+
   def shared_ownership?
     field_113 == 1
   end
@@ -478,7 +482,7 @@ private
     attributes["purchid"] = field_1
     attributes["saledate"] = saledate
 
-    attributes["noint"] = 2 if field_6 == 1
+    attributes["noint"] = field_6
 
     attributes["details_known_2"] = details_known?(2)
     attributes["details_known_3"] = details_known?(3)
@@ -517,7 +521,7 @@ private
     attributes["relat5"] = field_22
     attributes["relat6"] = field_23
 
-    attributes["ecstat1"] = field_24
+    attributes["ecstat1"] = buyer_not_interviewed? && field_24.blank? ? 10 : field_24
     attributes["ecstat2"] = field_25
     attributes["ecstat3"] = field_26
     attributes["ecstat4"] = field_27
@@ -526,18 +530,18 @@ private
 
     attributes["ethnic_group"] = ethnic_group_from_ethnic
     attributes["ethnic"] = field_30
-    attributes["national"] = field_31
-    attributes["income1nk"] = field_32.present? ? 0 : 1
+    attributes["national"] = buyer_not_interviewed? && field_31.blank? ? 13 : field_31
+    attributes["income1nk"] = income1nk
     attributes["income1"] = field_32
     attributes["income2nk"] = field_33.present? ? 0 : 1
     attributes["income2"] = field_33
-    attributes["inc1mort"] = field_34
+    attributes["inc1mort"] = buyer_not_interviewed? && field_32.blank? ? 2 : field_34
     attributes["inc2mort"] = field_35
-    attributes["savingsnk"] = field_36.present? ? 0 : 1
+    attributes["savingsnk"] = savingsnk
     attributes["savings"] = field_36
-    attributes["prevown"] = field_37
+    attributes["prevown"] = buyer_not_interviewed? && field_37.blank? ? 3 : field_37
 
-    attributes["prevten"] = field_39
+    attributes["prevten"] = buyer_not_interviewed? && field_39.blank? ? 0 : field_39
     attributes["prevloc"] = field_40
     attributes["previous_la_known"] = previous_la_known
     attributes["ppcodenk"] = field_43
@@ -549,8 +553,8 @@ private
     attributes["pregother"] = field_47
     attributes["pregblank"] = 1 if [field_44, field_45, field_46, field_47].all?(&:blank?)
 
-    attributes["disabled"] = field_48
-    attributes["wheel"] = field_49
+    attributes["disabled"] = buyer_not_interviewed? && field_48.blank? ? 3 : field_48
+    attributes["wheel"] = buyer_not_interviewed? && field_49.blank? ? 3 : field_49
     attributes["beds"] = field_50
     attributes["proptype"] = field_51
     attributes["builtype"] = field_52
@@ -620,6 +624,22 @@ private
     attributes
   end
 
+  def income1nk
+    if field_32.present?
+      0
+    else
+      buyer_not_interviewed? ? 1 : nil
+    end
+  end
+
+  def savingsnk
+    if field_36.present?
+      0
+    else
+      buyer_not_interviewed? ? 1 : nil
+    end
+  end
+
   def saledate
     Date.new(field_4 + 2000, field_3, field_2) if field_2.present? && field_3.present? && field_4.present?
   rescue Date::Error
@@ -686,21 +706,23 @@ private
   end
 
   def ethnic_group_from_ethnic
-    return nil if field_30.blank?
-
-    case field_30
-    when 1, 2, 3, 18
-      0
-    when 4, 5, 6, 7
-      1
-    when 8, 9, 10, 11, 15
-      2
-    when 12, 13, 14
-      3
-    when 16, 19
-      4
-    when 17
-      17
+    if field_30.blank?
+      buyer_not_interviewed? ? 17 : nil
+    else
+      case field_30
+      when 1, 2, 3, 18
+        0
+      when 4, 5, 6, 7
+        1
+      when 8, 9, 10, 11, 15
+        2
+      when 12, 13, 14
+        3
+      when 16, 19
+        4
+      when 17
+        17
+      end
     end
   end
 

spec/mailers/bulk_upload_mailer_spec.rb

@@ -46,7 +46,7 @@ RSpec.describe BulkUploadMailer do
           filename: bulk_upload.filename,
           upload_timestamp: bulk_upload.created_at.to_fs(:govuk_date_and_time),
           success_description: "The lettings 2022/23 data you uploaded has been checked. The 0 logs you uploaded are now complete.",
-          logs_link: lettings_logs_url,
+          logs_link: lettings_logs_url("[years][]" => "", "[status][]" => "", user: :all, organisation_select: :all),
         },
       )
 

spec/services/bulk_upload/sales/year2022/row_parser_spec.rb

@@ -540,4 +540,46 @@ RSpec.describe BulkUpload::Sales::Year2022::RowParser do
       end
     end
   end
+
+  describe "inferences" do
+    context "when buyer not interviewed and optional values nil" do
+      let(:attributes) { valid_attributes.merge(field_6: "1", field_24: nil, field_30: nil, field_31: nil, field_32: nil, field_34: nil, field_36: nil, field_37: nil, field_39: nil, field_48: nil, field_49: nil) }
+
+      it "infers correctly" do
+        log = parser.log
+        expect(log["noint"]).to eq(1)
+        expect(log["ecstat1"]).to eq(10)
+        expect(log["ethnic"]).to eq(nil)
+        expect(log["ethnic_group"]).to eq(17)
+        expect(log["national"]).to eq(13)
+        expect(log["income1nk"]).to eq(1)
+        expect(log["inc1mort"]).to eq(2)
+        expect(log["savingsnk"]).to eq(1)
+        expect(log["prevown"]).to eq(3)
+        expect(log["prevten"]).to eq(0)
+        expect(log["disabled"]).to eq(3)
+        expect(log["wheel"]).to eq(3)
+      end
+    end
+
+    context "when buyer not interviewed and optional values present" do
+      let(:attributes) { valid_attributes.merge(field_6: "1", field_24: "1", field_30: "1", field_31: "1", field_32: "1", field_34: "1", field_36: "1", field_37: "1", field_39: "1", field_48: "1", field_49: "1") }
+
+      it "does not override variables correctly" do
+        log = parser.log
+        expect(log["noint"]).to eq(1)
+        expect(log["ecstat1"]).to eq(1)
+        expect(log["ethnic"]).to eq(1)
+        expect(log["ethnic_group"]).to eq(0)
+        expect(log["national"]).to eq(1)
+        expect(log["income1nk"]).to eq(0)
+        expect(log["inc1mort"]).to eq(1)
+        expect(log["savingsnk"]).to eq(0)
+        expect(log["prevown"]).to eq(1)
+        expect(log["prevten"]).to eq(1)
+        expect(log["disabled"]).to eq(1)
+        expect(log["wheel"]).to eq(1)
+      end
+    end
+  end
 end