CLDC-638: Bulk upload (#84)

* Add file broswer view

* Read file

* Happy path

* Route constraint

* Some placeholders

* Add more fields

* Inferred vals

* Add some file type error handling

* Spec empty file handling

* Fail silently for now

* Collection route

* Rubocop

* Conventional routes are easier to reason about than constraints

* Remove uneeded fields

* Add a guard clause

* Allow log creation with bad rows.

* Update major repairs field name

* Make route bulk upload singular

* More realistic spec file

* Fix merge conflict resolution

* Set hhemb

Gemfile

@@ -29,6 +29,8 @@ gem "discard"
 gem "activeadmin"
 # Admin charts
 gem "chartkick"
+# Spreadsheet parsing
+gem "roo"
 # Json Schema
 gem "json-schema"
 gem "uk_postcode"

Gemfile.lock

@@ -294,6 +294,9 @@ GEM
       actionpack (>= 5.0)
       railties (>= 5.0)
     rexml (3.2.5)
+    roo (2.8.3)
+      nokogiri (~> 1)
+      rubyzip (>= 1.3.0, < 3.0.0)
     rubocop (1.21.0)
       parallel (~> 1.10)
       parser (>= 3.0.0.0)
@@ -409,6 +412,7 @@ DEPENDENCIES
   puma (~> 5.0)
   rack-mini-profiler (~> 2.0)
   rails (~> 6.1.4)
+  roo
   rspec-core!
   rspec-expectations!
   rspec-mocks!

app/controllers/bulk_upload_controller.rb

@@ -0,0 +1,22 @@
+class BulkUploadController < ApplicationController
+  def show
+    @bulk_upload = BulkUpload.new(nil, nil)
+    render "case_logs/bulk_upload"
+  end
+
+  def bulk_upload
+    file = upload_params.tempfile
+    content_type = upload_params.content_type
+    @bulk_upload = BulkUpload.new(file, content_type)
+    @bulk_upload.process
+    if @bulk_upload.errors.present?
+      render "case_logs/bulk_upload", status: :unprocessable_entity
+    else
+      redirect_to(case_logs_path)
+    end
+  end
+
+  def upload_params
+    params.require("bulk_upload")["case_log_bulk_upload"]
+  end
+end

app/models/bulk_upload.rb

@@ -0,0 +1,199 @@
+class BulkUpload
+  include ActiveModel::Model
+  include ActiveModel::Validations
+  include ActiveModel::Conversion
+
+  SPREADSHEET_CONTENT_TYPES = %w[
+    application/vnd.ms-excel
+    application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
+  ].freeze
+
+  FIRST_DATA_ROW = 7
+
+  def initialize(file, content_type)
+    @file = file
+    @content_type = content_type
+  end
+
+  def process
+    return unless valid_content_type?
+
+    xlsx = Roo::Spreadsheet.open(@file, extension: :xlsx)
+    sheet = xlsx.sheet(0)
+    last_row = sheet.last_row
+    if last_row < FIRST_DATA_ROW
+      errors.add(:case_log_bulk_upload, "No data found")
+    else
+      data_range = FIRST_DATA_ROW..last_row
+      data_range.map do |row_num|
+        case_log = CaseLog.create
+        map_row(sheet.row(row_num)).each do |attr_key, attr_val|
+          begin
+            case_log.update_attribute(attr_key, attr_val)
+          rescue ArgumentError
+          end
+        end
+      end
+    end
+  end
+
+  def valid_content_type?
+    if SPREADSHEET_CONTENT_TYPES.include?(@content_type)
+      true
+    else
+      errors.add(:case_log_bulk_upload, "Invalid file type")
+      false
+    end
+  end
+
+  def map_row(row)
+    {
+      lettype: row[1],
+      landlord: row[2],
+      # reg_num_la_core_code: row[3],
+      # managementgroup: row[4],
+      # schemecode: row[5],
+      # firstletting: row[6],
+      tenant_code: row[7],
+      startertenancy: row[8],
+      tenancy: row[9],
+      tenancyother: row[10],
+      # tenancyduration: row[11],
+      other_hhmemb: other_hhmemb(row),
+      hhmemb: other_hhmemb(row) + 1,
+      age1: row[12],
+      age2: row[13],
+      age3: row[14],
+      age4: row[15],
+      age5: row[16],
+      age6: row[17],
+      age7: row[18],
+      age8: row[19],
+      sex1: row[20],
+      sex2: row[21],
+      sex3: row[22],
+      sex4: row[23],
+      sex5: row[24],
+      sex6: row[25],
+      sex7: row[26],
+      sex8: row[27],
+      relat2: row[28],
+      relat3: row[29],
+      relat4: row[30],
+      relat5: row[31],
+      relat6: row[32],
+      relat7: row[33],
+      relat8: row[34],
+      ecstat1: row[35],
+      ecstat2: row[36],
+      ecstat3: row[37],
+      ecstat4: row[38],
+      ecstat5: row[39],
+      ecstat6: row[40],
+      ecstat7: row[41],
+      ecstat8: row[42],
+      ethnic: row[43],
+      national: row[44],
+      armed_forces: row[45],
+      reservist: row[46],
+      preg_occ: row[47],
+      hb: row[48],
+      benefits: row[49],
+      net_income_known: row[50].present? ? 1 : nil,
+      earnings: row[50],
+      # increfused: row[51],
+      reason: row[52],
+      other_reason_for_leaving_last_settled_home: row[53],
+      underoccupation_benefitcap: row[54],
+      housingneeds_a: row[55],
+      housingneeds_b: row[56],
+      housingneeds_c: row[57],
+      housingneeds_f: row[58],
+      housingneeds_g: row[59],
+      housingneeds_h: row[60],
+      prevten: row[61],
+      prevloc: row[62],
+      # ppostc1: row[63],
+      # ppostc2: row[64],
+      # prevpco_unknown: row[65],
+      layear: row[66],
+      lawaitlist: row[67],
+      homeless: row[68],
+      reasonpref: row[69],
+      rp_homeless: row[70],
+      rp_insan_unsat: row[71],
+      rp_medwel: row[72],
+      rp_hardship: row[73],
+      rp_dontknow: row[74],
+      cbl: row[75],
+      chr: row[76],
+      cap: row[77],
+      # referral_source: row[78],
+      period: row[79],
+      brent: row[80],
+      scharge: row[81],
+      pscharge: row[82],
+      supcharg: row[83],
+      tcharge: row[84],
+      # tcharge_care_homes: row[85],
+      # no_rent_or_charge: row[86],
+      hbrentshortfall: row[87],
+      tshortfall: row[88],
+      property_void_date: row[89].to_s + row[90].to_s + row[91].to_s,
+      # property_void_date_day: row[89],
+      # property_void_date_month: row[90],
+      # property_void_date_year: row[91],
+      majorrepairs: row[92].present? ? "1" : nil,
+      mrcdate: row[92].to_s + row[93].to_s + row[94].to_s,
+      mrcday: row[92],
+      mrcmonth: row[93],
+      mrcyear: row[94],
+      # supported_scheme: row[95],
+      startdate: row[96].to_s + row[97].to_s + row[98].to_s,
+      # startdate_day: row[96],
+      # startdate_month: row[97],
+      # startdate_year: row[98],
+      offered: row[99],
+      # property_reference: row[100],
+      beds: row[101],
+      unittype_gn: row[102],
+      property_building_type: row[103],
+      wchair: row[104],
+      property_relet: row[105],
+      rsnvac: row[106],
+      la: row[107],
+      # postcode: row[108],
+      # postcod2: row[109],
+      # row[110] removed
+      property_owner_organisation: row[111],
+      # username: row[112],
+      property_manager_organisation: row[113],
+      leftreg: row[114],
+      # uprn: row[115],
+      incfreq: row[116],
+      # sheltered_accom: row[117],
+      illness: row[118],
+      illness_type_1: row[119],
+      illness_type_2: row[120],
+      illness_type_3: row[121],
+      illness_type_4: row[122],
+      illness_type_8: row[123],
+      illness_type_5: row[124],
+      illness_type_6: row[125],
+      illness_type_7: row[126],
+      illness_type_9: row[127],
+      illness_type_10: row[128],
+      # london_affordable: row[129],
+      rent_type: row[130],
+      intermediate_rent_product_name: row[131],
+      # data_protection: row[132],
+      sale_or_letting: "letting",
+      gdpr_acceptance: 1,
+      gdpr_declined: 0,
+    }
+  end
+
+  def other_hhmemb(row)
+    [13, 14, 15, 16, 17, 18, 19].count { |idx| row[idx].present? }
+  end
+end

app/models/case_log.rb

@@ -39,7 +39,6 @@ class CaseLog < ApplicationRecord
   include SoftValidations
   include DbEnums
   default_scope -> { kept }
-  scope :not_completed, -> { where.not(status: "completed") }
 
   validates_with CaseLogValidator
   before_save :update_status!
@@ -129,6 +128,8 @@ class CaseLog < ApplicationRecord
   end
 
   def weekly_net_income
+    return unless earnings && incfreq
+
     case incfreq
     when "Weekly"
       earnings

app/views/case_logs/bulk_upload.html.erb

@@ -0,0 +1,10 @@
+<div class="govuk-form-group">
+  <%= form_for @bulk_upload, url: bulk_upload_case_logs_path, method: "post", builder: GOVUKDesignSystemFormBuilder::FormBuilder do |f| %>
+    <%= f.govuk_error_summary %>
+    <%= f.govuk_file_field :case_log_bulk_upload,
+        label: { text: "Bulk Upload", size: "l" },
+        hint: { text: "Upload a spreadsheet using the template" }
+    %>
+    <%= f.govuk_submit "Upload"  %>
+  <% end %>
+</div>

app/views/form/page.html.erb

@@ -10,7 +10,7 @@
           <%= page_info["header"] %>
         </h1>
       <% end %>
-      <%= form_with model: @case_log, method: "submit_form", builder: GOVUKDesignSystemFormBuilder::FormBuilder do |f| %>
+      <%= form_with model: @case_log, url: form_case_log_path(@case_log), method: "post", builder: GOVUKDesignSystemFormBuilder::FormBuilder do |f| %>
         <%= f.govuk_error_summary %>
         <% page_info["questions"].map do |question_key, question| %>
           <div id=<%= question_key + "_div " %><%= display_question_key_div(page_info, question_key) %> >

config/routes.rb

@@ -4,15 +4,24 @@ Rails.application.routes.draw do
   root to: "test#index"
   get "about", to: "about#index"
 
-  post "/case_logs/:id", to: "case_logs#submit_form"
-
   form_handler = FormHandler.instance
   form = form_handler.get_form("2021_2022")
+
   resources :case_logs do
+    collection do
+      post "/bulk_upload", to: "bulk_upload#bulk_upload"
+      get "/bulk_upload", to: "bulk_upload#show"
+    end
+
+    member do
+      post "/form", to: "case_logs#submit_form"
+    end
+
     form.all_pages.keys.map do |page|
       get page.to_s, to: "case_logs##{page}"
-      get "#{page}/soft_validations", to: "soft_validations#show"
+      get "#{page}/soft_validations", to: "soft_validations#show" if form.soft_validations_for_page(page)
     end
+
     form.all_subsections.keys.map do |subsection|
       get "#{subsection}/check_answers", to: "case_logs#check_answers"
     end

db/schema.rb

@@ -122,6 +122,8 @@ ActiveRecord::Schema.define(version: 2021_11_12_105348) do
     t.integer "rp_dontknow"
     t.datetime "discarded_at"
     t.string "tenancyother"
+    t.integer "override_net_income_validation"
+    t.string "net_income_known"
     t.string "gdpr_acceptance"
     t.string "gdpr_declined"
     t.string "property_owner_organisation"
@@ -133,8 +135,6 @@ ActiveRecord::Schema.define(version: 2021_11_12_105348) do
     t.string "needs_type"
     t.string "sale_completion_date"
     t.string "purchaser_code"
-    t.integer "override_net_income_validation"
-    t.string "net_income_known"
     t.integer "reason"
     t.string "propcode"
     t.integer "majorrepairs"

spec/requests/bulk_upload_controller_spec.rb

@@ -0,0 +1,60 @@
+require "rails_helper"
+
+RSpec.describe BulkUploadController, type: :request do
+  let(:url) { "/case_logs/bulk_upload" }
+
+  describe "GET #show" do
+    before do
+      get url, params: {}
+    end
+
+    it "returns a success response" do
+      expect(response).to be_successful
+    end
+
+    it "returns a page with a file upload form" do
+      expect(response.body).to match(/<input id="bulk-upload-case-log-bulk-upload-field" class="govuk-file-upload"/)
+      expect(response.body).to match(/<button type="submit" formnovalidate="formnovalidate" class="govuk-button"/)
+    end
+  end
+
+  describe "POST #bulk upload" do
+    subject { post url, params: { bulk_upload: { case_log_bulk_upload: @file } } }
+
+    context "given a valid file based on the upload template" do
+      before do
+        @file = fixture_file_upload("2021_22_lettings_bulk_upload.xlsx", "application/vnd.ms-excel")
+      end
+
+      it "creates case logs for each row in the template" do
+        expect { subject }.to change(CaseLog, :count).by(9)
+      end
+
+      it "redirects to the case log index page" do
+        expect(subject).to redirect_to(case_logs_path)
+      end
+    end
+
+    context "given an invalid file type" do
+      before do
+        @file = fixture_file_upload("random.txt", "text/plain")
+        subject
+      end
+
+      it "displays an error message" do
+        expect(response.body).to match(/Invalid file type/)
+      end
+    end
+
+    context "given an empty file" do
+      before do
+        @file = fixture_file_upload("2021_22_lettings_bulk_upload_empty.xlsx", "application/vnd.ms-excel")
+        subject
+      end
+
+      it "displays an error message" do
+        expect(response.body).to match(/No data found/)
+      end
+    end
+  end
+end