Merged in main

Co-authored-by: Milo <magicmilo@users.noreply.github.com>

Gemfile

@@ -29,6 +29,8 @@ gem "discard"
 gem "activeadmin"
 # Admin charts
 gem "chartkick"
+# Spreadsheet parsing
+gem "roo"
 # Json Schema
 gem "json-schema"
 gem "uk_postcode"

Gemfile.lock

@@ -302,6 +302,9 @@ GEM
       actionpack (>= 5.0)
       railties (>= 5.0)
     rexml (3.2.5)
+    roo (2.8.3)
+      nokogiri (~> 1)
+      rubyzip (>= 1.3.0, < 3.0.0)
     rubocop (1.21.0)
       parallel (~> 1.10)
       parser (>= 3.0.0.0)
@@ -420,6 +423,7 @@ DEPENDENCIES
   puma (~> 5.0)
   rack-mini-profiler (~> 2.0)
   rails (~> 6.1.4)
+  roo
   rspec-core!
   rspec-expectations!
   rspec-mocks!

README.md

@@ -163,12 +163,16 @@ Assumptions made by the format:
 ## JSON Form Validation against Schema
 
 To validate the form JSON against the schema you can run:
-`rake form_definition:validate`
+`rake form_definition:validate["config/forms/2021_2022.json"]`
 
-This will validate all forms in:
+n.b. You may have to escape square brackets in zsh 
-directories = ["config/forms", "spec/fixtures/forms"]
+`rake form_definition:validate\["config/forms/2021_2022.json"\]`
 
-against the schema in (config/forms/schema/generic.json)
+This will validate the given form definition against the schema in `config/forms/schema/generic.json`.
+
+You can also run:
+`rake form_definition:validate_all`
+This will validate all forms in directories = ["config/forms", "spec/fixtures/forms"]
 
 ## Useful documentation (external dependencies)
 

app/constants/db_enums.rb

@@ -166,11 +166,10 @@ module DbEnums
 
   def self.tenancy
     {
-      "Fixed term – Secure" => 1,
+      "Secure (including flexible)" => 1,
-      "Fixed term – Assured Shorthold Tenancy (AST)" => 4,
+      "Assured" => 2,
-      "Lifetime – Secure" => 100,
+      "Assured Shorthold" => 4,
-      "Lifetime – Assured" => 2,
+      "Licence agreement (almshouses only)" => 5,
-      "License agreement" => 5,
       "Other" => 3,
     }
   end

app/controllers/bulk_upload_controller.rb

@@ -0,0 +1,22 @@
+class BulkUploadController < ApplicationController
+  def show
+    @bulk_upload = BulkUpload.new(nil, nil)
+    render "case_logs/bulk_upload"
+  end
+
+  def bulk_upload
+    file = upload_params.tempfile
+    content_type = upload_params.content_type
+    @bulk_upload = BulkUpload.new(file, content_type)
+    @bulk_upload.process
+    if @bulk_upload.errors.present?
+      render "case_logs/bulk_upload", status: :unprocessable_entity
+    else
+      redirect_to(case_logs_path)
+    end
+  end
+
+  def upload_params
+    params.require("bulk_upload")["case_log_bulk_upload"]
+  end
+end

app/controllers/case_logs_controller.rb

@@ -109,6 +109,8 @@ private
         day = params["case_log"]["#{question_key}(3i)"]
         month = params["case_log"]["#{question_key}(2i)"]
         year = params["case_log"]["#{question_key}(1i)"]
+        next unless day.present? && month.present? && year.present?
+
         result[question_key] = Date.new(year.to_i, month.to_i, day.to_i)
       end
       next unless question_params

app/helpers/check_answers_helper.rb

@@ -16,7 +16,7 @@ module CheckAnswersHelper
 
     while page_name.to_s != "check_answers" && subsection_keys.include?(page_name)
       questions = form.questions_for_page(page_name)
-      applicable_questions = filter_conditional_questions(questions, case_log)
+      applicable_questions = form.filter_conditional_questions(questions, case_log)
       total_questions = total_questions.merge(applicable_questions)
 
       page_name = get_next_page_name(form, page_name, case_log)
@@ -25,19 +25,6 @@ module CheckAnswersHelper
     total_questions
   end
 
-  def filter_conditional_questions(questions, case_log)
-    applicable_questions = questions
-
-    questions.each do |k, question|
-      question.fetch("conditional_for", []).each do |conditional_question_key, condition|
-        if condition_not_met(case_log, k, question, condition)
-          applicable_questions = applicable_questions.reject { |z| z == conditional_question_key }
-        end
-      end
-    end
-    applicable_questions
-  end
-
   def get_next_page_name(form, page_name, case_log)
     page = form.all_pages[page_name]
     if page.key?("conditional_route_to")
@@ -50,21 +37,6 @@ module CheckAnswersHelper
     form.next_page(page_name)
   end
 
-  def condition_not_met(case_log, question_key, question, condition)
-    case question["type"]
-    when "numeric"
-      operator = condition[/[<>=]+/].to_sym
-      operand = condition[/\d+/].to_i
-      case_log[question_key].blank? || !case_log[question_key].send(operator, operand)
-    when "text"
-      case_log[question_key].blank? || !condition.include?(case_log[question_key])
-    when "radio"
-      case_log[question_key].blank? || !condition.include?(case_log[question_key])
-    else
-      raise "Not implemented yet"
-    end
-  end
-
   def create_update_answer_link(case_log_answer, case_log_id, page)
     link_name = case_log_answer.blank? ? "Answer" : "Change"
     link_to(link_name, "/case_logs/#{case_log_id}/#{page}", class: "govuk-link").html_safe

app/helpers/tasklist_helper.rb

@@ -13,31 +13,32 @@ module TasklistHelper
     in_progress: "govuk-tag--blue",
   }.freeze
 
-  def get_subsection_status(subsection_name, case_log, questions)
+  def get_subsection_status(subsection_name, case_log, form, questions)
+    applicable_questions = form.filter_conditional_questions(questions, case_log).keys
     if subsection_name == "declaration"
       return case_log.completed? ? :not_started : :cannot_start_yet
     end
 
-    return :not_started if questions.all? { |question| case_log[question].blank? }
+    return :not_started if applicable_questions.all? { |question| case_log[question].blank? }
-    return :completed if questions.all? { |question| case_log[question].present? }
+    return :completed if applicable_questions.all? { |question| case_log[question].present? }
 
     :in_progress
   end
 
   def get_next_incomplete_section(form, case_log)
     subsections = form.all_subsections.keys
-    subsections.find { |subsection| is_incomplete?(subsection, case_log, form.questions_for_subsection(subsection).keys) }
+    subsections.find { |subsection| is_incomplete?(subsection, case_log, form, form.questions_for_subsection(subsection)) }
   end
 
   def get_subsections_count(form, case_log, status = :all)
     subsections = form.all_subsections.keys
     return subsections.count if status == :all
 
-    subsections.count { |subsection| get_subsection_status(subsection, case_log, form.questions_for_subsection(subsection).keys) == status }
+    subsections.count { |subsection| get_subsection_status(subsection, case_log, form, form.questions_for_subsection(subsection)) == status }
   end
 
   def get_first_page_or_check_answers(subsection, case_log, form, questions)
-    path = if is_started?(subsection, case_log, questions)
+    path = if is_started?(subsection, case_log, form, questions)
              "case_log_#{subsection}_check_answers_path"
            else
              "case_log_#{form.first_page_for_subsection(subsection)}_path"
@@ -47,13 +48,13 @@ module TasklistHelper
 
 private
 
-  def is_incomplete?(subsection, case_log, questions)
+  def is_incomplete?(subsection, case_log, form, questions)
-    status = get_subsection_status(subsection, case_log, questions)
+    status = get_subsection_status(subsection, case_log, form, questions)
     %i[not_started in_progress].include?(status)
   end
 
-  def is_started?(subsection, case_log, questions)
+  def is_started?(subsection, case_log, form, questions)
-    status = get_subsection_status(subsection, case_log, questions)
+    status = get_subsection_status(subsection, case_log, form, questions)
     %i[in_progress completed].include?(status)
   end
 end

app/javascript/controllers/soft_validations_controller.js

@@ -5,7 +5,7 @@ export default class extends Controller {
 
   initialize() {
     let url = window.location.href + "/soft_validations"
-    this.fetch_retry(url, { headers: { accept: "application/json" } }, 2)
+    this.fetch_retry(url, { headers: { accept: "application/json" } }, 5)
   }
 
   fetch_retry(url, options, n) {

app/javascript/styles/_task-list.scss

@@ -81,11 +81,6 @@
   }
 }
 
-// turbo-frame {
-//   display: block;
-//   border: 1px solid blue
-// }
-
 .app-task-list__item:target,
 .tasklist_item_highlight{
   background-color: $govuk-focus-colour;

app/javascript/styles/application.scss

@@ -11,4 +11,11 @@ $govuk-image-url-function: frontend-image-url;
 
 @import "~govuk-frontend/govuk/all";
 
-@import '_task-list'
+@import '_task-list';
+
+$govuk-global-styles: true;
+
+// turbo-frame {
+//   display: block;
+//   border: 1px solid blue
+// }

app/models/bulk_upload.rb

@@ -0,0 +1,199 @@
+class BulkUpload
+  include ActiveModel::Model
+  include ActiveModel::Validations
+  include ActiveModel::Conversion
+
+  SPREADSHEET_CONTENT_TYPES = %w[
+    application/vnd.ms-excel
+    application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
+  ].freeze
+
+  FIRST_DATA_ROW = 7
+
+  def initialize(file, content_type)
+    @file = file
+    @content_type = content_type
+  end
+
+  def process
+    return unless valid_content_type?
+
+    xlsx = Roo::Spreadsheet.open(@file, extension: :xlsx)
+    sheet = xlsx.sheet(0)
+    last_row = sheet.last_row
+    if last_row < FIRST_DATA_ROW
+      errors.add(:case_log_bulk_upload, "No data found")
+    else
+      data_range = FIRST_DATA_ROW..last_row
+      data_range.map do |row_num|
+        case_log = CaseLog.create
+        map_row(sheet.row(row_num)).each do |attr_key, attr_val|
+          begin
+            case_log.update_attribute(attr_key, attr_val)
+          rescue ArgumentError
+          end
+        end
+      end
+    end
+  end
+
+  def valid_content_type?
+    if SPREADSHEET_CONTENT_TYPES.include?(@content_type)
+      true
+    else
+      errors.add(:case_log_bulk_upload, "Invalid file type")
+      false
+    end
+  end
+
+  def map_row(row)
+    {
+      lettype: row[1],
+      landlord: row[2],
+      # reg_num_la_core_code: row[3],
+      # managementgroup: row[4],
+      # schemecode: row[5],
+      # firstletting: row[6],
+      tenant_code: row[7],
+      startertenancy: row[8],
+      tenancy: row[9],
+      tenancyother: row[10],
+      # tenancyduration: row[11],
+      other_hhmemb: other_hhmemb(row),
+      hhmemb: other_hhmemb(row) + 1,
+      age1: row[12],
+      age2: row[13],
+      age3: row[14],
+      age4: row[15],
+      age5: row[16],
+      age6: row[17],
+      age7: row[18],
+      age8: row[19],
+      sex1: row[20],
+      sex2: row[21],
+      sex3: row[22],
+      sex4: row[23],
+      sex5: row[24],
+      sex6: row[25],
+      sex7: row[26],
+      sex8: row[27],
+      relat2: row[28],
+      relat3: row[29],
+      relat4: row[30],
+      relat5: row[31],
+      relat6: row[32],
+      relat7: row[33],
+      relat8: row[34],
+      ecstat1: row[35],
+      ecstat2: row[36],
+      ecstat3: row[37],
+      ecstat4: row[38],
+      ecstat5: row[39],
+      ecstat6: row[40],
+      ecstat7: row[41],
+      ecstat8: row[42],
+      ethnic: row[43],
+      national: row[44],
+      armed_forces: row[45],
+      reservist: row[46],
+      preg_occ: row[47],
+      hb: row[48],
+      benefits: row[49],
+      net_income_known: row[50].present? ? 1 : nil,
+      earnings: row[50],
+      # increfused: row[51],
+      reason: row[52],
+      other_reason_for_leaving_last_settled_home: row[53],
+      underoccupation_benefitcap: row[54],
+      housingneeds_a: row[55],
+      housingneeds_b: row[56],
+      housingneeds_c: row[57],
+      housingneeds_f: row[58],
+      housingneeds_g: row[59],
+      housingneeds_h: row[60],
+      prevten: row[61],
+      prevloc: row[62],
+      # ppostc1: row[63],
+      # ppostc2: row[64],
+      # prevpco_unknown: row[65],
+      layear: row[66],
+      lawaitlist: row[67],
+      homeless: row[68],
+      reasonpref: row[69],
+      rp_homeless: row[70],
+      rp_insan_unsat: row[71],
+      rp_medwel: row[72],
+      rp_hardship: row[73],
+      rp_dontknow: row[74],
+      cbl: row[75],
+      chr: row[76],
+      cap: row[77],
+      # referral_source: row[78],
+      period: row[79],
+      brent: row[80],
+      scharge: row[81],
+      pscharge: row[82],
+      supcharg: row[83],
+      tcharge: row[84],
+      # tcharge_care_homes: row[85],
+      # no_rent_or_charge: row[86],
+      hbrentshortfall: row[87],
+      tshortfall: row[88],
+      property_void_date: row[89].to_s + row[90].to_s + row[91].to_s,
+      # property_void_date_day: row[89],
+      # property_void_date_month: row[90],
+      # property_void_date_year: row[91],
+      majorrepairs: row[92].present? ? "1" : nil,
+      mrcdate: row[92].to_s + row[93].to_s + row[94].to_s,
+      mrcday: row[92],
+      mrcmonth: row[93],
+      mrcyear: row[94],
+      # supported_scheme: row[95],
+      startdate: row[96].to_s + row[97].to_s + row[98].to_s,
+      # startdate_day: row[96],
+      # startdate_month: row[97],
+      # startdate_year: row[98],
+      offered: row[99],
+      # property_reference: row[100],
+      beds: row[101],
+      unittype_gn: row[102],
+      property_building_type: row[103],
+      wchair: row[104],
+      property_relet: row[105],
+      rsnvac: row[106],
+      la: row[107],
+      # postcode: row[108],
+      # postcod2: row[109],
+      # row[110] removed
+      property_owner_organisation: row[111],
+      # username: row[112],
+      property_manager_organisation: row[113],
+      leftreg: row[114],
+      # uprn: row[115],
+      incfreq: row[116],
+      # sheltered_accom: row[117],
+      illness: row[118],
+      illness_type_1: row[119],
+      illness_type_2: row[120],
+      illness_type_3: row[121],
+      illness_type_4: row[122],
+      illness_type_8: row[123],
+      illness_type_5: row[124],
+      illness_type_6: row[125],
+      illness_type_7: row[126],
+      illness_type_9: row[127],
+      illness_type_10: row[128],
+      # london_affordable: row[129],
+      rent_type: row[130],
+      intermediate_rent_product_name: row[131],
+      # data_protection: row[132],
+      sale_or_letting: "letting",
+      gdpr_acceptance: 1,
+      gdpr_declined: 0,
+    }
+  end
+
+  def other_hhmemb(row)
+    [13, 14, 15, 16, 17, 18, 19].count { |idx| row[idx].present? }
+  end
+end

app/models/case_log.rb

@@ -10,7 +10,7 @@ class CaseLogValidator < ActiveModel::Validator
     # If we've come from the form UI we only want to validate the specific fields
     # that have just been submitted. If we're submitting a log via API or Bulk Upload
     # we want to validate all data fields.
-    page_to_validate = options[:page]
+    page_to_validate = record.page
     if page_to_validate
       public_send("validate_#{page_to_validate}", record) if respond_to?("validate_#{page_to_validate}")
     else
@@ -39,9 +39,8 @@ class CaseLog < ApplicationRecord
   include SoftValidations
   include DbEnums
   default_scope -> { kept }
-  scope :not_completed, -> { where.not(status: "completed") }
 
-  validates_with CaseLogValidator, ({ page: @page } || {})
+  validates_with CaseLogValidator
   before_save :update_status!
 
   attr_accessor :page
@@ -129,6 +128,8 @@ class CaseLog < ApplicationRecord
   end
 
   def weekly_net_income
+    return unless earnings && incfreq
+
     case incfreq
     when "Weekly"
       earnings
@@ -230,7 +231,7 @@ private
       dynamically_not_required << "incfreq"
     end
 
-    if tenancy == "Fixed term – Secure"
+    if tenancy == "Secure (including flexible)"
       dynamically_not_required << "tenancylength"
     end
 

app/models/form.rb

@@ -97,6 +97,36 @@ class Form
     }.reduce(:merge)
   end
 
+  def filter_conditional_questions(questions, case_log)
+    applicable_questions = questions
+
+    questions.each do |k, question|
+      question.fetch("conditional_for", []).each do |conditional_question_key, condition|
+        if condition_not_met(case_log, k, question, condition)
+          applicable_questions = applicable_questions.reject { |z| z == conditional_question_key }
+        end
+      end
+    end
+    applicable_questions
+  end
+
+  def condition_not_met(case_log, question_key, question, condition)
+    case question["type"]
+    when "numeric"
+      operator = condition[/[<>=]+/].to_sym
+      operand = condition[/\d+/].to_i
+      case_log[question_key].blank? || !case_log[question_key].send(operator, operand)
+    when "text"
+      case_log[question_key].blank? || !condition.include?(case_log[question_key])
+    when "radio"
+      case_log[question_key].blank? || !condition.include?(case_log[question_key])
+    when "select"
+      case_log[question_key].blank? || !condition.include?(case_log[question_key])
+    else
+      raise "Not implemented yet"
+    end
+  end
+
   def get_answer_label(case_log, question_title)
     question = all_questions[question_title]
     if question["type"] == "checkbox"

app/validations/household_validations.rb

@@ -45,7 +45,7 @@ module HouseholdValidations
     end
   end
 
-  def validate_household_pregnancy(record)
+  def validate_pregnancy(record)
     if (record.preg_occ == "Yes" || record.preg_occ == "Prefer not to say") && !women_of_child_bearing_age_in_household(record)
       record.errors.add :preg_occ, "You must answer no as there are no female tenants aged 16-50 in the property"
     end

app/validations/tenancy_validations.rb

@@ -4,12 +4,12 @@ module TenancyValidations
   def validate_fixed_term_tenancy(record)
     is_present = record.tenancylength.present?
     is_in_range = record.tenancylength.to_i.between?(2, 99)
-    is_secure = record.tenancy == "Fixed term – Secure"
+    is_secure = record.tenancy == "Secure (including flexible)"
-    is_ast = record.tenancy == "Fixed term – Assured Shorthold Tenancy (AST)"
+    is_ast = record.tenancy == "Assured Shorthold"
     conditions = [
       { condition: !(is_secure || is_ast) && is_present, error: "You must only answer the fixed term tenancy length question if the tenancy type is fixed term" },
       { condition: is_ast && !is_in_range,  error: "Fixed term – Assured Shorthold Tenancy (AST) should be between 2 and 99 years" },
-      { condition: is_secure && (!is_in_range && is_present), error: "Fixed term – Secure should be between 2 and 99 years or not specified" },
+      { condition: is_secure && (!is_in_range && is_present), error: "Secure (including flexible) should be between 2 and 99 years or not specified" },
     ]
 
     conditions.each { |condition| condition[:condition] ? (record.errors.add :tenancylength, condition[:error]) : nil }

app/views/case_logs/_log_list.html.erb

@@ -12,7 +12,7 @@
     <% case_logs.map do |log| %>
     <tr class="govuk-table__row">
       <th scope="row" class="govuk-table__header">
-        <%= link_to log.id, case_log_path(log) %>
+        <%= link_to log.id, case_log_path(log), class: "govuk-link" %>
       </th>
       <td class="govuk-table__cell govuk-table__cell">
         <%= log.property_postcode %>

app/views/case_logs/_tasklist.html.erb

@@ -9,10 +9,10 @@
       <ul class="app-task-list__items">
         <% section_value["subsections"].map do |subsection_key, subsection_value| %>
           <li class="app-task-list__item" id=<%= subsection_key %>>
-            <% questions_for_subsection = @form.questions_for_subsection(subsection_key).keys %>
+            <% questions_for_subsection = @form.questions_for_subsection(subsection_key) %>
             <% next_page_path = get_first_page_or_check_answers(subsection_key, @case_log, @form, questions_for_subsection) %>
             <%= link_to subsection_value["label"], next_page_path, class: "task-name govuk-link" %>
-            <% subsection_status = get_subsection_status(subsection_key, @case_log, questions_for_subsection) %>
+            <% subsection_status = get_subsection_status(subsection_key, @case_log, @form, questions_for_subsection) %>
             <strong class="govuk-tag app-task-list__tag <%= TasklistHelper::STYLES[subsection_status] %>">
               <%= TasklistHelper::STATUSES[subsection_status] %>
             </strong>

app/views/case_logs/bulk_upload.html.erb

@@ -0,0 +1,10 @@
+<div class="govuk-form-group">
+  <%= form_for @bulk_upload, url: bulk_upload_case_logs_path, method: "post", builder: GOVUKDesignSystemFormBuilder::FormBuilder do |f| %>
+    <%= f.govuk_error_summary %>
+    <%= f.govuk_file_field :case_log_bulk_upload,
+        label: { text: "Bulk Upload", size: "l" },
+        hint: { text: "Upload a spreadsheet using the template" }
+    %>
+    <%= f.govuk_submit "Upload"  %>
+  <% end %>
+</div>

app/views/case_logs/edit.html.erb

@@ -5,11 +5,11 @@
         <%= @case_log.id %></h1>
 
       <h2 class="govuk-heading-s govuk-!-margin-bottom-2">This submission is
-        <%= @case_log.status %></h2>
+        <%= @case_log.status.to_s.humanize.downcase %></h2>
       <p class="govuk-body govuk-!-margin-bottom-7">You've completed <%= get_subsections_count(@form, @case_log, :completed) %> of <%= get_subsections_count(@form, @case_log, :all) %> sections.</p>
       <p class="govuk-body govuk-!-margin-bottom-7">
         <% next_incomplete_section=get_next_incomplete_section(@form, @case_log) %>
-        <a href="#<%= next_incomplete_section %>"
+        <a class="govuk-link" href="#<%= next_incomplete_section %>"
            data-controller="tasklist"
            data-action="tasklist#addHighlight"
            data-info=<%= next_incomplete_section %>>

app/views/form/check_answers.html.erb

@@ -10,7 +10,7 @@
           <%end %>
         <%end %>
       <% end %>
-      <%= form_with action: '/case_logs', method: "next_page", builder: GOVUKDesignSystemFormBuilder::FormBuilder do |f| %>
+      <%= form_with model: @case_log, method: "get", builder: GOVUKDesignSystemFormBuilder::FormBuilder do |f| %>
         <%= f.govuk_submit "Save and continue"  %>
       <% end %>
     </div>

app/views/form/page.html.erb

@@ -3,13 +3,14 @@
 <% end %>
 
 <%= turbo_frame_tag "case_log_form", target: "_top" do %>
-
+  <div class="govuk-grid-row">
+    <div class="govuk-grid-column-two-thirds">
       <% if page_info["header"].present? %>
         <h1 class="govuk-heading-xl">
           <%= page_info["header"] %>
         </h1>
       <% end %>
-  <%= form_with model: @case_log, method: "submit_form", builder: GOVUKDesignSystemFormBuilder::FormBuilder do |f| %>
+      <%= form_with model: @case_log, url: form_case_log_path(@case_log), method: "post", builder: GOVUKDesignSystemFormBuilder::FormBuilder do |f| %>
         <%= f.govuk_error_summary %>
         <% page_info["questions"].map do |question_key, question| %>
           <div id=<%= question_key + "_div " %><%= display_question_key_div(page_info, question_key) %> >
@@ -24,4 +25,6 @@
         <%= f.hidden_field :page, value: page_key %>
         <%= f.govuk_submit "Save and continue"  %>
       <% end %>
+    </div>
+  </div>
 <% end %>

app/views/layouts/_footer.html.erb

@@ -0,0 +1,48 @@
+<div class="govuk-width-container ">
+
+  <div class="govuk-footer__meta">
+    <div class="govuk-footer__meta-item govuk-footer__meta-item--grow">
+
+      <h2 class="govuk-visually-hidden">Support links</h2>
+
+      <div class="govuk-footer__meta-custom">
+
+        <h2 class="govuk-heading-m">Get help with this service</h2>
+        <h3 class="govuk-heading-s govuk-!-margin-bottom-1">Online helpdesk</h3>
+        <p class="govuk-body govuk-!-font-size-16">
+          <a class="govuk-link govuk-footer__link" href="https://digital.dclg.gov.uk/jira/servicedesk/customer/portal/4/group/21" target="_blank">CORE helpdesk</a>
+          (opens in a new tab)</p>
+
+        <h3 class="govuk-heading-s govuk-!-margin-bottom-1">Telephone</h3>
+        <ul class="govuk-list govuk-!-font-size-16">
+          <li>Telephone: 0333 202 5084</li>
+          <li>Monday to Friday, 9am to 5:30pm (except public holidays)</li>
+        </ul>
+
+        <h3 class="govuk-heading-s govuk-!-margin-bottom-1">Email</h3>
+        <ul class="govuk-list govuk-!-font-size-16">
+          <li>
+            <a class="govuk-link govuk-footer__link" href="mailto:mhclg.digital-services@communities.gov.uk?subject=CORE">dluhc.digital-services@communities.gov.uk</a>
+          </li>
+          <li>We aim to respond within 2 working days</li>
+        </ul>
+
+        <h2 class="govuk-visually-hidden">Footer links</h2>
+
+      </div>
+
+      <svg aria-hidden="true" focusable="false" class="govuk-footer__licence-logo" xmlns="http://www.w3.org/2000/svg" viewbox="0 0 483.2 195.7" height="17" width="41">
+        <path
+          fill="currentColor"
+          d="M421.5 142.8V.1l-50.7 32.3v161.1h112.4v-50.7zm-122.3-9.6A47.12 47.12 0 0 1 221 97.8c0-26 21.1-47.1 47.1-47.1 16.7 0 31.4 8.7 39.7 21.8l42.7-27.2A97.63 97.63 0 0 0 268.1 0c-36.5 0-68.3 20.1-85.1 49.7A98 98 0 0 0 97.8 0C43.9 0 0 43.9 0 97.8s43.9 97.8 97.8 97.8c36.5 0 68.3-20.1 85.1-49.7a97.76 97.76 0 0 0 149.6 25.4l19.4 22.2h3v-87.8h-80l24.3 27.5zM97.8 145c-26 0-47.1-21.1-47.1-47.1s21.1-47.1 47.1-47.1 47.2 21 47.2 47S123.8 145 97.8 145"/>
+      </svg>
+      <span class="govuk-footer__licence-description">
+        All content is available under the
+        <a class="govuk-footer__link" href="https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/" rel="license">Open Government Licence v3.0</a>, except where otherwise stated
+      </span>
+    </div>
+    <div class="govuk-footer__meta-item">
+      <a class="govuk-footer__link govuk-footer__copyright-logo" href="https://www.nationalarchives.gov.uk/information-management/re-using-public-sector-information/uk-government-licensing-framework/crown-copyright/">© Crown copyright</a>
+    </div>
+  </div>
+</div>

app/views/layouts/application.html.erb

@@ -74,34 +74,7 @@
     </div>
 
     <footer class="govuk-footer">
-      <div class="govuk-width-container ">
+      <%= render partial: "layouts/footer" %>
-        <div class="govuk-footer__meta">
-          <div class="govuk-footer__meta-item govuk-footer__meta-item--grow">
-            <h2 class="govuk-visually-hidden">Support links</h2>
-
-            <div class="govuk-footer__meta-custom">
-              <h2 class="govuk-heading-m">Support and guidance</h2>
-
-              <p class="govuk-body-s">
-                If you have a question, or you've had a problem using this service, please contact us at <%= mail_to "test@mhclg.gov.uk", "test@mhclg.gov.uk", class: "govuk-footer__link" %>
-              </p>
-
-            </div>
-
-            <svg aria-hidden="true" focusable="false" class="govuk-footer__licence-logo" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 483.2 195.7" height="17" width="41">
-              <path fill="currentColor" d="M421.5 142.8V.1l-50.7 32.3v161.1h112.4v-50.7zm-122.3-9.6A47.12 47.12 0 0 1 221 97.8c0-26 21.1-47.1 47.1-47.1 16.7 0 31.4 8.7 39.7 21.8l42.7-27.2A97.63 97.63 0 0 0 268.1 0c-36.5 0-68.3 20.1-85.1 49.7A98 98 0 0 0 97.8 0C43.9 0 0 43.9 0 97.8s43.9 97.8 97.8 97.8c36.5 0 68.3-20.1 85.1-49.7a97.76 97.76 0 0 0 149.6 25.4l19.4 22.2h3v-87.8h-80l24.3 27.5zM97.8 145c-26 0-47.1-21.1-47.1-47.1s21.1-47.1 47.1-47.1 47.2 21 47.2 47S123.8 145 97.8 145" />
-            </svg>
-
-            <span class="govuk-footer__licence-description">
-              All content is available under the
-              <a class="govuk-footer__link" href="https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/" rel="license">Open Government Licence v3.0</a>, except where otherwise stated
-            </span>
-          </div>
-          <div class="govuk-footer__meta-item">
-            <a class="govuk-footer__link govuk-footer__copyright-logo" href="https://www.nationalarchives.gov.uk/information-management/re-using-public-sector-information/uk-government-licensing-framework/crown-copyright/">© Crown copyright</a>
-          </div>
-        </div>
-      </div>
     </footer>
   </body>
 </html>

config/forms/2021_2022.json

@@ -80,9 +80,8 @@
                 }
               },
               "conditional_route_to": {
-                "tenant_same_property_renewal": { "sale_or_letting": "Letting" }
+                "sale_completion_date": { "sale_or_letting": "Sale" }
-              },
+              }
-              "default_next_page" : "check_answers"
             },
             "tenant_same_property_renewal": {
               "header": "About this log",
@@ -100,7 +99,7 @@
                 }
               }
             },
-            "tenancy_start_date": {
+            "startdate": {
               "header": "About this log",
               "description": "",
               "questions": {
@@ -112,8 +111,8 @@
                 }
               }
             },
-            "rent_type": {
+            "about_this_letting": {
-              "header": "About this log",
+              "header": "Tell us about this letting",
               "description": "",
               "questions": {
                 "rent_type": {
@@ -150,12 +149,25 @@
                 }
               }
             },
+            "tenant_code": {
+              "header": "",
+              "description": "",
+              "questions": {
+                "tenant_code": {
+                  "check_answer_label": "Tenant code",
+                  "header": "What is the tenant code?",
+                  "hint_text": "",
+                  "type": "text"
+                }
+              },
+              "default_next_page": "check_answers"
+            },
             "sale_completion_date": {
-              "header": "About this log",
+              "header": "Sale Completion Date",
               "description": "",
               "questions": {
                 "sale_completion_date": {
-                  "check_answer_label": "What is the sale completion date?",
+                  "check_answer_label": "Sale completion date",
                   "header": "What is the sale completion date?",
                   "hint_text": "For example, 27 3 2007",
                   "type": "date"
@@ -172,7 +184,8 @@
                   "hint_text": "",
                   "type": "text"
                 }
-              }
+              },
+              "default_next_page": "check_answers"
             }
           }
         }
@@ -196,7 +209,7 @@
                 }
               }
             },
-            "age1": {
+            "person_1_age": {
               "header": "",
               "description": "",
               "questions": {
@@ -294,7 +307,7 @@
                 }
               }
             },
-            "tenant_economic_status": {
+            "person_1_economic": {
               "header": "",
               "description": "",
               "questions": {
@@ -1068,12 +1081,11 @@
                   "hint_text": "",
                   "type": "radio",
                   "answer_options": {
-                    "0": "Fixed term – Secure",
+                    "0": "Secure (including flexible)",
-                    "1": "Fixed term – Assured Shorthold Tenancy (AST)",
+                    "1": "Assured",
-                    "2": "Lifetime – Secure",
+                    "2": "Assured Shorthold",
-                    "3": "Lifetime – Assured",
+                    "3": "Licence agreement (almshouses only)",
-                    "4": "License agreement",
+                    "4": "Other"
-                    "5": "Other"
                   },
                   "conditional_for": {
                     "other_tenancy_type": ["Other"]

config/routes.rb

@@ -9,15 +9,24 @@ Rails.application.routes.draw do
   root to: "test#index"
   get "about", to: "about#index"
 
-  post "/case_logs/:id", to: "case_logs#submit_form"
-
   form_handler = FormHandler.instance
   form = form_handler.get_form("2021_2022")
+
   resources :case_logs do
+    collection do
+      post "/bulk_upload", to: "bulk_upload#bulk_upload"
+      get "/bulk_upload", to: "bulk_upload#show"
+    end
+
+    member do
+      post "/form", to: "case_logs#submit_form"
+    end
+
     form.all_pages.keys.map do |page|
       get page.to_s, to: "case_logs##{page}"
-      get "#{page}/soft_validations", to: "soft_validations#show"
+      get "#{page}/soft_validations", to: "soft_validations#show" if form.soft_validations_for_page(page)
     end
+
     form.all_subsections.keys.map do |subsection|
       get "#{subsection}/check_answers", to: "case_logs#check_answers"
     end

db/migrate/20211116102527_change_datetime.rb

@@ -0,0 +1,10 @@
+class ChangeDatetime < ActiveRecord::Migration[6.1]
+  def change
+    change_table :case_logs, bulk: true do |t|
+      t.remove :sale_completion_date
+      t.column :sale_completion_date, :datetime
+      t.remove :startdate
+      t.column :startdate, :datetime
+    end
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2021_11_12_105348) do
+ActiveRecord::Schema.define(version: 2021_11_16_102527) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -66,7 +66,6 @@ ActiveRecord::Schema.define(version: 2021_11_12_105348) do
     t.string "accessibility_requirements"
     t.string "condition_effects"
     t.string "tenancy_code"
-    t.string "startdate"
     t.integer "startertenancy"
     t.integer "tenancylength"
     t.integer "tenancy"
@@ -131,7 +130,6 @@ ActiveRecord::Schema.define(version: 2021_11_12_105348) do
     t.string "rent_type"
     t.string "intermediate_rent_product_name"
     t.string "needs_type"
-    t.string "sale_completion_date"
     t.string "purchaser_code"
     t.integer "override_net_income_validation"
     t.string "net_income_known"
@@ -154,6 +152,8 @@ ActiveRecord::Schema.define(version: 2021_11_12_105348) do
     t.integer "mrcyear"
     t.integer "other_hhmemb"
     t.integer "incref"
+    t.datetime "sale_completion_date"
+    t.datetime "startdate"
     t.index ["discarded_at"], name: "index_case_logs_on_discarded_at"
   end
 

docs/api/DLUHC-CORE-Data.v1.json

@@ -306,7 +306,7 @@
             "startdate": "12/03/2019",
             "startertenancy": "No",
             "tenancylength": "No",
-            "tenancy": "Fixed term – Secure",
+            "tenancy": "Secure (including flexible)",
             "lettype": "Affordable Rent - General Needs",
             "landlord": "This landlord",
             "la": "Barnet",

lib/tasks/form_definition.rake

@@ -12,40 +12,46 @@ def get_all_form_paths(directories)
 end
 
 namespace :form_definition do
-  desc "Validate JSON against Generic Form Schema"
+  desc "Validate all JSON against Generic Form Schema"
-  task validate: :environment do
-    puts Rails.root.to_s
-    path = "config/forms/schema/generic.json"
 
+  task validate_all: :environment do
+    directories = ["config/forms", "spec/fixtures/forms"]
+    paths = get_all_form_paths(directories)
+
+    paths.each do |path|
+      Rake::Task["form_definition:validate"].reenable
+      Rake::Task["form_definition:validate"].invoke(path)
+    end
+  end
+
+  desc "Validate Single JSON against Generic Form Schema"
+
+  task :validate, %i[path] => :environment do |_task, args|
+    path = Rails.root.join("config/forms/schema/generic.json")
     file = File.open(path)
     schema = JSON.parse(file.read)
-    metaschema = JSON::Validator.validator_for_name("draft4").metaschema
+    meta_schema = JSON::Validator.validator_for_name("draft4").metaschema
 
-    puts path
+    puts path unless Rails.env.test?
 
-    if JSON::Validator.validate(metaschema, schema)
+    if JSON::Validator.validate(meta_schema, schema)
-      puts "schema valid"
+      puts "Schema Definition is Valid" unless Rails.env.test?
     else
-      puts "schema not valid"
+      puts "Schema Definition in #{path} is not valid against draft4 json schema." unless Rails.env.test?
-      return
+      next
     end
 
-    directories = ["config/forms", "spec/fixtures/forms"]
+    path = Rails.root.join(args.path)
-    # directories = ["config/forms"]
-
-    get_all_form_paths(directories).each do |path|
-      puts path
     file = File.open(path)
-      data = JSON.parse(file.read)
+    form_definition = JSON.parse(file.read)
 
-      puts JSON::Validator.fully_validate(schema, data, strict: true)
+    puts path unless Rails.env.test?
+    puts JSON::Validator.fully_validate(schema, form_definition, strict: true) unless Rails.env.test?
 
     begin
-        JSON::Validator.validate!(schema, data)
+      JSON::Validator.validate!(schema, form_definition)
     rescue JSON::Schema::ValidationError => e
       e.message
     end
   end
 end
-end
-# rubocop:enable Lint/ShadowingOuterLocalVariable

spec/controllers/case_logs_controller_spec.rb

@@ -127,7 +127,7 @@ RSpec.describe CaseLogsController, type: :controller do
 
     context "conditional routing" do
       before do
-        allow_any_instance_of(CaseLogValidator).to receive(:validate_household_pregnancy).and_return(true)
+        allow_any_instance_of(CaseLogValidator).to receive(:validate_pregnancy).and_return(true)
       end
 
       let(:case_log_form_conditional_question_yes_params) do

spec/factories/case_log.rb

@@ -19,6 +19,15 @@ FactoryBot.define do
       earnings { 750 }
       incfreq { "Weekly" }
     end
+    trait :conditional_section_complete do
+      tenant_code { "TH356" }
+      age1 { 34 }
+      sex1 { "M" }
+      ethnic { 2 }
+      national { 4 }
+      ecstat1 { 2 }
+      other_hhmemb { 0 }
+    end
     created_at { Time.zone.now }
     updated_at { Time.zone.now }
   end

spec/features/case_log_spec.rb

@@ -47,7 +47,7 @@ RSpec.describe "Form Features" do
       it "displays a tasklist header" do
         visit("/case_logs/#{id}")
         expect(page).to have_content("Tasklist for log #{id}")
-        expect(page).to have_content("This submission is #{status}")
+        expect(page).to have_content("This submission is #{status.humanize.downcase}")
       end
 
       it "displays a section status" do
@@ -453,7 +453,7 @@ RSpec.describe "Form Features" do
 
   describe "conditional page routing", js: true do
     before do
-      allow_any_instance_of(CaseLogValidator).to receive(:validate_household_pregnancy).and_return(true)
+      allow_any_instance_of(CaseLogValidator).to receive(:validate_pregnancy).and_return(true)
     end
 
     it "can route the user to a different page based on their answer on the current page" do

spec/fixtures/complete_case_log.json

@@ -52,7 +52,7 @@
     "startdate": "12/03/2019",
     "startertenancy": "No",
     "tenancylength": "5",
-    "tenancy": "Fixed term – Secure",
+    "tenancy": "Secure (including flexible)",
     "lettype": "Affordable Rent - General Needs",
     "landlord": "This landlord",
     "la": "Barnet",
@@ -129,10 +129,9 @@
     "rent_type":  "",
     "intermediate_rent_product_name": "",
     "needs_type":  "",
-    "sale_completion_date": "",
+    "sale_completion_date": "01/01/2020",
     "purchaser_code":  "",
     "propcode": "123",
-    "majorrepairs": "Yes",
     "postcode": "a1",
     "postcod2": "w3",
     "ppostc1": "w3",

spec/helpers/tasklist_helper_spec.rb

@@ -6,26 +6,27 @@ RSpec.describe TasklistHelper do
   let(:completed_case_log) { FactoryBot.build(:case_log, :completed) }
   form_handler = FormHandler.instance
   let(:form) { form_handler.get_form("test_form") }
+  let(:household_characteristics_questions) { form.questions_for_subsection("household_characteristics") }
 
   describe "get subsection status" do
     let(:section) { "income_and_benefits" }
-    let(:income_and_benefits_questions) { form.questions_for_subsection("income_and_benefits").keys }
+    let(:income_and_benefits_questions) { form.questions_for_subsection("income_and_benefits") }
-    let(:declaration_questions) { form.questions_for_subsection("declaration").keys }
+    let(:declaration_questions) { form.questions_for_subsection("declaration") }
-    let(:local_authority_questions) { form.questions_for_subsection("local_authority").keys }
+    let(:local_authority_questions) { form.questions_for_subsection("local_authority") }
 
     it "returns not started if none of the questions in the subsection are answered" do
-      status = get_subsection_status("income_and_benefits", case_log, income_and_benefits_questions)
+      status = get_subsection_status("income_and_benefits", case_log, form, income_and_benefits_questions)
       expect(status).to eq(:not_started)
     end
 
     it "returns cannot start yet if the subsection is declaration" do
-      status = get_subsection_status("declaration", case_log, declaration_questions)
+      status = get_subsection_status("declaration", case_log, form, declaration_questions)
       expect(status).to eq(:cannot_start_yet)
     end
 
     it "returns in progress if some of the questions have been answered" do
       case_log["previous_postcode"] = "P0 5TT"
-      status = get_subsection_status("local_authority", case_log, local_authority_questions)
+      status = get_subsection_status("local_authority", case_log, form, local_authority_questions)
       expect(status).to eq(:in_progress)
     end
 
@@ -35,14 +36,22 @@ RSpec.describe TasklistHelper do
       case_log["benefits"] = "All"
       case_log["hb"] = "Do not know"
 
-      status = get_subsection_status("income_and_benefits", case_log, income_and_benefits_questions)
+      status = get_subsection_status("income_and_benefits", case_log, form, income_and_benefits_questions)
       expect(status).to eq(:completed)
     end
 
     it "returns not started if the subsection is declaration and all the questions are completed" do
-      status = get_subsection_status("declaration", completed_case_log, declaration_questions)
+      status = get_subsection_status("declaration", completed_case_log, form, declaration_questions)
       expect(status).to eq(:not_started)
     end
+
+    let(:conditional_section_complete_case_log) { FactoryBot.build(:case_log, :conditional_section_complete) }
+    it "sets the correct status for sections with conditional questions" do
+      status = get_subsection_status(
+        "household_characteristics", conditional_section_complete_case_log, form, household_characteristics_questions
+      )
+      expect(status).to eq(:completed)
+    end
   end
 
   describe "get next incomplete section" do
@@ -79,8 +88,6 @@ RSpec.describe TasklistHelper do
   end
 
   describe "get_first_page_or_check_answers" do
-    let(:household_characteristics_questions) { form.questions_for_subsection("household_characteristics").keys }
-
     it "returns the check answers page path if the section has been started already" do
       expect(get_first_page_or_check_answers("household_characteristics", case_log, form, household_characteristics_questions)).to match(/check_answers/)
     end

spec/lib/tasks/form_definition_validator_spec.rb

@@ -0,0 +1,33 @@
+require "rails_helper"
+require "rake"
+
+describe "rake form_definition:validate_all", type: :task do
+  subject(:task) { Rake::Task["form_definition:validate_all"] }
+
+  before do
+    Rake.application.rake_require("tasks/form_definition")
+    Rake::Task.define_task(:environment)
+    task.reenable
+  end
+
+  it "runs the validate task for each form definition in the project" do
+    expect(Rake::Task["form_definition:validate"]).to receive(:invoke).exactly(4).times
+    task.invoke
+  end
+end
+
+describe "rake form_definition:validate", type: :task do
+  subject(:task) { Rake::Task["form_definition:validate"] }
+
+  before do
+    Rake.application.rake_require("tasks/form_definition")
+    Rake::Task.define_task(:environment)
+    allow(JSON::Validator).to receive(:validate).and_return(true)
+    task.reenable
+  end
+
+  it "runs the validate task for the given form definition" do
+    expect(JSON::Validator).to receive(:validate!).at_least(1).time
+    task.invoke("config/forms/2021_2022.json")
+  end
+end

spec/models/case_log_spec.rb

@@ -182,39 +182,39 @@ RSpec.describe Form, type: :model do
 
       it "Must be completed and between 2 and 99 if type of tenancy is Assured shorthold" do
         expect {
-          CaseLog.create!(tenancy: "Fixed term – Assured Shorthold Tenancy (AST)",
+          CaseLog.create!(tenancy: "Assured Shorthold",
                           tenancylength: 1)
         }.to raise_error(ActiveRecord::RecordInvalid)
 
         expect {
-          CaseLog.create!(tenancy: "Fixed term – Assured Shorthold Tenancy (AST)",
+          CaseLog.create!(tenancy: "Assured Shorthold",
                           tenancylength: nil)
         }.to raise_error(ActiveRecord::RecordInvalid)
 
         expect {
-          CaseLog.create!(tenancy: "Fixed term – Assured Shorthold Tenancy (AST)",
+          CaseLog.create!(tenancy: "Assured Shorthold",
                           tenancylength: 2)
         }.not_to raise_error
       end
 
       it "Must be empty or between 2 and 99 if type of tenancy is Secure" do
         expect {
-          CaseLog.create!(tenancy: "Fixed term – Secure",
+          CaseLog.create!(tenancy: "Secure (including flexible)",
                           tenancylength: 1)
         }.to raise_error(ActiveRecord::RecordInvalid)
 
         expect {
-          CaseLog.create!(tenancy: "Fixed term – Secure",
+          CaseLog.create!(tenancy: "Secure (including flexible)",
                           tenancylength: 100)
         }.to raise_error(ActiveRecord::RecordInvalid)
 
         expect {
-          CaseLog.create!(tenancy: "Fixed term – Secure",
+          CaseLog.create!(tenancy: "Secure (including flexible)",
                           tenancylength: nil)
         }.not_to raise_error
 
         expect {
-          CaseLog.create!(tenancy: "Fixed term – Secure",
+          CaseLog.create!(tenancy: "Secure (including flexible)",
                           tenancylength: 2)
         }.not_to raise_error
       end
@@ -294,12 +294,12 @@ RSpec.describe Form, type: :model do
 
       it "must not be provided if tenancy type is not other" do
         expect {
-          CaseLog.create!(tenancy: "Fixed term – Secure",
+          CaseLog.create!(tenancy: "Secure (including flexible)",
                           tenancyother: "the other reason provided")
         }.to raise_error(ActiveRecord::RecordInvalid)
 
         expect {
-          CaseLog.create!(tenancy: "Fixed term – Secure",
+          CaseLog.create!(tenancy: "Secure (including flexible)",
                           tenancyother: nil)
         }.not_to raise_error
       end

spec/requests/bulk_upload_controller_spec.rb

@@ -0,0 +1,60 @@
+require "rails_helper"
+
+RSpec.describe BulkUploadController, type: :request do
+  let(:url) { "/case_logs/bulk_upload" }
+
+  describe "GET #show" do
+    before do
+      get url, params: {}
+    end
+
+    it "returns a success response" do
+      expect(response).to be_successful
+    end
+
+    it "returns a page with a file upload form" do
+      expect(response.body).to match(/<input id="bulk-upload-case-log-bulk-upload-field" class="govuk-file-upload"/)
+      expect(response.body).to match(/<button type="submit" formnovalidate="formnovalidate" class="govuk-button"/)
+    end
+  end
+
+  describe "POST #bulk upload" do
+    subject { post url, params: { bulk_upload: { case_log_bulk_upload: @file } } }
+
+    context "given a valid file based on the upload template" do
+      before do
+        @file = fixture_file_upload("2021_22_lettings_bulk_upload.xlsx", "application/vnd.ms-excel")
+      end
+
+      it "creates case logs for each row in the template" do
+        expect { subject }.to change(CaseLog, :count).by(9)
+      end
+
+      it "redirects to the case log index page" do
+        expect(subject).to redirect_to(case_logs_path)
+      end
+    end
+
+    context "given an invalid file type" do
+      before do
+        @file = fixture_file_upload("random.txt", "text/plain")
+        subject
+      end
+
+      it "displays an error message" do
+        expect(response.body).to match(/Invalid file type/)
+      end
+    end
+
+    context "given an empty file" do
+      before do
+        @file = fixture_file_upload("2021_22_lettings_bulk_upload_empty.xlsx", "application/vnd.ms-excel")
+        subject
+      end
+
+      it "displays an error message" do
+        expect(response.body).to match(/No data found/)
+      end
+    end
+  end
+end