Add accessible autocomplete improvements

app/frontend/controllers/accessible_autocomplete_controller.js

@@ -1,12 +1,36 @@
-import { Controller } from '@hotwired/stimulus'
-import accessibleAutocomplete from 'accessible-autocomplete'
-import 'accessible-autocomplete/dist/accessible-autocomplete.min.css'
+import { Controller } from "@hotwired/stimulus";
+import accessibleAutocomplete from "accessible-autocomplete";
+import "accessible-autocomplete/dist/accessible-autocomplete.min.css";
+import { enhanceOption, suggestion, sort } from "../modules/search";
 
 export default class extends Controller {
   connect() {
+    const selectEl = this.element;
+    const selectOptions = Array.from(selectEl.options);
+    const options = selectOptions.map((o) => enhanceOption(o));
+
+    const matches = /^(\w+)\[(\w+)\]$/.exec(selectEl.name);
+    const rawFieldName = `${matches[1]}[${matches[2]}_raw]`;
+
     accessibleAutocomplete.enhanceSelectElement({
-      defaultValue: '',
-      selectElement: this.element
-    })
+      defaultValue: "",
+      selectElement: selectEl,
+      minLength: 2,
+      source: (query, populateResults) => {
+        if (/\S/.test(query)) {
+          populateResults(sort(query, options));
+        }
+      },
+      autoselect: true,
+      templates: { suggestion: (value) => suggestion(value, options) },
+      name: rawFieldName,
+      onConfirm: (val) => {
+        const selectedOption = [].filter.call(
+          selectOptions,
+          (option) => (option.textContent || option.innerText) === val
+        )[0];
+        if (selectedOption) selectedOption.selected = true;
+      },
+    });
   }
 }

app/frontend/modules/search.js

@@ -0,0 +1,140 @@
+const addWeightWithBoost = (option, query) => {
+  option.weight = calculateWeight(option.clean, query) * option.clean.boost;
+
+  return option;
+};
+
+const clean = (text) =>
+  text
+    .trim()
+    .replace(/['’]/g, "")
+    .replace(/[.,"/#!$%^&*;:{}=\-_`~()]/g, " ")
+    .toLowerCase();
+
+const cleanseOption = (option) => {
+  const synonyms = (option.synonyms || []).map(clean);
+
+  option.clean = {
+    name: clean(option.name),
+    nameWithoutStopWords: removeStopWords(option.name),
+    synonyms: synonyms,
+    synonymsWithoutStopWords: synonyms.map(removeStopWords),
+    boost: option.boost || 1,
+  };
+
+  return option;
+};
+
+const hasWeight = (option) => option.weight > 0;
+
+const byWeightThenAlphabetically = (a, b) => {
+  if (a.weight > b.weight) return -1;
+  if (a.weight < b.weight) return 1;
+  if (a.name < b.name) return -1;
+  if (a.name > b.name) return 1;
+
+  return 0;
+};
+
+const optionName = (option) => option.name;
+const exactMatch = (word, query) => word === query;
+
+const startsWithRegExp = (query) => new RegExp("\\b" + query, "i");
+const startsWith = (word, query) => word.search(startsWithRegExp(query)) === 0;
+
+const wordsStartsWithQuery = (word, regExps) =>
+  regExps.every((regExp) => word.search(regExp) >= 0);
+
+const anyMatch = (words, query, evaluatorFunc) =>
+  words.some((word) => evaluatorFunc(word, query));
+const synonymsExactMatch = (synonyms, query) =>
+  anyMatch(synonyms, query, exactMatch);
+const synonymsStartsWith = (synonyms, query) =>
+  anyMatch(synonyms, query, startsWith);
+
+const wordInSynonymStartsWithQuery = (synonyms, startsWithQueryWordsRegexes) =>
+  anyMatch(synonyms, startsWithQueryWordsRegexes, wordsStartsWithQuery);
+
+const calculateWeight = (
+  { name, synonyms, nameWithoutStopWords, synonymsWithoutStopWords },
+  query
+) => {
+  const queryWithoutStopWords = removeStopWords(query);
+
+  if (exactMatch(name, query)) return 100;
+  if (exactMatch(nameWithoutStopWords, queryWithoutStopWords)) return 95;
+
+  if (synonymsExactMatch(synonyms, query)) return 75;
+  if (synonymsExactMatch(synonymsWithoutStopWords, queryWithoutStopWords))
+    return 70;
+
+  if (startsWith(name, query)) return 60;
+  if (startsWith(nameWithoutStopWords, queryWithoutStopWords)) return 55;
+
+  if (synonymsStartsWith(synonyms, query)) return 50;
+  if (synonymsStartsWith(synonyms, queryWithoutStopWords)) return 40;
+
+  const startsWithRegExps = queryWithoutStopWords
+    .split(/\s+/)
+    .map(startsWithRegExp);
+
+  if (wordsStartsWithQuery(nameWithoutStopWords, startsWithRegExps)) return 25;
+  if (wordInSynonymStartsWithQuery(synonymsWithoutStopWords, startsWithRegExps))
+    return 10;
+
+  return 0;
+};
+
+const stopWords = ["the", "of", "in", "and", "at", "&"];
+
+const removeStopWords = (text) => {
+  const isAllStopWords = text
+    .trim()
+    .split(" ")
+    .every((word) => stopWords.includes(word));
+
+  if (isAllStopWords) {
+    return text;
+  }
+
+  const regex = new RegExp(
+    stopWords.map((word) => `(\\s+)?${word}(\\s+)?`).join("|"),
+    "gi"
+  );
+  return text.replace(regex, " ").trim();
+};
+
+export const sort = (query, options) => {
+  const cleanQuery = clean(query);
+
+  return options
+    .map(cleanseOption)
+    .map((option) => addWeightWithBoost(option, cleanQuery))
+    .filter(hasWeight)
+    .sort(byWeightThenAlphabetically)
+    .map(optionName);
+};
+
+export const suggestion = (value, options) => {
+  const option = options.find((o) => o.name === value);
+  if (option) {
+    const html = option.append
+      ? `<span>${value}</span> ${option.append}`
+      : `<span>${value}</span>`;
+    return option.hint ? `${html}<br>${option.hint}` : html;
+  } else {
+    return `<span>No results found</span>`;
+  }
+};
+
+export const enhanceOption = (option) => {
+  return {
+    name: option.label,
+    synonyms: option.getAttribute("data-synonyms")
+      ? option.getAttribute("data-synonyms").split("|")
+      : [],
+    append: option.getAttribute("data-append"),
+    hint: option.getAttribute("data-hint"),
+    boost: parseFloat(option.getAttribute("data-boost")) || 1,
+  };
+};

spec/features/form/accessible_autocomplete_spec.rb

@@ -21,14 +21,32 @@ RSpec.describe "Accessible Automcomplete" do
     sign_in user
   end
 
-  it "allows type ahead filtering", js: true do
+  context "when using accessible autocomplete" do
+    before do
       visit("/logs/#{case_log.id}/accessible-select")
+    end
+
+    it "allows type ahead filtering", js: true do
       find("#case-log-prevloc-field").click.native.send_keys("T", "h", "a", "n", :down, :enter)
       expect(find("#case-log-prevloc-field").value).to eq("Thanet")
     end
 
+    it "ignores punctuation", js: true do
+      find("#case-log-prevloc-field").click.native.send_keys("T", "h", "a", "'", "n", :down, :enter)
+      expect(find("#case-log-prevloc-field").value).to eq("Thanet")
+    end
+
+    it "ignores stop words", js: true do
+      find("#case-log-prevloc-field").click.native.send_keys("t", "h", "e", " ", "W", "e", "s", "t", "m", :down, :enter)
+      expect(find("#case-log-prevloc-field").value).to eq("Westminster")
+    end
+
+    it "does not perform an exact match", js: true do
+      find("#case-log-prevloc-field").click.native.send_keys("o", "n", "l", "y", " ", "t", "o", "w", "n", :down, :enter)
+      expect(find("#case-log-prevloc-field").value).to eq("The one and only york town")
+    end
+
     it "maintains enhancement state across back navigation", js: true do
-    visit("/logs/#{case_log.id}/accessible-select")
       find("#case-log-prevloc-field").click.native.send_keys("T", "h", "a", "n", :down, :enter)
       click_button("Save and continue")
       click_link(text: "Back")
@@ -36,9 +54,9 @@ RSpec.describe "Accessible Automcomplete" do
     end
 
     it "has a disabled null option" do
-    visit("/logs/#{case_log.id}/accessible-select")
       expect(page).to have_select("case-log-prevloc-field", disabled_options: ["Select an option"])
     end
+  end
 
   it "has the correct option selected if one has been saved" do
     case_log.update!(postcode_known: 0, previous_la_known: 1, prevloc: "E07000178")

spec/fixtures/forms/2021_2022.json

@@ -326,7 +326,7 @@
                       "E07000178": "Oxford",
                       "E07000114": "Thanet",
                       "E09000033": "Westminster",
-                      "E06000014": "York"
+                      "E06000014": "The one and only york town"
                     }
                   }
                 },

spec/models/form/question_spec.rb

@@ -158,7 +158,7 @@ RSpec.describe Form::Question, type: :model do
     end
 
     it "can map label from value" do
-      expect(question.label_from_value("E06000014")).to eq("York")
+      expect(question.label_from_value("E09000033")).to eq("Westminster")
     end
 
     context "when the saved answer is not in the value map" do