Paul Robert Lloyd
2 years ago
committed by
GitHub
44 changed files with 714 additions and 289 deletions
@ -0,0 +1,9 @@
|
||||
source "https://rubygems.org" |
||||
|
||||
group :jekyll_plugins do |
||||
gem "github-pages" |
||||
end |
||||
|
||||
group :development do |
||||
gem "webrick" |
||||
end |
||||
@ -0,0 +1,263 @@
|
||||
GEM |
||||
remote: https://rubygems.org/ |
||||
specs: |
||||
activesupport (6.0.5) |
||||
concurrent-ruby (~> 1.0, >= 1.0.2) |
||||
i18n (>= 0.7, < 2) |
||||
minitest (~> 5.1) |
||||
tzinfo (~> 1.1) |
||||
zeitwerk (~> 2.2, >= 2.2.2) |
||||
addressable (2.8.0) |
||||
public_suffix (>= 2.0.2, < 5.0) |
||||
coffee-script (2.4.1) |
||||
coffee-script-source |
||||
execjs |
||||
coffee-script-source (1.11.1) |
||||
colorator (1.1.0) |
||||
commonmarker (0.23.5) |
||||
concurrent-ruby (1.1.10) |
||||
dnsruby (1.61.9) |
||||
simpleidn (~> 0.1) |
||||
em-websocket (0.5.3) |
||||
eventmachine (>= 0.12.9) |
||||
http_parser.rb (~> 0) |
||||
ethon (0.15.0) |
||||
ffi (>= 1.15.0) |
||||
eventmachine (1.2.7) |
||||
execjs (2.8.1) |
||||
faraday (2.3.0) |
||||
faraday-net_http (~> 2.0) |
||||
ruby2_keywords (>= 0.0.4) |
||||
faraday-net_http (2.0.3) |
||||
ffi (1.15.5) |
||||
forwardable-extended (2.6.0) |
||||
gemoji (3.0.1) |
||||
github-pages (226) |
||||
github-pages-health-check (= 1.17.9) |
||||
jekyll (= 3.9.2) |
||||
jekyll-avatar (= 0.7.0) |
||||
jekyll-coffeescript (= 1.1.1) |
||||
jekyll-commonmark-ghpages (= 0.2.0) |
||||
jekyll-default-layout (= 0.1.4) |
||||
jekyll-feed (= 0.15.1) |
||||
jekyll-gist (= 1.5.0) |
||||
jekyll-github-metadata (= 2.13.0) |
||||
jekyll-include-cache (= 0.2.1) |
||||
jekyll-mentions (= 1.6.0) |
||||
jekyll-optional-front-matter (= 0.3.2) |
||||
jekyll-paginate (= 1.1.0) |
||||
jekyll-readme-index (= 0.3.0) |
||||
jekyll-redirect-from (= 0.16.0) |
||||
jekyll-relative-links (= 0.6.1) |
||||
jekyll-remote-theme (= 0.4.3) |
||||
jekyll-sass-converter (= 1.5.2) |
||||
jekyll-seo-tag (= 2.8.0) |
||||
jekyll-sitemap (= 1.4.0) |
||||
jekyll-swiss (= 1.0.0) |
||||
jekyll-theme-architect (= 0.2.0) |
||||
jekyll-theme-cayman (= 0.2.0) |
||||
jekyll-theme-dinky (= 0.2.0) |
||||
jekyll-theme-hacker (= 0.2.0) |
||||
jekyll-theme-leap-day (= 0.2.0) |
||||
jekyll-theme-merlot (= 0.2.0) |
||||
jekyll-theme-midnight (= 0.2.0) |
||||
jekyll-theme-minimal (= 0.2.0) |
||||
jekyll-theme-modernist (= 0.2.0) |
||||
jekyll-theme-primer (= 0.6.0) |
||||
jekyll-theme-slate (= 0.2.0) |
||||
jekyll-theme-tactile (= 0.2.0) |
||||
jekyll-theme-time-machine (= 0.2.0) |
||||
jekyll-titles-from-headings (= 0.5.3) |
||||
jemoji (= 0.12.0) |
||||
kramdown (= 2.3.2) |
||||
kramdown-parser-gfm (= 1.1.0) |
||||
liquid (= 4.0.3) |
||||
mercenary (~> 0.3) |
||||
minima (= 2.5.1) |
||||
nokogiri (>= 1.13.4, < 2.0) |
||||
rouge (= 3.26.0) |
||||
terminal-table (~> 1.4) |
||||
github-pages-health-check (1.17.9) |
||||
addressable (~> 2.3) |
||||
dnsruby (~> 1.60) |
||||
octokit (~> 4.0) |
||||
public_suffix (>= 3.0, < 5.0) |
||||
typhoeus (~> 1.3) |
||||
html-pipeline (2.14.2) |
||||
activesupport (>= 2) |
||||
nokogiri (>= 1.4) |
||||
http_parser.rb (0.8.0) |
||||
i18n (0.9.5) |
||||
concurrent-ruby (~> 1.0) |
||||
jekyll (3.9.2) |
||||
addressable (~> 2.4) |
||||
colorator (~> 1.0) |
||||
em-websocket (~> 0.5) |
||||
i18n (~> 0.7) |
||||
jekyll-sass-converter (~> 1.0) |
||||
jekyll-watch (~> 2.0) |
||||
kramdown (>= 1.17, < 3) |
||||
liquid (~> 4.0) |
||||
mercenary (~> 0.3.3) |
||||
pathutil (~> 0.9) |
||||
rouge (>= 1.7, < 4) |
||||
safe_yaml (~> 1.0) |
||||
jekyll-avatar (0.7.0) |
||||
jekyll (>= 3.0, < 5.0) |
||||
jekyll-coffeescript (1.1.1) |
||||
coffee-script (~> 2.2) |
||||
coffee-script-source (~> 1.11.1) |
||||
jekyll-commonmark (1.4.0) |
||||
commonmarker (~> 0.22) |
||||
jekyll-commonmark-ghpages (0.2.0) |
||||
commonmarker (~> 0.23.4) |
||||
jekyll (~> 3.9.0) |
||||
jekyll-commonmark (~> 1.4.0) |
||||
rouge (>= 2.0, < 4.0) |
||||
jekyll-default-layout (0.1.4) |
||||
jekyll (~> 3.0) |
||||
jekyll-feed (0.15.1) |
||||
jekyll (>= 3.7, < 5.0) |
||||
jekyll-gist (1.5.0) |
||||
octokit (~> 4.2) |
||||
jekyll-github-metadata (2.13.0) |
||||
jekyll (>= 3.4, < 5.0) |
||||
octokit (~> 4.0, != 4.4.0) |
||||
jekyll-include-cache (0.2.1) |
||||
jekyll (>= 3.7, < 5.0) |
||||
jekyll-mentions (1.6.0) |
||||
html-pipeline (~> 2.3) |
||||
jekyll (>= 3.7, < 5.0) |
||||
jekyll-optional-front-matter (0.3.2) |
||||
jekyll (>= 3.0, < 5.0) |
||||
jekyll-paginate (1.1.0) |
||||
jekyll-readme-index (0.3.0) |
||||
jekyll (>= 3.0, < 5.0) |
||||
jekyll-redirect-from (0.16.0) |
||||
jekyll (>= 3.3, < 5.0) |
||||
jekyll-relative-links (0.6.1) |
||||
jekyll (>= 3.3, < 5.0) |
||||
jekyll-remote-theme (0.4.3) |
||||
addressable (~> 2.0) |
||||
jekyll (>= 3.5, < 5.0) |
||||
jekyll-sass-converter (>= 1.0, <= 3.0.0, != 2.0.0) |
||||
rubyzip (>= 1.3.0, < 3.0) |
||||
jekyll-sass-converter (1.5.2) |
||||
sass (~> 3.4) |
||||
jekyll-seo-tag (2.8.0) |
||||
jekyll (>= 3.8, < 5.0) |
||||
jekyll-sitemap (1.4.0) |
||||
jekyll (>= 3.7, < 5.0) |
||||
jekyll-swiss (1.0.0) |
||||
jekyll-theme-architect (0.2.0) |
||||
jekyll (> 3.5, < 5.0) |
||||
jekyll-seo-tag (~> 2.0) |
||||
jekyll-theme-cayman (0.2.0) |
||||
jekyll (> 3.5, < 5.0) |
||||
jekyll-seo-tag (~> 2.0) |
||||
jekyll-theme-dinky (0.2.0) |
||||
jekyll (> 3.5, < 5.0) |
||||
jekyll-seo-tag (~> 2.0) |
||||
jekyll-theme-hacker (0.2.0) |
||||
jekyll (> 3.5, < 5.0) |
||||
jekyll-seo-tag (~> 2.0) |
||||
jekyll-theme-leap-day (0.2.0) |
||||
jekyll (> 3.5, < 5.0) |
||||
jekyll-seo-tag (~> 2.0) |
||||
jekyll-theme-merlot (0.2.0) |
||||
jekyll (> 3.5, < 5.0) |
||||
jekyll-seo-tag (~> 2.0) |
||||
jekyll-theme-midnight (0.2.0) |
||||
jekyll (> 3.5, < 5.0) |
||||
jekyll-seo-tag (~> 2.0) |
||||
jekyll-theme-minimal (0.2.0) |
||||
jekyll (> 3.5, < 5.0) |
||||
jekyll-seo-tag (~> 2.0) |
||||
jekyll-theme-modernist (0.2.0) |
||||
jekyll (> 3.5, < 5.0) |
||||
jekyll-seo-tag (~> 2.0) |
||||
jekyll-theme-primer (0.6.0) |
||||
jekyll (> 3.5, < 5.0) |
||||
jekyll-github-metadata (~> 2.9) |
||||
jekyll-seo-tag (~> 2.0) |
||||
jekyll-theme-slate (0.2.0) |
||||
jekyll (> 3.5, < 5.0) |
||||
jekyll-seo-tag (~> 2.0) |
||||
jekyll-theme-tactile (0.2.0) |
||||
jekyll (> 3.5, < 5.0) |
||||
jekyll-seo-tag (~> 2.0) |
||||
jekyll-theme-time-machine (0.2.0) |
||||
jekyll (> 3.5, < 5.0) |
||||
jekyll-seo-tag (~> 2.0) |
||||
jekyll-titles-from-headings (0.5.3) |
||||
jekyll (>= 3.3, < 5.0) |
||||
jekyll-watch (2.2.1) |
||||
listen (~> 3.0) |
||||
jemoji (0.12.0) |
||||
gemoji (~> 3.0) |
||||
html-pipeline (~> 2.2) |
||||
jekyll (>= 3.0, < 5.0) |
||||
kramdown (2.3.2) |
||||
rexml |
||||
kramdown-parser-gfm (1.1.0) |
||||
kramdown (~> 2.0) |
||||
liquid (4.0.3) |
||||
listen (3.7.1) |
||||
rb-fsevent (~> 0.10, >= 0.10.3) |
||||
rb-inotify (~> 0.9, >= 0.9.10) |
||||
mercenary (0.3.6) |
||||
minima (2.5.1) |
||||
jekyll (>= 3.5, < 5.0) |
||||
jekyll-feed (~> 0.9) |
||||
jekyll-seo-tag (~> 2.1) |
||||
minitest (5.16.2) |
||||
nokogiri (1.13.6-arm64-darwin) |
||||
racc (~> 1.4) |
||||
octokit (4.25.1) |
||||
faraday (>= 1, < 3) |
||||
sawyer (~> 0.9) |
||||
pathutil (0.16.2) |
||||
forwardable-extended (~> 2.6) |
||||
public_suffix (4.0.7) |
||||
racc (1.6.0) |
||||
rb-fsevent (0.11.1) |
||||
rb-inotify (0.10.1) |
||||
ffi (~> 1.0) |
||||
rexml (3.2.5) |
||||
rouge (3.26.0) |
||||
ruby2_keywords (0.0.5) |
||||
rubyzip (2.3.2) |
||||
safe_yaml (1.0.5) |
||||
sass (3.7.4) |
||||
sass-listen (~> 4.0.0) |
||||
sass-listen (4.0.0) |
||||
rb-fsevent (~> 0.9, >= 0.9.4) |
||||
rb-inotify (~> 0.9, >= 0.9.7) |
||||
sawyer (0.9.2) |
||||
addressable (>= 2.3.5) |
||||
faraday (>= 0.17.3, < 3) |
||||
simpleidn (0.2.1) |
||||
unf (~> 0.1.4) |
||||
terminal-table (1.8.0) |
||||
unicode-display_width (~> 1.1, >= 1.1.1) |
||||
thread_safe (0.3.6) |
||||
typhoeus (1.4.0) |
||||
ethon (>= 0.9.0) |
||||
tzinfo (1.2.9) |
||||
thread_safe (~> 0.1) |
||||
unf (0.1.4) |
||||
unf_ext |
||||
unf_ext (0.0.8.2) |
||||
unicode-display_width (1.8.0) |
||||
webrick (1.7.0) |
||||
zeitwerk (2.6.0) |
||||
|
||||
PLATFORMS |
||||
arm64-darwin-21 |
||||
|
||||
DEPENDENCIES |
||||
github-pages |
||||
webrick |
||||
|
||||
BUNDLED WITH |
||||
2.3.4 |
||||
@ -0,0 +1,12 @@
|
||||
title: "CORE Tech Docs" |
||||
remote_theme: just-the-docs/just-the-docs |
||||
lang: en-GB |
||||
permalink: pretty |
||||
color_scheme: govuk |
||||
aux_links: |
||||
"API browser": |
||||
- /api |
||||
"Design history": |
||||
- https://core-design-history.herokuapp.com |
||||
"GitHub": |
||||
- https://github.com/communitiesuk/submit-social-housing-lettings-and-sales-data |
||||
@ -0,0 +1,3 @@
|
||||
$btn-primary-color: #00703c; |
||||
$link-color: #1d70b8; |
||||
$grey-lt-000: #f8f8f8; |
||||
@ -1,19 +1,21 @@
|
||||
### ADR - 003: Form Submission Flow |
||||
--- |
||||
parent: Architecture decisions |
||||
--- |
||||
|
||||
Turbo Frames (https://github.com/hotwired/turbo-rails) for form pages/questions with data saved (but not necessarily fully validated) to Active Record model on each submit. |
||||
# 003: Form submission flow |
||||
|
||||
[Turbo Frames](https://github.com/hotwired/turbo-rails) for form pages/questions with data saved (but not necessarily fully validated) to Active Record model on each submit. |
||||
|
||||
#### Impact on Performance |
||||
## Impact on performance |
||||
|
||||
Using Turbo Frames allows us to swap out just the question part of the page without needing full page refreshes as you go through the form and provides a "Single Page Application like" user experience. Each question still gets a unique URL that can be navigated to directly with the Case Log ID and the overall user experience is that form navigation feels faster. |
||||
|
||||
#### Impact on interrupted sessions |
||||
## Impact on interrupted sessions |
||||
|
||||
We currently have a single Active Record model for Case Logs that contains all the question fields. Every time a question is submitted the answer will be saved in the Active Record model instance before the next frame is rendered. This model will need to be able to handle partial records and partial validation anyway since not all API users will have all the required data. Validation can occur based on the data already saved and/or once the form is finally submitted. Front end validation will still happen additionally as you go through the form to help make sure users don't get a long list of errors at the end. Using session data here and updating the model only once the form is completed would not seem to have any advantages over this approach. |
||||
We currently have a single Active Record model for Case Logs that contains all the question fields. Every time a question is submitted the answer will be saved in the Active Record model instance before the next frame is rendered. This model will need to be able to handle partial records and partial validation anyway since not all API users will have all the required data. Validation can occur based on the data already saved and/or once the form is finally submitted. Front end validation will still happen additionally as you go through the form to help make sure users don’t get a long list of errors at the end. Using session data here and updating the model only once the form is completed would not seem to have any advantages over this approach. |
||||
|
||||
This means that when a user navigates away from the form or closes the tab etc, they can use the URL to navigate directly back to where they left off, or follow the form flow through again, and in both cases their submitted answers will still be there. |
||||
|
||||
|
||||
#### Impact on API |
||||
## Impact on API |
||||
|
||||
The API will still expect to take a JSON describing the case log, instantiate the model with the given fields, and run validations as if it had been submitted. |
||||
|
||||
@ -1,11 +1,11 @@
|
||||
### ADR - 008: Field Names |
||||
--- |
||||
parent: Architecture decisions |
||||
--- |
||||
|
||||
We are changing the schema to reflect the way the data is stored in CORE. |
||||
This is due to the SPSS queries that are being performed by ADD and the complexity that would come with changing them. |
||||
# 008: Field names |
||||
|
||||
The field names are saved lowercase as opposed to the uppercase versions we see in CORE. |
||||
This is due to Ruby expecting the uppercase parameters to be constants and database fields are expected to be lower case. |
||||
These fields could be mapped to their uppercase versions during the replication if needed. |
||||
We are changing the schema to reflect the way the data is stored in CORE. This is due to the SPSS queries that are being performed by ADD and the complexity that would come with changing them. |
||||
|
||||
A lot of the values are now also being stored as enums. |
||||
This gives as some validation by default as the values not defined in the enums will fail to save. |
||||
The field names are saved lowercase as opposed to the uppercase versions we see in CORE. This is due to Ruby expecting the uppercase parameters to be constants and database fields are expected to be lower case. These fields could be mapped to their uppercase versions during the replication if needed. |
||||
|
||||
A lot of the values are now also being stored as enums. This gives as some validation by default as the values not defined in the enums will fail to save. |
||||
|
||||
@ -1,12 +1,16 @@
|
||||
### ADR - 009: Form Routing Logic |
||||
--- |
||||
parent: Architecture decisions |
||||
--- |
||||
|
||||
# 009: Form routing logic |
||||
|
||||
There are 2 ways you can think about form (page) routing logic: |
||||
|
||||
1. Based on the answer you give to a page you are navigated to some point in the form, i.e. a "Jump to" |
||||
1. Based on the answer you give to a page you are navigated to some point in the form, i.e. a ‘jump to’ |
||||
2. Each question is considered sequentially and independently and we evaluate whether it should be shown or not |
||||
|
||||
Our Form Definition DSL takes the second approach. This has a couple of advantages: |
||||
|
||||
- It makes the check answers pattern easier to code as you can ask each page directly: "Have the conditions for you to be shown been met?", with approach 1, you would effectively have to traverse the full route branch to see if a particular page was shown for each page/question which adds complexity. |
||||
- It makes the check answers pattern easier to code as you can ask each page directly: “Have the conditions for you to be shown been met?”, with approach 1, you would effectively have to traverse the full route branch to see if a particular page was shown for each page/question which adds complexity. |
||||
|
||||
- It makes it easier to look at the JSON and see at a glance what conditions will show or hide a page, which is closer to how the business logic is discussed and is easier to reason about. |
||||
|
||||
@ -1,9 +1,13 @@
|
||||
### ADR - 010: Admin Users vs Users |
||||
--- |
||||
parent: Architecture decisions |
||||
--- |
||||
|
||||
#### Why do we have 2 User classes, AdminUser and User? |
||||
# 010: Admin users vs Users |
||||
|
||||
This is modelling a real life split. `AdminUsers` are internal DLUHC users or helpdesk employees. While `Users` are external users working at data providing organisations. So local authority/housing association's "admin" users, i.e. Data Co-ordinators are a type of the User class. They have the ability to add or remove other users to or from their organisation, and to update their organisation details etc, but only through the designed UI. They do not get direct access to ActiveAdmin. |
||||
## Why do we have 2 user classes, `AdminUser` and `User`? |
||||
|
||||
This is modelling a real life split. `AdminUsers` are internal DLUHC users or help desk employees. While `Users` are external users working at data providing organisations. So local authority/housing association’s "admin" users, i.e. Data Co-ordinators are a type of the User class. They have the ability to add or remove other users to or from their organisation, and to update their organisation details etc, but only through the designed UI. They do not get direct access to ActiveAdmin. |
||||
|
||||
AdminUsers on the other hand get direct access to ActiveAdmin. From there they can download entire datasets (via CSV, XML, JSON), view any log from any organisation, and add or remove users of any type including other Admin users. This means TDA will likely also require more stringent authentication for them using MFA (which users will likely not require). So the class split also helps there. |
||||
|
||||
A potential downside to this approach is that it does not currently allow for `AdminUsers` to sign into the application UI itself with their Admin credentials. However, we need to see if there's an actual use case for this and what it would be (since they aren't part of an organisation to be uploading data for, but could add or amend data or user or org details through ActiveAdmin anyway). If there is a strong use case for it this could be work around by either: providing them with two sets of credentials, or modifying the `authenticate_user` method to also check `AdminUser` credentials. |
||||
A potential downside to this approach is that it does not currently allow for `AdminUsers` to sign into the application UI itself with their Admin credentials. However, we need to see if there’s an actual use case for this and what it would be (since they aren’t part of an organisation to be uploading data for, but could add or amend data or user or org details through ActiveAdmin anyway). If there is a strong use case for it this could be work around by either: providing them with two sets of credentials, or modifying the `authenticate_user` method to also check `AdminUser` credentials. |
||||
|
||||
@ -1,10 +1,16 @@
|
||||
### ADR - 011: Splitting the form parsing into objects |
||||
--- |
||||
parent: Architecture decisions |
||||
--- |
||||
|
||||
Initially a single "Form" class handled the parsing of the form definition JSON as well as a lot of the logic around what different sections meant. This works fine but led to a lot of places in code where we're passing around arguments to determine whether a page or section should or shouldn't do something rather than being able to ask it directly. Refactoring this into smaller form domain object classes has several benefits: |
||||
# 011: Splitting the form parsing into objects |
||||
|
||||
- It's easier to compare the form definition JSON to the code classes and reason about what fields can be passed and what effect they'll have |
||||
Initially a single `Form` class handled the parsing of the form definition JSON as well as a lot of the logic around what different sections meant. This works fine but led to a lot of places in code where we’re passing around arguments to determine whether a page or section should or shouldn’t do something rather than being able to ask it directly. |
||||
|
||||
Refactoring this into smaller form domain object classes has several benefits: |
||||
|
||||
- It’s easier to compare the form definition JSON to the code classes and reason about what fields can be passed and what effect they’ll have |
||||
- It moves business logic out of the helpers and keeps them to just dealing with display logic |
||||
- It makes it easier to unit test form functionality, and group that into smaller chunks |
||||
- It allows for less passing of arguments. e.g. `page.routed_to?(case_log)` vs `form.was_page_routed_to?(page, case_log)` |
||||
|
||||
This abstraction is likely still not the best (the form vs case log split) but this seems like an improvement that can be iterated on. |
||||
This abstraction is likely still not the best (the form vs case log split) but this seems like an improvement that can be iterated on. |
||||
|
||||
@ -1,12 +1,13 @@
|
||||
### ADR - 013: Inferring LA from postcode |
||||
--- |
||||
parent: Architecture decisions |
||||
--- |
||||
|
||||
# 013: Inferring LA from postcode |
||||
|
||||
We use ONS data to infer local authority from postcode in the property information section. |
||||
The Office for National Statistics (ONS) publishes the National Statistics |
||||
Postcode Lookup (NSPL) and ONS Postcode Directory (ONSPD) datasets, |
||||
which may be used to find a local authority district for a postcode when compiling statistics. |
||||
|
||||
We're using postcodes.io API with postcodes_io gem. |
||||
Postcodes.io uses OS and ONS data which is updated as soon as new data becomes available. |
||||
The Office for National Statistics (ONS) publishes the National Statistics Postcode Lookup (NSPL) and ONS Postcode Directory (ONSPD) datasets, which may be used to find a local authority district for a postcode when compiling statistics. |
||||
|
||||
We’re using postcodes.io API with postcodes_io gem. Postcodes.io uses OS and ONS data which is updated as soon as new data becomes available. |
||||
|
||||
We are not using OS places API due to the lack of data. |
||||
Closest datapoint to LA in OS places api is ADMINISTRATIVE_AREA which does not always match with local authority. |
||||
We are not using OS places API due to the lack of data. Closest data point to LA in OS places api is ADMINISTRATIVE_AREA which does not always match with local authority. |
||||
|
||||
@ -0,0 +1,8 @@
|
||||
--- |
||||
has_children: true |
||||
nav_order: 9 |
||||
--- |
||||
|
||||
# Architecture decisions |
||||
|
||||
A record of architectural decisions made on this project. |
||||
@ -0,0 +1,20 @@
|
||||
<!DOCTYPE html> |
||||
<html lang="en"> |
||||
<head> |
||||
<meta charset="UTF-8"> |
||||
<meta name="viewport" content="width=device-width, initial-scale=1"> |
||||
<link rel="stylesheet" type="text/css" href="https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/4.12.0/swagger-ui.css"> |
||||
<title>DLUHC CORE Data Collection API</title> |
||||
</head> |
||||
<body> |
||||
<div id="openapi"></div> |
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/4.12.0/swagger-ui-bundle.min.js"></script> |
||||
<script> |
||||
window.onload = function () { |
||||
const ui = SwaggerUIBundle({ |
||||
url: "v1.json", |
||||
dom_id: "#openapi" |
||||
}) |
||||
} |
||||
</script> |
||||
</body> |
||||
@ -1,13 +1,13 @@
|
||||
{ |
||||
"openapi": "3.0.0", |
||||
"info": { |
||||
"title": "DLUHC CORE Data", |
||||
"title": "DLUHC CORE Data Collection API", |
||||
"version": "1.0", |
||||
"description": "Submit or Update CORE Case Log Data on Lettings and Sales of Social Housing in England" |
||||
"description": "Submit social housing lettings and sales data (CORE)" |
||||
}, |
||||
"servers": [ |
||||
{ |
||||
"url": "https://dluhc-core.london.cloudapps.digital", |
||||
"url": "https://dluhc-core-staging.london.cloudapps.digital/logs", |
||||
"description": "Staging" |
||||
} |
||||
], |
||||
@ -0,0 +1,30 @@
|
||||
--- |
||||
has_children: true |
||||
nav_order: 8 |
||||
--- |
||||
|
||||
# Generating forms |
||||
|
||||
Social housing lettings and sales data is collected in annual collection windows that run from 1 April to 1 April the following year. |
||||
|
||||
During this window the form and questions generally stay constant. The form will generally change by small amounts between each collection window. Typical changes are adding new questions, adding or removing answer options from questions or tweaking question wording for clarity. |
||||
|
||||
A paper form is produced for guidance and to help data providers collect the data offline, and a bulk upload template is circulated which need to match the online form. |
||||
|
||||
Data is accepted for a collection window for up to 3 months after it’s finished to allow for late data submission. This means that between April and July 2 versions of the form run simultaneously. |
||||
|
||||
Other considerations that went into our design are being able to re-use as much of this solution for other data collections, and possibly having the ability to generate the form and/or form changes from a user interface. |
||||
|
||||
We haven’t used micro-services, preferring to deploy a single application but we have modelled the form itself as configuration in the form of a JSON structure that acts as a sort of DSL/form builder for the form. |
||||
|
||||
The idea is to decouple the code that creates the required routes, controller methods, views etc to display the form from the actual wording of questions or order of pages such that it becomes possible to make changes to the form with little or no code changes. |
||||
|
||||
This should also mean that in the future it could be possible to create an interface that can construct the JSON config, which would open up the ability to make form changes to a wider audience. Doing this fully would require generating and running the necessary migrations for data storage, generating the required ActiveRecord methods to validate the data server side, and generating/updating API endpoints and documentation. All of this is likely to be beyond the scope of initial MVP but could be looked at in the future. |
||||
|
||||
Since initially the JSON config will not create database migrations or ActiveRecord model validations, it will instead assume that these have been correctly created for the config provided. The reasoning for this is the following assumptions: |
||||
|
||||
- The form will be tweaked regularly (amending questions wording, changing the order of questions or the page a question is displayed on) |
||||
|
||||
- The actual data collected will change very infrequently. Time series continuity is very important to ADD (Analysis and Data Directorate) so the actual data collected should stay largely consistent i.e. in general we can change the question wording in ways that makes the intent clearer or easier to understand, but not in ways that would make the data provider give a different answer. |
||||
|
||||
A form parser class will parse this config into ruby objects/methods that can be used as an API by the rest of the application, such that we could change the underlying config if needed (for example swap JSON for YAML or for DataBase objects) without needing to change the rest of the application. We’ll call this the Form Runner part of the application. |
||||
@ -1,26 +1,34 @@
|
||||
## Section |
||||
--- |
||||
parent: Form definition |
||||
grand_parent: Generating forms |
||||
nav_order: 1 |
||||
--- |
||||
|
||||
Sections are under the top level of the form definition. A example section might look something like this: |
||||
# Section |
||||
|
||||
```JSON |
||||
Sections sit at the top level of a form definition. |
||||
|
||||
An example section might look something like this: |
||||
|
||||
```json |
||||
"sections": { |
||||
"tenancy_and_property": { |
||||
"label": "Property and tenancy information", |
||||
"subsections": { |
||||
"property_information": { |
||||
... |
||||
}, |
||||
"tenancy_information": { |
||||
... |
||||
} |
||||
"tenancy_and_property": { |
||||
"label": "Property and tenancy information", |
||||
"subsections": { |
||||
"property_information": { |
||||
... |
||||
}, |
||||
"tenancy_information": { |
||||
... |
||||
} |
||||
}, |
||||
... |
||||
} |
||||
}, |
||||
... |
||||
} |
||||
``` |
||||
|
||||
In the above example the section id would be `tenancy_and_property` and its subsections would be `property_information` and `tenancy_information`. |
||||
|
||||
The label contains the text that users will see for that section in the tasklist page of a case log. |
||||
The label contains the text that users will see for that section in the task list page of a case log. |
||||
|
||||
Sections can contain one or more subsections. |
||||
Sections can contain one or more [subsections](subsection). |
||||
|
||||
@ -1,17 +0,0 @@
|
||||
<!DOCTYPE html> |
||||
<html lang="en"> |
||||
<head> |
||||
<meta charset="UTF-8"> |
||||
<meta name="viewport" content="width=device-width, initial-scale=1"> |
||||
<link rel="stylesheet" type="text/css" href="https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/4.12.0/swagger-ui.css"> |
||||
<title>OpenAPI DLUHC CORE Data Collection</title> |
||||
<body><div id="openapi"><script src="https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/4.12.0/swagger-ui-bundle.min.js"></script> |
||||
<script> |
||||
window.onload = function () { |
||||
const ui = SwaggerUIBundle({ |
||||
url: "api/DLUHC-CORE-Data.v1.json", |
||||
dom_id: "#openapi" |
||||
}) |
||||
} |
||||
</script> |
||||
</body> |
||||
@ -0,0 +1,67 @@
|
||||
--- |
||||
nav_order: 1 |
||||
--- |
||||
|
||||
# Overview |
||||
|
||||
All lettings and and sales of social housing in England need to be logged with the Department for levelling up, housing and communities (DLUHC). This is done by data providing organisations: Local Authorities and Private Registered Providers (PRPs, i.e. housing associations). |
||||
|
||||
Data is collected via a form that runs on an annual data collection window basis. Form changes are made annually to add new questions, remove any that are no longer needed, or adjust wording or answer options etc. |
||||
|
||||
Each data collection window runs from 1 April to 1 April the following year (plus an extra 3 months to allow for any late submissions). This means that between April and June, 2 collection windows are open simultaneously and logs can be submitted for either. |
||||
|
||||
ADD (Analytics & Data Directorate) statisticians are the other primary users of the service. The data collected is transferred to DLUHCs consolidated data store (CDS) via nightly XML exports to an S3 bucket. CDS ingests and transforms this data, ultimately storing it in a MS SQL database and exposing it to analysts and statisticians via Amazon Workspaces. |
||||
|
||||
 |
||||
|
||||
## Users |
||||
|
||||
External data providing organisations have 2 main user types: |
||||
|
||||
- **Data coordinators** are administrators for their organisation, but may also complete logs |
||||
- **Data providers** complete the logs |
||||
|
||||
Additionally there are data protection officers (DPO). For some organisations this is a separate role, but in our codebase this is modelled as an attribute of a user (i.e. a data coordinator or provider can additionally be a DPO). They are responsible for ensuring the organisation has signed the data sharing agreement. |
||||
|
||||
There are also 2 internal user types: |
||||
|
||||
- **Customer support:** can administrate all organisations |
||||
- **Statisticians:** primary consumers of the collected data |
||||
|
||||
## Organisations |
||||
|
||||
There are 2 types of organisation: |
||||
|
||||
- An **owning organisations** own housing stock. It may manage the allocation of people in and out of their accommodation, or contract this function out to managing agents. |
||||
|
||||
- A **managing organisation** (or managing agent) is responsible for the allocation of people in and out of accommodation, and/or responsible for the services provided to support those people in the accommodation (in the case of supported housing). |
||||
|
||||
### Relationships between organisations |
||||
|
||||
Organisations that own stock can contract out the management of that stock to another organisation. This relationship is often referred to as a parent/child relationship. |
||||
|
||||
This is a useful analogy as a parent can have multiple children, and a child can have many parents. A child organisation can also be a parent, and a parent organisation can also be a child organisation: |
||||
|
||||
 |
||||
|
||||
### User permissions within organisations |
||||
|
||||
The case logs that a user can see depends on their role: |
||||
|
||||
- Customer support users can access any case log |
||||
|
||||
- Data coordinators can access any case log for which the organisation they work for is ultimately responsible for, meaning they can see logs managed by a child organisation |
||||
|
||||
- Data providers can only access case logs for which their organisation manages (or directly owns) |
||||
|
||||
Taking the relationships from the above diagram, and looking at which logs each user can access: |
||||
|
||||
 |
||||
|
||||
## Supported housing schemes |
||||
|
||||
A supported housing scheme (or service) provides shared or self-contained housing for a particular client group, for example younger or vulnerable people. A scheme can be run at multiple locations, and a single location may contain multiple units (for example bedrooms in shared houses or a bungalow with 3 bedrooms). |
||||
|
||||
Logs for supported housing will share a number of similar characteristics at this location. Additional data also needs to be collected specifically regarding the supported housing scheme, such as the type of client groups served and type of support provided. |
||||
|
||||
Asking these questions would require data inputters to re-enter the same information repeatedly and answer more questions than those asked for general needs lettings. Schemes exist in CORE to reduce this burden, and effectively act as predefined answer sets. |
||||
@ -1,25 +0,0 @@
|
||||
# Organisational relationships |
||||
|
||||
## Definitions |
||||
|
||||
- **Stock owning organisation**: An organisation that owns housing stock. It may manage the allocation of people in and out of their accommodation, or it may contract this out to managing agents. |
||||
|
||||
- **Managing agent**: In scenarios where one organisation owns stock and another organisation is contracted to manage the stock and tenants, the latter organisation is often called a ‘managing agent’. Managing agents are responsible for the allocation of people in and out of the accommodation, and/or responsible for the services provided to support those people in the accommodation (in the case of supported housing). |
||||
|
||||
## Permissions |
||||
|
||||
Organisations that own stock can contract out the management of that stock to another organisation. This relationship is often referred to as a parent/child relationship. This is a useful analogy as a parent can have multiple children, and a child can have many parents. A child organisation can also be a parent, and a parent organisation can also be a child organisation: |
||||
|
||||
 |
||||
|
||||
The case logs that a user can see depends on their role: |
||||
|
||||
- Customer support users can access any case log |
||||
|
||||
- Data coordinators can access any case log for which the organisation they work for is ultimately responsible for, meaning they can see logs managed by a child organisation |
||||
|
||||
- Data providers can only access case logs for which their organisation manages (or directly owns) |
||||
|
||||
Taking the relationships from the above diagram, and looking at which logs each user can access: |
||||
|
||||
 |
||||
@ -1,7 +0,0 @@
|
||||
# Supported housing schemes |
||||
|
||||
A supported housing scheme (or service) provides shared or self-contained housing for a particular client group, for example younger or vulnerable people. A scheme can be run at multiple locations, and a single location may contain multiple units (for example bedrooms in shared houses or a bungalow with 3 bedrooms). |
||||
|
||||
Logs for supported housing will share a number of similar characteristics at this location. Additional data also needs to be collected specifically regarding the supported housing scheme, such as the type of client groups served and type of support provided. |
||||
|
||||
Asking these questions would require data inputters to re-enter the same information repeatedly and answer more questions than those asked for general needs lettings. Schemes exist in CORE to reduce this burden, and effectively act as predefined answer sets. |
||||
@ -1,5 +0,0 @@
|
||||
# Service overview |
||||
|
||||
All lettings and and sales of social housing in England need to be logged with the Department for levelling up, housing and communities (DLUHC). This is done by Local Authorities and Housing Associations, who are the primary users of this service. Data is collected via a form that runs on an annual data collection window basis. Form changes are made annually to add new questions, remove any that are no longer needed, or adjust wording or answer options etc. Each data collection window runs from 1st April to 1st April + an extra 3 months to allow for any late submissions, meaning that between April and June, two collection windows are open simultaneously and logs can be submitted for either. |
||||
|
||||
ADD (Analytics & Data Directorate) statisticians are the other primary users of the service. The data collected is transferred to DLUHCs data warehouse (CDS - consolidated data store), via nightly exports to XML which are transferred to S3 and ingested from there. CDS ingests and transforms the data, ultimately storing it in a MS SQL database and exposing it to analysts and statisticians via Amazon Workspaces. |
||||
@ -1,15 +0,0 @@
|
||||
# User roles |
||||
|
||||
## External users |
||||
|
||||
The primary users of the system are external data providing organisations: Local Authorities and Private Registered Providers (Housing Associations). These have 2 main user types: |
||||
|
||||
- Data coordinators – administrators for their own organisation, can also complete logs |
||||
- Data providers – complete the logs |
||||
|
||||
Additionally there are Data Protection Officers (DPO), which for some organisations is a separate role, but in our codebase is modelled as an attribute of the user (i.e. a data coordinator or provider can additionally be a DPO). They are responsible for ensuring the organisation has signed the data sharing agreement. |
||||
|
||||
## Internal users |
||||
|
||||
- Customer support (help desk) – can administrate all organisations |
||||
- ADD statisticians – primary consumers of the data collected via CDS/DAP |
||||
Loading…
Reference in new issue