CLDC-3241: Part 1 - Explanation of variables on CSVs (#2585)

* Create CsvVariableDefinition migration and model

* Add definition files

* Add CsvVariableDefinitions to rails admin

* Create rake and service

* Create tests

* Additional fixes

* Rollback unwanted changes to schema

* Update Gemfile.lock

app/models/csv_variable_definition.rb

@@ -0,0 +1,10 @@
+class CsvVariableDefinition < ApplicationRecord
+  validates :variable, presence: true
+  validates :definition, presence: true
+  validates :log_type, presence: true, inclusion: { in: %w[lettings sales] }
+  validates :year, presence: true, numericality: { only_integer: true, greater_than_or_equal_to: 2000, less_than_or_equal_to: 2099 }
+  attribute :last_accessed, :datetime
+
+  scope :lettings, -> { where(log_type: "lettings") }
+  scope :sales, -> { where(log_type: "sales") }
+end

app/services/imports/variable_definitions_service.rb

@@ -0,0 +1,53 @@
+require "csv"
+
+module Imports
+  class VariableDefinitionsService
+    attr_reader :path, :count
+
+    def initialize(path:)
+      @path = path
+      @count = 0
+    end
+
+    def call
+      files = Dir.glob(File.join(@path, "*.csv"))
+      files.each do |file|
+        process_file(file)
+      end
+    end
+
+  private
+
+    def process_file(file)
+      file_name = File.basename(file, ".csv")
+      parsed_file_name = file_name.split("_")
+      log_type = parsed_file_name[0]
+      year = "20#{parsed_file_name[2]}".to_i
+
+      records_added = 0
+
+      CSV.foreach(file) do |row|
+        next if row.empty?
+
+        variable = row[0].downcase
+        definition = row[1..].join(",")
+        next if variable.nil? || definition.nil?
+
+        existing_record = CsvVariableDefinition.find_by(variable: variable.strip, definition: definition.strip, log_type:)
+
+        if existing_record.nil?
+          CsvVariableDefinition.create!(
+            variable: variable.strip,
+            definition: definition.strip,
+            log_type:,
+            year:,
+          )
+          records_added += 1
+        end
+      end
+
+      Rails.logger.debug "Added #{records_added} variable/definition records for file: #{file_name}. Duplicates excluded."
+      @count += records_added
+    end
+  end
+end

config/csv/definitions/lettings_download_23_24.csv

@@ -0,0 +1,209 @@
+id,Log ID
+status,Status of log
+duplicate_set_id,ID of a set of duplicate logs
+created_by,User the log is created by
+is_dpo,Is the user in the assigned_to column the data protection officer?
+created_at,Time and date the log was created
+updated_by,User who last updated the log
+updated_at,Time and date the log was last updated
+creation_method,Was the log submitted in-service or via bulk upload?
+old_id,The (internal) ID on the old service
+old_form_id,The ID the users saw on the old service
+collection_start_year,Year collection period opened
+assigned_to,User the log is assigned to
+owning_organisation_name,Which organisation owns this property?
+managing_organisation_name,Which organisation manages this letting?
+needstype,What is the needs type?
+lettype,What is the letting type?
+renewal,Is this letting a renewal?
+startdate,What is the tenancy start date?
+renttype,What is the rent type? (grouped into SR, IR or AR)
+renttype_detail,What is the rent type?
+irproduct,Which type of Intermediate Rent is this letting?
+irproduct_other,Which 'Other' type of Intermediate Rent is this letting?
+lar,Is this a London Affordable Rent letting?
+tenancycode,What is the tenant code?
+propcode,What is the property reference?
+uprn_known,Is the UPRN known?
+uprn,If known, property's UPRN
+uprn_confirmed,We found an address that might be this property. Is this the property address?
+address_line1,Address line 1
+address_line2,Address line 2
+town_or_city,Town or City
+county,County
+postcode_full,Postcode
+is_la_inferred,The internal value to indicate if the LA was inferred from the postcode
+la_label,What is the property's local authority?
+la,Local authority code
+first_time_property_let_as_social_housing,Is this the first time the property has been let as social housing?
+unitletas,What rent product was the property most recently let as?
+rsnvac,What is the reason for the property being vacant?
+newprop,Is this property new to the social rented sector?
+offered,How many times was the property offered between becoming vacant and this letting?
+unittype_gn,What type of unit is the property?
+builtype,Which type of building is the property?
+wchair,Is the property built or adapted to wheelchair-user standards?
+beds,How many bedrooms does the property have?
+voiddate,What is the void date?
+vacdays,Number of days the property was vacant
+void_date_value_check,The following soft validation was confirmed: You told us that the property has been vacant for more than 2 years. This is higher than we would expect.
+majorrepairs,Were any major repairs carried out during the void period?
+mrcdate,What date were any major repairs completed on?
+major_repairs_date_value_check,The following soft validation was confirmed: You told us the property has been vacant for 2 years. This is higher than we would expect.
+joint,Is this a joint tenancy?
+startertenancy,Is this a starter tenancy?
+tenancy,What is the type of tenancy?
+tenancyother,If 'Other', what is the type of tenancy?
+tenancylength,What is the length of the fixed-term tenancy to the nearest year?
+sheltered,Is this letting in sheltered accommodation?
+declaration,Has the tenant seen the MHCLG privacy notice?
+hhmemb,How many people live in the household at this letting?
+pregnancy_value_check,The following soft validation was confirmed: You told us somebody in the household is pregnant. You also told us there are no female tenants living at the property.
+refused,Where household characteristics have a 'Refused' option for some or all of: AGE1-AGE8, SEX1-SEX8, RELAT2-RELAT8, ECSTAT1-ECSTAT8
+hhtype,Type of household 1 = 1 elder; 2 = 2 adults, including elder(s); 3 = 1 adult; 4 = 2 adults; 5 = 1 adult & 1+ children; 6 = 2+ adults & 1+ children; 9 = Other
+totchild,Total number of dependent children in the household (Sum of when RELAT2-8 = C)
+totelder,Total number of elders in household (Sum of when AGE1-8 >= 60)
+totadult,Total number of adults in household
+age1,What is the lead tenant's age?
+retirement_value_check,The following soft validation was confirmed: You told us this person is aged %{age} years and retired. The minimum expected retirement age for %{gender} in England is %{age}.
+sex1,Which of these best describes the lead tenant's gender identity?
+ethnic_group,What is the lead tenant's ethnic group?
+ethnic,Which of these best describes the lead tenant's ethnic background?
+national,What is the lead tenant's nationality?
+ecstat1,Which of these best describes the lead tenant's working situation?
+details_known_2,Are the details of tenant 2 known?
+relat2,What is person 2's relationship to the lead tenant?
+age2,What is person 2's age?
+sex2,Which of these best describes person 2's gender identity?
+ecstat2,Which of these best describes person 2's working situation?
+details_known_3,Are the details of tenant 3 known?
+relat3,What is person 3's relationship to the lead tenant?
+age3,What is person 3's age?
+sex3,Which of these best describes person 3's gender identity?
+ecstat3,Which of these best describes person 3's working situation?
+details_known_4,Are the details of tenant 4 known?
+relat4,What is person 4's relationship to the lead tenant?
+age4,What is person 4's age?
+sex4,Which of these best describes person 4's gender identity?
+ecstat4,Which of these best describes person 4's working situation?
+details_known_5,Are the details of tenant 5 known?
+relat5,What is person 5's relationship to the lead tenant?
+age5,What is person 5's age?
+sex5,Which of these best describes person 5's gender identity?
+ecstat5,Which of these best describes person 5's working situation?
+details_known_6,Are the details of tenant 6 known?
+relat6,What is person 6's relationship to the lead tenant?
+age6,What is person 6's age?
+sex6,Which of these best describes person 6's gender identity?
+ecstat6,Which of these best describes person 6's working situation?
+details_known_7,Are the details of tenant 7 known?
+relat7,What is person 7's relationship to the lead tenant?
+age7,What is person 7's age?
+sex7,Which of these best describes person 7's gender identity?
+ecstat7,Which of these best describes person 7's working situation?
+details_known_8,Are the details of tenant 8 known?
+relat8,What is person 8's relationship to the lead tenant?
+age8,What is person 8's age?
+sex8,Which of these best describes person 8's gender identity?
+ecstat8,Which of these best describes person 8's working situation?
+armedforces,Does anybody in the household have links to the UK armed forces?
+leftreg,Is this person still serving in the UK armed forces?
+reservist,Was this person seriously injured or ill as a result of serving in the UK armed forces?
+preg_occ,Is anybody in the household pregnant?
+housingneeds,Does anybody in the household have any disabled access needs?
+housingneeds_type,What access needs do they have? (Fully wheelchair-accessible housing, Level access housing or Wheelchair access to essential rooms)
+housingneeds_a,Disabled access needs a) Fully wheelchair-accessible housing
+housingneeds_b,Disabled access needs b) Wheelchair access to essential rooms
+housingneeds_c,Disabled access needs c) Level access housing
+housingneeds_f,Disabled access needs f) Other disabled access needs
+housingneeds_g,Disabled access needs g) No disabled access needs
+housingneeds_h,Disabled access needs h) Don't know
+housingneeds_other,Do they have any other disabled access needs?
+illness,Does anybody in the household have a physical or mental health condition (or other illness) expected to last 12 months or more?
+illness_type_4,Does this person's condition affect their dexterity?
+illness_type_5,Does this person's condition affect their learning or understanding or concentrating?
+illness_type_2,Does this person's condition affect their hearing?
+illness_type_6,Does this person's condition affect their memory?
+illness_type_7,Does this person's condition affect their mental health?
+illness_type_3,Does this person's condition affect their mobility?
+illness_type_9,Does this person's condition affect them socially or behaviourally?
+illness_type_8,Does this person's condition affect their stamina or breathing or fatigue?
+illness_type_1,Does this person's condition affect their vision?
+illness_type_10,Does this person's condition affect them in another way?
+layear,How long has the household continuously lived in the local authority area of the new letting?
+waityear,How long has the household been on the local authority waiting list for the new letting?
+reason,What is the tenant's main reason for the household leaving their last settled home?
+reasonother,If 'Other', what was the main reason for leaving their last settled home?
+prevten,Where was the household immediately before this letting?
+homeless,Did the household experience homelessness immediately before this letting?
+ppcodenk,Previous postcode unknown or previous accommodation was temporary
+ppostcode_full,What is the postcode of the household's last settled home?
+previous_la_known,Was the local authority of the household's last settled home known?
+is_previous_la_inferred,The internal value to indicate if the previous LA was inferred from the postcode
+prevloc_label,Previous location LA name
+prevloc,Previous location's ONS LA Code
+reasonpref,Was the household given reasonable preference by the local authority?
+rp_homeless,Reasonable preference reason - They were homeless or about to lose their home (within 56 days)
+rp_insan_unsat,Reasonable preference reason - They were living in insanitary, overcrowded or unisatisfactory housing
+rp_medwel,Reasonable preference reason - They needed to move on medical and welfare reasons (including disability)
+rp_hardship,Reasonable preference reason - They needed to move to avoid hardship to themselves or others
+rp_dontknow,Reasonable preference reason - Don't Know
+cbl,Was the letting made under Choice-Based Lettings (CBL)?
+cap,Was the letting made under the Common Allocation Policy (CAP)?
+chr,Was the letting made under the Common Housing Register (CHR)?
+letting_allocation_none,The letting was not allocated under CBL, CAP, CHR or Accessible Register.
+referral,What was the source of referral for this letting?
+referral_value_check,The following soft validation was confirmed: Are you sure? This is a general needs log, and this referral type is for supported housing.
+net_income_known,Do you know the household's combined income after tax?
+incref,Was the household income refused?
+earnings,How much income does the household have in total?
+incfreq,How often does the household receive income?
+net_income_value_check,Populated when someone hits the soft validation and confirmed in the service
+hb,Is the tenant likely to be receiving any of these housing-related benefits?
+has_benefits,Does the tenant receive housing-related benefits? Yes if hb = Universal Credit housing element or Housing benefit, No if hb =  Don't Know, Neither, Tenant prefers not to say or blank
+benefits,How much of the household's income is from Universal Credit, state pensions or benefits?
+household_charge,Does the household pay rent or other charges for the accommodation?
+nocharge,Does the household pay rent or other charges for the accommodation? - flag for when household_charge is answered no
+period,How often does the household pay rent and other charges?
+is_carehome,Is this accommodation a care home?
+chcharge,If this is a care home, how much does the household pay every [time period]?
+wchchrg,Weekly care home charge
+carehome_charges_value_check,Populated when the soft validation and confirmed in the service
+brent,What is the basic rent?
+wrent,Weekly rent
+rent_value_check,Populated when the soft validation and confirmed in the service
+scharge,What is the service charge?
+wscharge,Weekly service charge
+pscharge,What is the personal service charge?
+wpschrge,Weekly personal service charge
+supcharg,What is the support charge?
+wsupchrg,Weekly support charge
+tcharge,Total charge to the tenant
+wtcharge,Weekly total charge to the tenant
+scharge_value_check,Populated when the soft validation and confirmed in the service
+pscharge_value_check,Populated when the soft validation and confirmed in the service
+supcharg_value_check,Populated when the soft validation and confirmed in the service
+hbrentshortfall,After the household has received any housing-related benefits, will they still need to pay for rent and charges?
+tshortfall_known,Can you estimate the outstanding amount?
+tshortfall,Estimated outstanding amount
+wtshortfall,Weekly total rent shortfall charge for tenant receiving housing benefit
+scheme_code,What scheme does this letting belong to?
+scheme_service_name,From scheme code, we map to the scheme name
+scheme_confidential,Does the scheme contain confidential information?
+SCHTYPE,What is this type of scheme? (Direct access hostel), Foyer, Housing for older people or Other supported housing
+scheme_registered_under_care_act,Is this scheme registered under the Care Standards Act 2000?
+scheme_owning_organisation_name,Which organisation owns the housing stock for this scheme?
+scheme_primary_client_group,What client group is this scheme intended for?
+scheme_has_other_client_group,Does this scheme provide for another client group?
+scheme_secondary_client_group,What is the other client group?
+scheme_support_type,What support does this scheme provide?
+scheme_intended_stay,Intended length of stay
+scheme_created_at,Date scheme was created
+location_code,Which location is this letting for?
+location_postcode,What is the postcode for this location?
+location_name,What is the name of this location?
+location_units,How many units are at this location?
+location_type_of_unit,What is the most common type of unit at this location?
+location_mobility_type,What are the mobility standards for the majority of the units in this location?
+location_local_authority,What is the local authority of this postcode?
+location_startdate,When did the first property in this location become available under this scheme?

config/csv/definitions/lettings_download_24_25.csv

@@ -0,0 +1,220 @@
+id,Log ID
+status,Status of log
+duplicate_set_id,ID of a set of duplicate logs
+created_by,User the log is created by
+is_dpo,Is the user in the assigned_to column the data protection officer?
+created_at,Time and date the log was created
+updated_by,User who last updated the log
+updated_at,Time and date the log was last updated
+creation_method,Was the log submitted in-service or via bulk upload?
+collection_start_year,Year collection period opened
+address_line1_as_entered,Address line 1 entered in bulk upload file
+address_line2_as_entered,Address line 2 entered in bulk upload file
+town_or_city_as_entered,Town or city entered in bulk upload file
+county_as_entered,County entered in bulk upload file
+postcode_full_as_entered,Postcode entered in bulk upload file
+la_as_entered,Local authority entered in bulk upload file
+bulk_upload_id,ID of a set of bulk uploaded logs
+assigned_to,User the log is assigned to
+owning_organisation_name,Which organisation owns this property?
+managing_organisation_name,Which organisation manages this letting?
+needstype,What is the needs type?
+lettype,What is the letting type?
+renewal,Is this letting a renewal?
+startdate,What is the tenancy start date?
+renttype,What is the rent type? (grouped into SR, IR or AR)
+renttype_detail,What is the rent type?
+irproduct,Which type of Intermediate Rent is this letting?
+irproduct_other,Which 'Other' type of Intermediate Rent is this letting?
+lar,Is this a London Affordable Rent letting?
+tenancycode,What is the tenant code?
+propcode,What is the property reference?
+uprn_known,Is the UPRN known?
+uprn,If known, property's UPRN
+address_line1_input,Address line 1 input from address matching feature
+postcode_full_input,Postcode input from address matching feature
+address_search_value_check,Was the 'No address found' page seen?
+uprn_selection,UPRN of the address selected
+address_line1,Address line 1
+address_line2,Address line 2
+town_or_city,Town or City
+county,County
+postcode_full,Postcode
+is_la_inferred,The internal value to indicate if the LA was inferred from the postcode
+la_label,What is the property's local authority?
+la,Local authority code
+first_time_property_let_as_social_housing,Is this the first time the property has been let as social housing?
+unitletas,What rent product was the property most recently let as?
+rsnvac,What is the reason for the property being vacant?
+newprop,Is this property new to the social rented sector?
+unittype_gn,What type of unit is the property?
+builtype,Which type of building is the property?
+wchair,Is the property built or adapted to wheelchair-user standards?
+beds,How many bedrooms does the property have?
+voiddate,What is the void date?
+vacdays,Number of days the property was vacant
+void_date_value_check,The following soft validation was confirmed: You told us that the property has been vacant for more than 2 years. This is higher than we would expect.
+majorrepairs,Were any major repairs carried out during the void period?
+mrcdate,What date were any major repairs completed on?
+major_repairs_date_value_check,The following soft validation was confirmed: You told us the property has been vacant for 2 years. This is higher than we would expect.
+joint,Is this a joint tenancy?
+startertenancy,Is this a starter tenancy?
+tenancy,What is the type of tenancy?
+tenancyother,If 'Other', what is the type of tenancy?
+tenancylength,What is the length of the fixed-term tenancy to the nearest year?
+sheltered,Is this letting in sheltered accommodation?
+declaration,Has the tenant seen the MHCLG privacy notice?
+hhmemb,How many people live in the household at this letting?
+pregnancy_value_check,The following soft validation was confirmed: You told us somebody in the household is pregnant. You also told us there are no female tenants living at the property.
+refused,Where household characteristics have a 'Refused' option for some or all of: AGE1-AGE8, SEX1-SEX8, RELAT2-RELAT8, ECSTAT1-ECSTAT8
+hhtype,Type of household 1 = 1 elder; 2 = 2 adults, including elder(s); 3 = 1 adult; 4 = 2 adults; 5 = 1 adult & 1+ children; 6 = 2+ adults & 1+ children; 9 = Other
+totchild,Total number of dependent children in the household (Sum of when RELAT2-8 = C)
+totelder,Total number of elders in household (Sum of when AGE1-8 >= 60)
+totadult,Total number of adults in household
+age1,What is the lead tenant's age?
+retirement_value_check,The following soft validation was confirmed: You told us this person is aged %{age} years and retired. The minimum expected retirement age for %{gender} in England is %{age}.
+sex1,Which of these best describes the lead tenant's gender identity?
+ethnic_group,What is the lead tenant's ethnic group?
+ethnic,Which of these best describes the lead tenant's ethnic background?
+nationality_all,What is the lead tenant's nationality?
+ecstat1,Which of these best describes the lead tenant's working situation?
+details_known_2,Are the details of tenant 2 known?
+relat2,What is person 2's relationship to the lead tenant?
+partner_under_16_value_check,The following soft validation was confirmed: You said that [person X]'s relationship to lead tenant is partner, and that their age is [AGEX]. Are you sure this is correct?
+multiple_partners_value_check,The following soft validation was confirmed: You said that more than one person in the household is the partner of the lead tenant. Are you sure this is correct?
+age2,What is person 2's age?
+sex2,Which of these best describes person 2's gender identity?
+ecstat2,Which of these best describes person 2's working situation?
+details_known_3,Are the details of tenant 3 known?
+relat3,What is person 3's relationship to the lead tenant?
+age3,What is person 3's age?
+sex3,Which of these best describes person 3's gender identity?
+ecstat3,Which of these best describes person 3's working situation?
+details_known_4,Are the details of tenant 4 known?
+relat4,What is person 4's relationship to the lead tenant?
+age4,What is person 4's age?
+sex4,Which of these best describes person 4's gender identity?
+ecstat4,Which of these best describes person 4's working situation?
+details_known_5,Are the details of tenant 5 known?
+relat5,What is person 5's relationship to the lead tenant?
+age5,What is person 5's age?
+sex5,Which of these best describes person 5's gender identity?
+ecstat5,Which of these best describes person 5's working situation?
+details_known_6,Are the details of tenant 6 known?
+relat6,What is person 6's relationship to the lead tenant?
+age6,What is person 6's age?
+sex6,Which of these best describes person 6's gender identity?
+ecstat6,Which of these best describes person 6's working situation?
+details_known_7,Are the details of tenant 7 known?
+relat7,What is person 7's relationship to the lead tenant?
+age7,What is person 7's age?
+sex7,Which of these best describes person 7's gender identity?
+ecstat7,Which of these best describes person 7's working situation?
+details_known_8,Are the details of tenant 8 known?
+relat8,What is person 8's relationship to the lead tenant?
+age8,What is person 8's age?
+sex8,Which of these best describes person 8's gender identity?
+ecstat8,Which of these best describes person 8's working situation?
+armedforces,Does anybody in the household have links to the UK armed forces?
+leftreg,Is this person still serving in the UK armed forces?
+reservist,Was this person seriously injured or ill as a result of serving in the UK armed forces?
+preg_occ,Is anybody in the household pregnant?
+housingneeds,Does anybody in the household have any disabled access needs?
+housingneeds_type,What access needs do they have? (Fully wheelchair-accessible housing, Level access housing or Wheelchair access to essential rooms)
+housingneeds_a,Disabled access needs a) Fully wheelchair-accessible housing
+housingneeds_b,Disabled access needs b) Wheelchair access to essential rooms
+housingneeds_c,Disabled access needs c) Level access housing
+housingneeds_f,Disabled access needs f) Other disabled access needs
+housingneeds_g,Disabled access needs g) No disabled access needs
+housingneeds_h,Disabled access needs h) Don't know
+housingneeds_other,Do they have any other disabled access needs?
+illness,Does anybody in the household have a physical or mental health condition (or other illness) expected to last 12 months or more?
+illness_type_4,Does this person's condition affect their dexterity?
+illness_type_5,Does this person's condition affect their learning or understanding or concentrating?
+illness_type_2,Does this person's condition affect their hearing?
+illness_type_6,Does this person's condition affect their memory?
+illness_type_7,Does this person's condition affect their mental health?
+illness_type_3,Does this person's condition affect their mobility?
+illness_type_9,Does this person's condition affect them socially or behaviourally?
+illness_type_8,Does this person's condition affect their stamina or breathing or fatigue?
+illness_type_1,Does this person's condition affect their vision?
+illness_type_10,Does this person's condition affect them in another way?
+layear,How long has the household continuously lived in the local authority area of the new letting?
+waityear,How long has the household been on the local authority waiting list for the new letting?
+reason,What is the tenant's main reason for the household leaving their last settled home?
+reasonother,If 'Other', what was the main reason for leaving their last settled home?
+reasonother_value_check,The soft validation was confirmed
+prevten,Where was the household immediately before this letting?
+homeless,Did the household experience homelessness immediately before this letting?
+ppcodenk,Previous postcode unknown or previous accommodation was temporary
+ppostcode_full,What is the postcode of the household's last settled home?
+previous_la_known,Was the local authority of the household's last settled home known?
+is_previous_la_inferred,The internal value to indicate if the previous LA was inferred from the postcode
+prevloc_label,Previous location LA name
+prevloc,Previous location's ONS LA Code
+reasonpref,Was the household given reasonable preference by the local authority?
+rp_homeless,Reasonable preference reason - They were homeless or about to lose their home (within 56 days)
+rp_insan_unsat,Reasonable preference reason - They were living in insanitary, overcrowded or unisatisfactory housing
+rp_medwel,Reasonable preference reason - They needed to move on medical and welfare reasons (including disability)
+rp_hardship,Reasonable preference reason - They needed to move to avoid hardship to themselves or others
+rp_dontknow,Reasonable preference reason - Don't Know
+cbl,Was the letting made under Choice-Based Lettings (CBL)?
+cap,Was the letting made under the Common Allocation Policy (CAP)?
+chr,Was the letting made under the Common Housing Register (CHR)?
+accessible_register,Was the letting made under the Accessible Register?
+letting_allocation_none,The letting was not allocated under CBL, CAP, CHR or Accessible Register.
+referral,What was the source of referral for this letting?
+referral_value_check,The following soft validation was confirmed: Are you sure? This is a general needs log, and this referral type is for supported housing.
+net_income_known,Do you know the household's combined income after tax?
+incref,Was the household income refused?
+earnings,How much income does the household have in total?
+incfreq,How often does the household receive income?
+net_income_value_check,Populated when someone hits the soft validation and confirmed in the service
+hb,Is the tenant likely to be receiving any of these housing-related benefits?
+has_benefits,Does the tenant receive housing-related benefits? Yes if hb = Universal Credit housing element or Housing benefit, No if hb =  Don't Know, Neither, Tenant prefers not to say or blank
+benefits,How much of the household's income is from Universal Credit, state pensions or benefits?
+household_charge,Does the household pay rent or other charges for the accommodation?
+nocharge,Does the household pay rent or other charges for the accommodation? - flag for when household_charge is answered no
+period,How often does the household pay rent and other charges?
+is_carehome,Is this accommodation a care home?
+chcharge,If this is a care home, how much does the household pay every [time period]?
+wchchrg,Weekly care home charge
+carehome_charges_value_check,Populated when the soft validation and confirmed in the service
+brent,What is the basic rent?
+wrent,Weekly rent
+rent_value_check,Populated when the soft validation and confirmed in the service
+scharge,What is the service charge?
+wscharge,Weekly service charge
+pscharge,What is the personal service charge?
+wpschrge,Weekly personal service charge
+supcharg,What is the support charge?
+wsupchrg,Weekly support charge
+tcharge,Total charge to the tenant
+wtcharge,Weekly total charge to the tenant
+scharge_value_check,Populated when the soft validation and confirmed in the service
+pscharge_value_check,Populated when the soft validation and confirmed in the service
+supcharg_value_check,Populated when the soft validation and confirmed in the service
+hbrentshortfall,After the household has received any housing-related benefits, will they still need to pay for rent and charges?
+tshortfall_known,Can you estimate the outstanding amount?
+tshortfall,Estimated outstanding amount
+wtshortfall,Weekly total rent shortfall charge for tenant receiving housing benefit
+scheme_code,What scheme does this letting belong to?
+scheme_service_name,From scheme code, we map to the scheme name
+scheme_confidential,Does the scheme contain confidential information?
+SCHTYPE,What is this type of scheme? (Direct access hostel), Foyer, Housing for older people or Other supported housing
+scheme_registered_under_care_act,Is this scheme registered under the Care Standards Act 2000?
+scheme_owning_organisation_name,Which organisation owns the housing stock for this scheme?
+scheme_primary_client_group,What client group is this scheme intended for?
+scheme_has_other_client_group,Does this scheme provide for another client group?
+scheme_secondary_client_group,What is the other client group?
+scheme_support_type,What support does this scheme provide?
+scheme_intended_stay,Intended length of stay
+scheme_created_at,Date scheme was created
+location_code,Which location is this letting for?
+location_postcode,What is the postcode for this location?
+location_name,What is the name of this location?
+location_units,How many units are at this location?
+location_type_of_unit,What is the most common type of unit at this location?
+location_mobility_type,What are the mobility standards for the majority of the units in this location?
+location_local_authority,What is the local authority of this postcode?
+location_startdate,When did the first property in this location become available under this scheme?

config/csv/definitions/sales_download_23_24.csv

@@ -0,0 +1,269 @@
+ID,Log ID
+STATUS,Status of log
+DUPLICATESET,ID of a set of duplicate logs
+CREATEDDATE,Time and date the log was created
+UPLOADDATE,Time and date the log was last updated
+FORM,The ID on the old service
+COLLECTIONYEAR,Year collection period opened
+CREATIONMETHOD,Was the log submitted in-service or via bulk upload?
+DATAPROTECT,Is the user in the created_by column the data protection officer?
+OWNINGORGNAME,Which organisation owned this property before the sale?
+MANINGORGNAME,Which organisation reported the sale?
+CREATEDBY,User that created the log
+USERNAME,User the log is assigned to
+DAY,Day of sale completion date
+MONTH,Month of sale completion date
+YEAR,Year of sale completion date
+PURCHID,What is the purchaser code?
+OWNERSHIP,Was this purchase made through an ownership scheme?
+TYPE,What is the type of shared ownership/discounted ownership/outright sale?
+OTHTYPE,If type = 'Other', what is the type of outright sale?
+COMPANY,Is the buyer a company?
+LIVEINBUYER,Will the buyer(s) live in the property?
+JOINT,Is this a joint purchase?
+JOINTMORE,Are there more than 2 joint buyers of this property?
+BEDS,How many bedrooms does the property have?
+PROPTYPE,What type of unit is the property?
+BUILTYPE,Which type of building is the property?
+UPRN,What is the UPRN of the property?
+UPRNCONFIRMED,We found an address that might be this property. Is this the property address?
+ADDRESS1,Address line 1
+ADDRESS2,Address line 2
+TOWNCITY,Town/City
+COUNTY,County
+PCODE1,Part 1 of the property's postcode
+PCODE2,Part 2 of the property's postcode
+LA,LA code
+LANAME,LA name
+WCHAIR,Is the property built or adapted to wheelchair-user standards?
+NOINT,Did you interview the buyer to answer these questions?
+PRIVACYNOTICE,Has the buyer seen the MHCLG privacy notice?
+AGE1,What is buyer 1's age?
+SEX1,Which of these best describes buyer 1's gender identity?
+ETHNICGROUP1,What is buyer 1's ethnic group?
+ETHNIC,Which of the following best describes buyer 1's ethnic background?
+NATIONAL,What is buyer 1's nationality?
+ECSTAT1,Which of these best describes buyer 1's working situation?
+LIVEINBUYER1,Will buyer 1 live in the property?
+RELAT2,What is buyer 2 or person 2's relationship to buyer 1?
+AGE2,What is buyer 2 or person 2's age?
+SEX2,Which of these best describes buyer 2 or person 2's gender identity?
+ETHNICGROUP2,What is buyer 2's ethnic group?
+ETHNIC2,Which of the following best describes buyer 2's ethnic background?
+NATIONAL2,What is buyer 2's nationality?
+ECSTAT2,What is buyer 2 or person 2's working situation?
+LIVEINBUYER2,Will buyer 2 live in the property?
+HHTYPE,Besides the buyer(s), how many other people live or will live in the property?
+RELAT3,What is person 3's relationship to buyer 1?
+AGE3,What is person 3's age?
+SEX3,What is person 3's gender identity?
+ECSTAT3,What is person 3's working situation?
+RELAT4,What is person 4's relationship to buyer 1?
+AGE4,What is person 4's age?
+SEX4,What is person 4's gender identity?
+ECSTAT4,What is person 4's working situation?
+RELAT5,What is person 5's relationship to buyer 1?
+AGE5,What is person 5's age?
+SEX5,What is person 5's gender identity?
+ECSTAT5,What is person 5's working situation?
+RELAT6,What is person 6's relationship to buyer 1?
+AGE6,What is person 6's age?
+SEX6,What is person 6's gender identity?
+ECSTAT6,What is person 6's working situation?
+PREVTEN,What was buyer 1's previous tenure?
+PPCODENK,Do you know the postcode of buyer 1's last settled accommodation?
+PPOSTC1,Part 1 of postcode of buyer 1's last settled accommodation
+PPOSTC2,Part 2 of postcode of buyer 1's last settled accommodation
+PREVIOUSLAKNOWN,Do you know the local authority of buyer 1's last settled accommodation?
+PREVLOC,The local authority code of buyer 1's last settled accommodation
+PREVLOCNAME,The local authority name of buyer 1's last settled accommodation
+PREGYRHA,Was the buyer registered with their PRP (HA)?
+PREGOTHER,Was the buyer registered with another PRP (HA)?
+PREGLA,Was the buyer registered with the local authority?
+PREGGHB,Was the buyer registered with a Help to Buy agent?
+PREGBLANK,Populated if pregyrha, pregother, pregla and pregghb are blank
+BUY2LIVING,At the time of purchase, was buyer 2 living at the same address as buyer 1?
+PREVTEN2,What was buyer 2's previous tenure?
+HHREGRES,Have any of the buyers ever served as a regular in the UK armed forces?
+HHREGRESSTILL,Is the buyer still serving in the UK armed forces?
+ARMEDFORCESSPOUSE,Are any of the buyers a spouse or civil partner of a UK armed forces regular who died in service within the last 2 years?
+DISABLED,Does anyone in the household consider themselves to have a disability?
+WHEEL,Does anyone in the household use a wheelchair?
+INC1NK,Is buyer 1's annual income known?
+INCOME1,What is buyer 1's annual income?
+INC1MORT,Was buyer 1's income used for a mortgage application?
+INC2NK,Is buyer 1's annual income known?
+INCOME2,What is buyer 2's annual income?
+INC2MORT,Was buyer 2's income used for a mortgage application?
+HB,Were the buyers receiving any of these housing-related benefits immediately before buying this property?
+SAVINGSNK,Is the the total amount the buyers had in savings known?
+SAVINGS,What is the total amount the buyers had in savings before they paid any deposit for the property?
+PREVOWN,Have any of the buyers previously owned a property?
+PREVSHARED,Was the previous property under shared ownership?
+PROPLEN,How long did the buyer(s) live in the property before purchasing it?
+STAIRCASE,Is this a staircasing transaction?
+STAIRBOUGHT,What percentage of the property has been bought in this staircasing transaction?
+STAIROWNED,What percentage of the property do the buyers now own in total?
+STAIRCASETOSALE,Was this transaction part of a back-to-back staircasing transaction to facilitate sale of the home on the open market?
+RESALE,Is this a resale?
+EXDAY,Day of the exchange of contracts
+EXMONTH,Month of the exchange of contracts
+EXYEAR,Year of the exchange of contracts
+HODAY,Day of the practical completion or handover date
+HOMONTH,Month of the practical completion or handover date
+HOYEAR,Year of the practical completion or handover date
+LANOMAGR,Was the household rehoused under a local authority nominations agreement?
+SOCTEN,Was the buyer a private registered provider, housing association or local authority tenant immediately before this sale?
+FROMBEDS,How many bedrooms did the buyer's previous property have?
+FROMPROP,What was the previous property type?
+SOCPREVTEN,What was the rent type of buyer's previous tenure?
+VALUE,What is the full purchase price?
+VALUE_VALUE_CHECK,Populated if a soft validation is confirmed.
+EQUITY,What was the initial percentage equity stake purchased?
+MORTGAGEUSED,Was a mortgage used to buy this property?
+MORTGAGE,What is the mortgage amount?
+MORTGAGELENDER,What is the name of the mortgage lender?
+MORTGAGELENDEROTHER,If mortgagelender = 'Other', what is the name of the mortgage lender?
+MORTLEN1,What is the length of the mortgage in years?
+EXTRABOR,Does this include any extra borrowing?
+DEPOSIT,How much was the cash deposit paid on the property?
+CASHDIS,How much cash discount was given through Social Homebuy?
+MRENT,What is the basic monthly rent?
+HASMSCHARGE,Does the property have any monthly leasehold charges?
+MSCHARGE,What are the total monthly leasehold charges for the property?
+MSCHARGE_VALUE_CHECK,Populated if a soft validation is confirmed.
+DISCOUNT,What was the percentage discount?
+GRANT,What was the amount of any loan, grant, discount or subsidy given?
+id,Log ID
+status,Status of log
+duplicate_set_id,ID of a set of duplicate logs
+created_at,Time and date the log was created
+updated_at,Time and date the log was last updated
+old_form_id,The ID on the old service
+collection_start_year,Year collection period opened
+creation_method,Was the log submitted in-service or via bulk upload?
+is_dpo,Is the user in the assigned_to column the data protection officer?
+owning_organisation_name,Which organisation owned this property before the sale?
+managing_organisation_name,Which organisation reported the sale?
+assigned_to,User the log is assigned to
+day,Day of sale completion date
+month,Month of sale completion date
+year,Year of sale completion date
+purchid,What is the purchaser code?
+ownershipsch,Was this purchase made through an ownership scheme?
+type,What is the type of shared ownership/discounted ownership/outright sale?
+othtype,If type = 'Other', what is the type of outright sale?
+companybuy,Is the buyer a company?
+buylivein,Will the buyer(s) live in the property?
+jointpur,Is this a joint purchase?
+jointmore,Are there more than 2 joint buyers of this property?
+beds,How many bedrooms does the property have?
+proptype,What type of unit is the property?
+builtype,Which type of building is the property?
+uprn,What is the UPRN of the property?
+uprn_confirmed,We found an address that might be this property. Is this the property address?
+address_line1,Address line 1
+address_line2,Address line 2
+town_or_city,Town/City
+county,County
+pcode1,Part 1 of the property's postcode
+pcode2,Part 2 of the property's postcode
+la,LA code
+la_label,LA name
+wchair,Is the property built or adapted to wheelchair-user standards?
+noint,Did you interview the buyer to answer these questions?
+privacynotice,Has the buyer seen the MHCLG privacy notice?
+age1,What is buyer 1's age?
+sex1,Which of these best describes buyer 1's gender identity?
+ethnic_group,What is buyer 1's ethnic group?
+ethnic,Which of the following best describes buyer 1's ethnic background?
+national,What is buyer 1's nationality?
+ecstat1,Which of these best describes buyer 1's working situation?
+buy1livein,Will buyer 1 live in the property?
+relat2,What is buyer 2 or person 2's relationship to buyer 1?
+age2,What is buyer 2 or person 2's age?
+sex2,Which of these best describes buyer 2 or person 2's gender identity?
+ethnic_group2,What is buyer 2's ethnic group?
+ethnicbuy2,Which of the following best describes buyer 2's ethnic background?
+nationalbuy2,What is buyer 2's nationality?
+ecstat2,What is buyer 2 or person 2's working situation?
+buy2livein,Will buyer 2 live in the property?
+hholdcount,Besides the buyer(s), how many other people live or will live in the property?
+relat3,What is person 3's relationship to buyer 1?
+age3,What is person 3's age?
+sex3,What is person 3's gender identity?
+ecstat3,What is person 3's working situation?
+relat4,What is person 4's relationship to buyer 1?
+age4,What is person 4's age?
+sex4,What is person 4's gender identity?
+ecstat4,What is person 4's working situation?
+relat5,What is person 5's relationship to buyer 1?
+age5,What is person 5's age?
+sex5,What is person 5's gender identity?
+ecstat5,What is person 5's working situation?
+relat6,What is person 6's relationship to buyer 1?
+age6,What is person 6's age?
+sex6,What is person 6's gender identity?
+ecstat6,What is person 6's working situation?
+prevten,What was buyer 1's previous tenure?
+ppcodenk,Do you know the postcode of buyer 1's last settled accommodation?
+ppostc1,Part 1 of postcode of buyer 1's last settled accommodation
+ppostc2,Part 2 of postcode of buyer 1's last settled accommodation
+previous_la_known,Do you know the local authority of buyer 1's last settled accommodation?
+prevloc,The local authority code of buyer 1's last settled accommodation
+prevloc_label,The local authority name of buyer 1's last settled accommodation
+pregyrha,Was the buyer registered with their PRP (HA)?
+pregother,Was the buyer registered with another PRP (HA)?
+pregla,Was the buyer registered with the local authority?
+pregghb,Was the buyer registered with a Help to Buy agent?
+pregblank,Populated if pregyrha, pregother, pregla and pregghb are blank
+buy2living,At the time of purchase, was buyer 2 living at the same address as buyer 1?
+prevtenbuy2,What was buyer 2's previous tenure?
+hhregres,Have any of the buyers ever served as a regular in the UK armed forces?
+hhregresstill,Is the buyer still serving in the UK armed forces?
+armedforcesspouse,Are any of the buyers a spouse or civil partner of a UK armed forces regular who died in service within the last 2 years?
+disabled,Does anyone in the household consider themselves to have a disability?
+wheel,Does anyone in the household use a wheelchair?
+income1nk,Is buyer 1's annual income known?
+income1,What is buyer 1's annual income?
+inc1mort,Was buyer 1's income used for a mortgage application?
+income2nk,Is buyer 2's annual income known?
+income2,What is buyer 2's annual income?
+inc2mort,Was buyer 2's income used for a mortgage application?
+hb,Were the buyers receiving any of these housing-related benefits immediately before buying this property?
+savingsnk,Is the the total amount the buyers had in savings known?
+savings,What is the total amount the buyers had in savings before they paid any deposit for the property?
+prevown,Have any of the buyers previously owned a property?
+prevshared,Was the previous property under shared ownership?
+proplen,How long did the buyer(s) live in the property before purchasing it?
+staircase,Is this a staircasing transaction?
+stairbought,What percentage of the property has been bought in this staircasing transaction?
+stairowned,What percentage of the property do the buyers now own in total?
+staircasesale,Was this transaction part of a back-to-back staircasing transaction to facilitate sale of the home on the open market?
+resale,Is this a resale?
+exday,Day of the exchange of contracts
+exmonth,Month of the exchange of contracts
+exyear,Year of the exchange of contracts
+hoday,Day of the practical completion or handover date
+homonth,Month of the practical completion or handover date
+hoyear,Year of the practical completion or handover date
+lanomagr,Was the household rehoused under a local authority nominations agreement?
+soctenant,Was the buyer a private registered provider, housing association or local authority tenant immediately before this sale?
+frombeds,How many bedrooms did the buyer's previous property have?
+fromprop,What was the previous property type?
+socprevten,What was the rent type of buyer's previous tenure?
+value,What is the full purchase price?
+equity,What was the initial percentage equity stake purchased?
+mortgageused,Was a mortgage used to buy this property?
+mortgage,What is the mortgage amount?
+mortgagelender,What is the name of the mortgage lender?
+mortgagelenderother,If mortgagelender = 'Other', what is the name of the mortgage lender?
+mortlen,What is the length of the mortgage in years?
+extrabor,Does this include any extra borrowing?
+deposit,How much was the cash deposit paid on the property?
+cashdis,How much cash discount was given through Social Homebuy?
+mrent,What is the basic monthly rent?
+has_mscharge,Does the property have any monthly leasehold charges?
+mscharge,What are the total monthly leasehold charges for the property?
+discount,What was the percentage discount?
+grant,What was the amount of any loan, grant, discount or subsidy given?

config/csv/definitions/sales_download_24_25.csv

@@ -0,0 +1,281 @@
+ID,Log ID
+STATUS,Status of log
+DUPLICATESET,ID of a set of duplicate logs
+CREATEDDATE,Time and date the log was created
+UPLOADDATE,Time and date the log was last updated
+COLLECTIONYEAR,Year collection period opened
+CREATIONMETHOD,Was the log submitted in-service or via bulk upload?
+BULKUPLOADID,ID of a set of bulk uploaded logs
+DATAPROTECT,Is the user in the created_by column the data protection officer?
+OWNINGORGNAME,Which organisation owned this property before the sale?
+MANINGORGNAME,Which organisation reported the sale?
+CREATEDBY,User that created the log
+USERNAME,User the log is assigned to
+DAY,Day of sale completion date
+MONTH,Month of sale completion date
+YEAR,Year of sale completion date
+PURCHID,What is the purchaser code?
+OWNERSHIP,Was this purchase made through an ownership scheme?
+TYPE,What is the type of shared ownership/discounted ownership/outright sale?
+OTHTYPE,If type = 'Other', what is the type of outright sale?
+COMPANY,Is the buyer a company?
+LIVEINBUYER,Will the buyer(s) live in the property?
+JOINT,Is this a joint purchase?
+JOINTMORE,Are there more than 2 joint buyers of this property?
+NOINT,Did you interview the buyer to answer these questions?
+PRIVACYNOTICE,Has the buyer seen the MHCLG privacy notice?
+UPRN,What is the UPRN of the property?
+ADDRESS1,Address line 1
+ADDRESS2,Address line 2
+TOWNCITY,Town/City
+COUNTY,County
+POSTCODE,Postcode
+ISLAINFERRED,The internal value to indicate if the LA was inferred from the postcode
+LANAME,LA name
+LA,LA code
+UPRNSELECTED,UPRN of the address selected
+ADDRESS_SEARCH_VALUE_CHECK,Was the 'No address found' page seen?
+ADDRESS1INPUT,Address line 1 input from address matching feature
+POSTCODEINPUT,Postcode input from address matching feature
+BULKADDRESS1,Address line 1 entered in bulk upload file
+BULKADDRESS2,Address line 2 entered in bulk upload file
+BULKTOWNCITY,Town or city entered in bulk upload file
+BULKCOUNTY,County entered in bulk upload file
+BULKPOSTCODE,Postcode entered in bulk upload file
+BULKLA,Local authority entered in bulk upload file
+BEDS,How many bedrooms does the property have?
+PROPTYPE,What type of unit is the property?
+BUILTYPE,Which type of building is the property?
+WCHAIR,Is the property built or adapted to wheelchair-user standards?
+AGE1,What is buyer 1's age?
+SEX1,Which of these best describes buyer 1's gender identity?
+ETHNICGROUP1,What is buyer 1's ethnic group?
+ETHNIC,Which of the following best describes buyer 1's ethnic background?
+NATIONALITYALL1,What is buyer 1's nationality?
+ECSTAT1,Which of these best describes buyer 1's working situation?
+LIVEINBUYER1,Will buyer 1 live in the property?
+RELAT2,What is buyer 2 or person 2's relationship to buyer 1?
+AGE2,What is buyer 2 or person 2's age?
+SEX2,Which of these best describes buyer 2 or person 2's gender identity?
+ETHNICGROUP2,What is buyer 2's ethnic group?
+ETHNIC2,Which of the following best describes buyer 2's ethnic background?
+NATIONALITYALL2,What is buyer 2's nationality?
+ECSTAT2,What is buyer 2 or person 2's working situation?
+LIVEINBUYER2,Will buyer 2 live in the property?
+HHTYPE,Besides the buyer(s), how many other people live or will live in the property?
+RELAT3,What is person 3's relationship to buyer 1?
+AGE3,What is person 3's age?
+SEX3,What is person 3's gender identity?
+ECSTAT3,What is person 3's working situation?
+RELAT4,What is person 4's relationship to buyer 1?
+AGE4,What is person 4's age?
+SEX4,What is person 4's gender identity?
+ECSTAT4,What is person 4's working situation?
+RELAT5,What is person 5's relationship to buyer 1?
+AGE5,What is person 5's age?
+SEX5,What is person 5's gender identity?
+ECSTAT5,What is person 5's working situation?
+RELAT6,What is person 6's relationship to buyer 1?
+AGE6,What is person 6's age?
+SEX6,What is person 6's gender identity?
+ECSTAT6,What is person 6's working situation?
+PREVTEN,What was buyer 1's previous tenure?
+PPCODENK,Do you know the postcode of buyer 1's last settled accommodation?
+PPOSTC1,Part 1 of postcode of buyer 1's last settled accommodation
+PPOSTC2,Part 2 of postcode of buyer 1's last settled accommodation
+PREVIOUSLAKNOWN,Do you know the local authority of buyer 1's last settled accommodation?
+PREVLOC,The local authority code of buyer 1's last settled accommodation
+PREVLOCNAME,The local authority name of buyer 1's last settled accommodation
+PREGYRHA,Was the buyer registered with their PRP (HA)?
+PREGOTHER,Was the buyer registered with another PRP (HA)?
+PREGLA,Was the buyer registered with the local authority?
+PREGGHB,Was the buyer registered with a Help to Buy agent?
+PREGBLANK,Populated if pregyrha, pregother, pregla and pregghb are blank
+BUY2LIVING,At the time of purchase, was buyer 2 living at the same address as buyer 1?
+PREVTEN2,What was buyer 2's previous tenure?
+HHREGRES,Have any of the buyers ever served as a regular in the UK armed forces?
+HHREGRESSTILL,Is the buyer still serving in the UK armed forces?
+ARMEDFORCESSPOUSE,Are any of the buyers a spouse or civil partner of a UK armed forces regular who died in service within the last 2 years?
+DISABLED,Does anyone in the household consider themselves to have a disability?
+WHEEL,Does anyone in the household use a wheelchair?
+INC1NK,Is buyer 1's annual income known?
+INCOME1,What is buyer 1's annual income?
+INC1MORT,Was buyer 1's income used for a mortgage application?
+INC2NK,Is buyer 1's annual income known?
+INCOME2,What is buyer 2's annual income?
+INC2MORT,Was buyer 2's income used for a mortgage application?
+HB,Were the buyers receiving any of these housing-related benefits immediately before buying this property?
+SAVINGSNK,Is the the total amount the buyers had in savings known?
+SAVINGS,What is the total amount the buyers had in savings before they paid any deposit for the property?
+PREVOWN,Have any of the buyers previously owned a property?
+PREVSHARED,Was the previous property under shared ownership?
+PROPLEN,How long did the buyer(s) live in the property before purchasing it?
+STAIRCASE,Is this a staircasing transaction?
+STAIRBOUGHT,What percentage of the property has been bought in this staircasing transaction?
+STAIROWNED,What percentage of the property do the buyers now own in total?
+STAIRCASETOSALE,Was this transaction part of a back-to-back staircasing transaction to facilitate sale of the home on the open market?
+RESALE,Is this a resale?
+EXDAY,Day of the exchange of contracts
+EXMONTH,Month of the exchange of contracts
+EXYEAR,Year of the exchange of contracts
+HODAY,Day of the practical completion or handover date
+HOMONTH,Month of the practical completion or handover date
+HOYEAR,Year of the practical completion or handover date
+LANOMAGR,Was the household rehoused under a local authority nominations agreement?
+SOCTEN,Was the buyer a private registered provider, housing association or local authority tenant immediately before this sale?
+FROMBEDS,How many bedrooms did the buyer's previous property have?
+FROMPROP,What was the previous property type?
+SOCPREVTEN,What was the rent type of buyer's previous tenure?
+VALUE,What is the full purchase price?
+VALUE_VALUE_CHECK,Populated if a soft validation is confirmed.
+EQUITY,What was the initial percentage equity stake purchased?
+MORTGAGEUSED,Was a mortgage used to buy this property?
+MORTGAGE,What is the mortgage amount?
+MORTGAGELENDER,What is the name of the mortgage lender?
+MORTGAGELENDEROTHER,If mortgagelender = 'Other', what is the name of the mortgage lender?
+MORTLEN1,What is the length of the mortgage in years?
+EXTRABOR,Does this include any extra borrowing?
+DEPOSIT,How much was the cash deposit paid on the property?
+CASHDIS,How much cash discount was given through Social Homebuy?
+MRENT,What is the basic monthly rent?
+HASMSCHARGE,Does the property have any monthly leasehold charges?
+MSCHARGE,What are the total monthly leasehold charges for the property?
+MSCHARGE_VALUE_CHECK,Populated if a soft validation is confirmed.
+DISCOUNT,What was the percentage discount?
+GRANT,What was the amount of any loan, grant, discount or subsidy given?
+id,Log ID
+status,Status of log
+duplicate_set_id,ID of a set of duplicate logs
+created_at,Time and date the log was created
+updated_at,Time and date the log was last updated
+old_form_id,The ID on the old service
+collection_start_year,Year collection period opened
+creation_method,Was the log submitted in-service or via bulk upload?
+is_dpo,Is the user in the assigned_to column the data protection officer?
+owning_organisation_name,Which organisation owned this property before the sale?
+managing_organisation_name,Which organisation reported the sale?
+assigned_to,User the log is assigned to
+day,Day of sale completion date
+month,Month of sale completion date
+year,Year of sale completion date
+purchid,What is the purchaser code?
+ownershipsch,Was this purchase made through an ownership scheme?
+type,What is the type of shared ownership/discounted ownership/outright sale?
+othtype,If type = 'Other', what is the type of outright sale?
+companybuy,Is the buyer a company?
+buylivein,Will the buyer(s) live in the property?
+jointpur,Is this a joint purchase?
+jointmore,Are there more than 2 joint buyers of this property?
+beds,How many bedrooms does the property have?
+proptype,What type of unit is the property?
+builtype,Which type of building is the property?
+uprn,What is the UPRN of the property?
+uprn_confirmed,We found an address that might be this property. Is this the property address?
+address_line1_input,Address line 1 input from address matching feature
+postcode_full_input,Postcode input from address matching feature
+uprn_selection,UPRN of the address selected
+address_line1,Address line 1
+address_line2,Address line 2
+town_or_city,Town/City
+county,County
+pcode1,Part 1 of the property's postcode
+pcode2,Part 2 of the property's postcode
+la,LA code
+la_label,LA name
+wchair,Is the property built or adapted to wheelchair-user standards?
+noint,Did you interview the buyer to answer these questions?
+privacynotice,Has the buyer seen the MHCLG privacy notice?
+age1,What is buyer 1's age?
+sex1,Which of these best describes buyer 1's gender identity?
+ethnic_group,What is buyer 1's ethnic group?
+ethnic,Which of the following best describes buyer 1's ethnic background?
+nationality_all,What is buyer 1's nationality?
+ecstat1,Which of these best describes buyer 1's working situation?
+buy1livein,Will buyer 1 live in the property?
+relat2,What is buyer 2 or person 2's relationship to buyer 1?
+age2,What is buyer 2 or person 2's age?
+sex2,Which of these best describes buyer 2 or person 2's gender identity?
+ethnic_group2,What is buyer 2's ethnic group?
+ethnicbuy2,Which of the following best describes buyer 2's ethnic background?
+nationality_all_buyer2,What is buyer 2's nationality?
+ecstat2,What is buyer 2 or person 2's working situation?
+buy2livein,Will buyer 2 live in the property?
+hholdcount,Besides the buyer(s), how many other people live or will live in the property?
+relat3,What is person 3's relationship to buyer 1?
+age3,What is person 3's age?
+sex3,What is person 3's gender identity?
+ecstat3,What is person 3's working situation?
+relat4,What is person 4's relationship to buyer 1?
+age4,What is person 4's age?
+sex4,What is person 4's gender identity?
+ecstat4,What is person 4's working situation?
+relat5,What is person 5's relationship to buyer 1?
+age5,What is person 5's age?
+sex5,What is person 5's gender identity?
+ecstat5,What is person 5's working situation?
+relat6,What is person 6's relationship to buyer 1?
+age6,What is person 6's age?
+sex6,What is person 6's gender identity?
+ecstat6,What is person 6's working situation?
+prevten,What was buyer 1's previous tenure?
+ppcodenk,Do you know the postcode of buyer 1's last settled accommodation?
+ppostc1,Part 1 of postcode of buyer 1's last settled accommodation
+ppostc2,Part 2 of postcode of buyer 1's last settled accommodation
+previous_la_known,Do you know the local authority of buyer 1's last settled accommodation?
+prevloc,The local authority code of buyer 1's last settled accommodation
+prevloc_label,The local authority name of buyer 1's last settled accommodation
+pregyrha,Was the buyer registered with their PRP (HA)?
+pregother,Was the buyer registered with another PRP (HA)?
+pregla,Was the buyer registered with the local authority?
+pregghb,Was the buyer registered with a Help to Buy agent?
+pregblank,Populated if pregyrha, pregother, pregla and pregghb are blank
+buy2living,At the time of purchase, was buyer 2 living at the same address as buyer 1?
+prevtenbuy2,What was buyer 2's previous tenure?
+hhregres,Have any of the buyers ever served as a regular in the UK armed forces?
+hhregresstill,Is the buyer still serving in the UK armed forces?
+armedforcesspouse,Are any of the buyers a spouse or civil partner of a UK armed forces regular who died in service within the last 2 years?
+disabled,Does anyone in the household consider themselves to have a disability?
+wheel,Does anyone in the household use a wheelchair?
+income1nk,Is buyer 1's annual income known?
+income1,What is buyer 1's annual income?
+inc1mort,Was buyer 1's income used for a mortgage application?
+income2nk,Is buyer 2's annual income known?
+income2,What is buyer 2's annual income?
+inc2mort,Was buyer 2's income used for a mortgage application?
+hb,Were the buyers receiving any of these housing-related benefits immediately before buying this property?
+savingsnk,Is the the total amount the buyers had in savings known?
+savings,What is the total amount the buyers had in savings before they paid any deposit for the property?
+prevown,Have any of the buyers previously owned a property?
+prevshared,Was the previous property under shared ownership?
+proplen,How long did the buyer(s) live in the property before purchasing it?
+staircase,Is this a staircasing transaction?
+stairbought,What percentage of the property has been bought in this staircasing transaction?
+stairowned,What percentage of the property do the buyers now own in total?
+staircasesale,Was this transaction part of a back-to-back staircasing transaction to facilitate sale of the home on the open market?
+resale,Is this a resale?
+exday,Day of the exchange of contracts
+exmonth,Month of the exchange of contracts
+exyear,Year of the exchange of contracts
+hoday,Day of the practical completion or handover date
+homonth,Month of the practical completion or handover date
+hoyear,Year of the practical completion or handover date
+lanomagr,Was the household rehoused under a local authority nominations agreement?
+soctenant,Was the buyer a private registered provider, housing association or local authority tenant immediately before this sale?
+frombeds,How many bedrooms did the buyer's previous property have?
+fromprop,What was the previous property type?
+socprevten,What was the rent type of buyer's previous tenure?
+value,What is the full purchase price?
+equity,What was the initial percentage equity stake purchased?
+mortgageused,Was a mortgage used to buy this property?
+mortgage,What is the mortgage amount?
+mortgagelender,What is the name of the mortgage lender?
+mortgagelenderother,If mortgagelender = 'Other', what is the name of the mortgage lender?
+mortlen,What is the length of the mortgage in years?
+extrabor,Does this include any extra borrowing?
+deposit,How much was the cash deposit paid on the property?
+cashdis,How much cash discount was given through Social Homebuy?
+mrent,What is the basic monthly rent?
+has_mscharge,Does the property have any monthly leasehold charges?
+mscharge,What are the total monthly leasehold charges for the property?
+discount,What was the percentage discount?
+grant,What was the amount of any loan, grant, discount or subsidy given?

config/initializers/rails_admin.rb

@@ -25,8 +25,25 @@ RailsAdmin.config do |config|
 
   ## == Gravatar integration ==
   ## To disable Gravatar integration in Navigation Bar set to false
-  # config.show_gravatar = true
-  config.included_models = %w[LogValidation]
+  config.show_gravatar = false
+  config.included_models = %w[LogValidation CsvVariableDefinition]
+
+  config.model "LogValidation" do
+    label "Log Validation"
+  end
+
+  config.model "CsvVariableDefinition" do
+    label "CSV Variable Definition"
+    edit do
+      exclude_fields :last_accessed
+      field :log_type do
+        help "Required. Specify the type of log associated with this variable: 'lettings' or 'sales'."
+      end
+      field :year do
+        help "Required. Specify the year this definition should be available from. This definition will be used in subsequent years unless superseded by a newer definition."
+      end
+    end
+  end
 
   config.actions do
     dashboard                     # mandatory

db/migrate/20240726152326_create_csv_variable_definitions.rb

@@ -0,0 +1,15 @@
+class CreateCsvVariableDefinitions < ActiveRecord::Migration[7.0]
+  def change
+    create_table :csv_variable_definitions do |t|
+      t.string :variable, null: false
+      t.string :definition, null: false
+      t.string :log_type, null: false
+      t.integer :year, null: false
+      t.datetime :last_accessed
+      t.timestamps
+    end
+
+    add_check_constraint :csv_variable_definitions, "log_type IN ('lettings', 'sales')", name: "log_type_check"
+    add_check_constraint :csv_variable_definitions, "year BETWEEN 2000 AND 2099", name: "year_check"
+  end
+end

db/schema.rb

@@ -46,6 +46,18 @@ ActiveRecord::Schema[7.0].define(version: 2024_08_19_143150) do
     t.index ["user_id"], name: "index_bulk_uploads_on_user_id"
   end
 
+  create_table "csv_variable_definitions", force: :cascade do |t|
+    t.string "variable", null: false
+    t.string "definition", null: false
+    t.string "log_type", null: false
+    t.integer "year", null: false
+    t.datetime "last_accessed"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.check_constraint "log_type::text = ANY (ARRAY['lettings'::character varying::text, 'sales'::character varying::text])", name: "log_type_check"
+    t.check_constraint "year >= 2000 AND year <= 2099", name: "year_check"
+  end
+
   create_table "data_protection_confirmations", force: :cascade do |t|
     t.bigint "organisation_id"
     t.bigint "data_protection_officer_id"

lib/tasks/log_variable_definitions.rake

@@ -0,0 +1,9 @@
+namespace :data_import do
+  desc "Add CsvVariableDefinition records for each file in the specified directory"
+  task :add_variable_definitions, [:path] => :environment do |_task, args|
+    path = Rails.root.join(args[:path])
+    service = Imports::VariableDefinitionsService.new(path:)
+    service.call
+    Rails.logger.info "CSV Variable Definitions: #{service.count} total records added"
+  end
+end

spec/fixtures/variable_definitions/lettings_download_23_24.csv

@@ -0,0 +1,209 @@
+id,Log ID
+status,Status of log
+duplicate_set_id,ID of a set of duplicate logs
+created_by,User the log is created by
+is_dpo,Is the user in the assigned_to column the data protection officer?
+created_at,Time and date the log was created
+updated_by,User who last updated the log
+updated_at,Time and date the log was last updated
+creation_method,Was the log submitted in-service or via bulk upload?
+old_id,The (internal) ID on the old service
+old_form_id,The ID the users saw on the old service
+collection_start_year,Year collection period opened
+assigned_to,User the log is assigned to
+owning_organisation_name,Which organisation owns this property?
+managing_organisation_name,Which organisation manages this letting?
+needstype,What is the needs type?
+lettype,What is the letting type?
+renewal,Is this letting a renewal?
+startdate,What is the tenancy start date?
+renttype,What is the rent type? (grouped into SR, IR or AR)
+renttype_detail,What is the rent type?
+irproduct,Which type of Intermediate Rent is this letting?
+irproduct_other,Which 'Other' type of Intermediate Rent is this letting?
+lar,Is this a London Affordable Rent letting?
+tenancycode,What is the tenant code?
+propcode,What is the property reference?
+uprn_known,Is the UPRN known?
+uprn,If known, property's UPRN
+uprn_confirmed,We found an address that might be this property. Is this the property address?
+address_line1,Address line 1
+address_line2,Address line 2
+town_or_city,Town or City
+county,County
+postcode_full,Postcode
+is_la_inferred,The internal value to indicate if the LA was inferred from the postcode
+la_label,What is the property's local authority?
+la,Local authority code
+first_time_property_let_as_social_housing,Is this the first time the property has been let as social housing?
+unitletas,What rent product was the property most recently let as?
+rsnvac,What is the reason for the property being vacant?
+newprop,Is this property new to the social rented sector?
+offered,How many times was the property offered between becoming vacant and this letting?
+unittype_gn,What type of unit is the property?
+builtype,Which type of building is the property?
+wchair,Is the property built or adapted to wheelchair-user standards?
+beds,How many bedrooms does the property have?
+voiddate,What is the void date?
+vacdays,Number of days the property was vacant
+void_date_value_check,The following soft validation was confirmed: You told us that the property has been vacant for more than 2 years. This is higher than we would expect.
+majorrepairs,Were any major repairs carried out during the void period?
+mrcdate,What date were any major repairs completed on?
+major_repairs_date_value_check,The following soft validation was confirmed: You told us the property has been vacant for 2 years. This is higher than we would expect.
+joint,Is this a joint tenancy?
+startertenancy,Is this a starter tenancy?
+tenancy,What is the type of tenancy?
+tenancyother,If 'Other', what is the type of tenancy?
+tenancylength,What is the length of the fixed-term tenancy to the nearest year?
+sheltered,Is this letting in sheltered accommodation?
+declaration,Has the tenant seen the MHCLG privacy notice?
+hhmemb,How many people live in the household at this letting?
+pregnancy_value_check,The following soft validation was confirmed: You told us somebody in the household is pregnant. You also told us there are no female tenants living at the property.
+refused,Where household characteristics have a 'Refused' option for some or all of: AGE1-AGE8, SEX1-SEX8, RELAT2-RELAT8, ECSTAT1-ECSTAT8
+hhtype,Type of household 1 = 1 elder; 2 = 2 adults, including elder(s); 3 = 1 adult; 4 = 2 adults; 5 = 1 adult & 1+ children; 6 = 2+ adults & 1+ children; 9 = Other
+totchild,Total number of dependent children in the household (Sum of when RELAT2-8 = C)
+totelder,Total number of elders in household (Sum of when AGE1-8 >= 60)
+totadult,Total number of adults in household
+age1,What is the lead tenant's age?
+retirement_value_check,The following soft validation was confirmed: You told us this person is aged %{age} years and retired. The minimum expected retirement age for %{gender} in England is %{age}.
+sex1,Which of these best describes the lead tenant's gender identity?
+ethnic_group,What is the lead tenant's ethnic group?
+ethnic,Which of these best describes the lead tenant's ethnic background?
+national,What is the lead tenant's nationality?
+ecstat1,Which of these best describes the lead tenant's working situation?
+details_known_2,Are the details of tenant 2 known?
+relat2,What is person 2's relationship to the lead tenant?
+age2,What is person 2's age?
+sex2,Which of these best describes person 2's gender identity?
+ecstat2,Which of these best describes person 2's working situation?
+details_known_3,Are the details of tenant 3 known?
+relat3,What is person 3's relationship to the lead tenant?
+age3,What is person 3's age?
+sex3,Which of these best describes person 3's gender identity?
+ecstat3,Which of these best describes person 3's working situation?
+details_known_4,Are the details of tenant 4 known?
+relat4,What is person 4's relationship to the lead tenant?
+age4,What is person 4's age?
+sex4,Which of these best describes person 4's gender identity?
+ecstat4,Which of these best describes person 4's working situation?
+details_known_5,Are the details of tenant 5 known?
+relat5,What is person 5's relationship to the lead tenant?
+age5,What is person 5's age?
+sex5,Which of these best describes person 5's gender identity?
+ecstat5,Which of these best describes person 5's working situation?
+details_known_6,Are the details of tenant 6 known?
+relat6,What is person 6's relationship to the lead tenant?
+age6,What is person 6's age?
+sex6,Which of these best describes person 6's gender identity?
+ecstat6,Which of these best describes person 6's working situation?
+details_known_7,Are the details of tenant 7 known?
+relat7,What is person 7's relationship to the lead tenant?
+age7,What is person 7's age?
+sex7,Which of these best describes person 7's gender identity?
+ecstat7,Which of these best describes person 7's working situation?
+details_known_8,Are the details of tenant 8 known?
+relat8,What is person 8's relationship to the lead tenant?
+age8,What is person 8's age?
+sex8,Which of these best describes person 8's gender identity?
+ecstat8,Which of these best describes person 8's working situation?
+armedforces,Does anybody in the household have links to the UK armed forces?
+leftreg,Is this person still serving in the UK armed forces?
+reservist,Was this person seriously injured or ill as a result of serving in the UK armed forces?
+preg_occ,Is anybody in the household pregnant?
+housingneeds,Does anybody in the household have any disabled access needs?
+housingneeds_type,What access needs do they have? (Fully wheelchair-accessible housing, Level access housing or Wheelchair access to essential rooms)
+housingneeds_a,Disabled access needs a) Fully wheelchair-accessible housing
+housingneeds_b,Disabled access needs b) Wheelchair access to essential rooms
+housingneeds_c,Disabled access needs c) Level access housing
+housingneeds_f,Disabled access needs f) Other disabled access needs
+housingneeds_g,Disabled access needs g) No disabled access needs
+housingneeds_h,Disabled access needs h) Don't know
+housingneeds_other,Do they have any other disabled access needs?
+illness,Does anybody in the household have a physical or mental health condition (or other illness) expected to last 12 months or more?
+illness_type_4,Does this person's condition affect their dexterity?
+illness_type_5,Does this person's condition affect their learning or understanding or concentrating?
+illness_type_2,Does this person's condition affect their hearing?
+illness_type_6,Does this person's condition affect their memory?
+illness_type_7,Does this person's condition affect their mental health?
+illness_type_3,Does this person's condition affect their mobility?
+illness_type_9,Does this person's condition affect them socially or behaviourally?
+illness_type_8,Does this person's condition affect their stamina or breathing or fatigue?
+illness_type_1,Does this person's condition affect their vision?
+illness_type_10,Does this person's condition affect them in another way?
+layear,How long has the household continuously lived in the local authority area of the new letting?
+waityear,How long has the household been on the local authority waiting list for the new letting?
+reason,What is the tenant's main reason for the household leaving their last settled home?
+reasonother,If 'Other', what was the main reason for leaving their last settled home?
+prevten,Where was the household immediately before this letting?
+homeless,Did the household experience homelessness immediately before this letting?
+ppcodenk,Previous postcode unknown or previous accommodation was temporary
+ppostcode_full,What is the postcode of the household's last settled home?
+previous_la_known,Was the local authority of the household's last settled home known?
+is_previous_la_inferred,The internal value to indicate if the previous LA was inferred from the postcode
+prevloc_label,Previous location LA name
+prevloc,Previous location's ONS LA Code
+reasonpref,Was the household given reasonable preference by the local authority?
+rp_homeless,Reasonable preference reason - They were homeless or about to lose their home (within 56 days)
+rp_insan_unsat,Reasonable preference reason - They were living in insanitary, overcrowded or unisatisfactory housing
+rp_medwel,Reasonable preference reason - They needed to move on medical and welfare reasons (including disability)
+rp_hardship,Reasonable preference reason - They needed to move to avoid hardship to themselves or others
+rp_dontknow,Reasonable preference reason - Don't Know
+cbl,Was the letting made under Choice-Based Lettings (CBL)?
+cap,Was the letting made under the Common Allocation Policy (CAP)?
+chr,Was the letting made under the Common Housing Register (CHR)?
+letting_allocation_none,The letting was not allocated under CBL, CAP, CHR or Accessible Register.
+referral,What was the source of referral for this letting?
+referral_value_check,The following soft validation was confirmed: Are you sure? This is a general needs log, and this referral type is for supported housing.
+net_income_known,Do you know the household's combined income after tax?
+incref,Was the household income refused?
+earnings,How much income does the household have in total?
+incfreq,How often does the household receive income?
+net_income_value_check,Populated when someone hits the soft validation and confirmed in the service
+hb,Is the tenant likely to be receiving any of these housing-related benefits?
+has_benefits,Does the tenant receive housing-related benefits? Yes if hb = Universal Credit housing element or Housing benefit, No if hb =  Don't Know, Neither, Tenant prefers not to say or blank
+benefits,How much of the household's income is from Universal Credit, state pensions or benefits?
+household_charge,Does the household pay rent or other charges for the accommodation?
+nocharge,Does the household pay rent or other charges for the accommodation? - flag for when household_charge is answered no
+period,How often does the household pay rent and other charges?
+is_carehome,Is this accommodation a care home?
+chcharge,If this is a care home, how much does the household pay every [time period]?
+wchchrg,Weekly care home charge
+carehome_charges_value_check,Populated when the soft validation and confirmed in the service
+brent,What is the basic rent?
+wrent,Weekly rent
+rent_value_check,Populated when the soft validation and confirmed in the service
+scharge,What is the service charge?
+wscharge,Weekly service charge
+pscharge,What is the personal service charge?
+wpschrge,Weekly personal service charge
+supcharg,What is the support charge?
+wsupchrg,Weekly support charge
+tcharge,Total charge to the tenant
+wtcharge,Weekly total charge to the tenant
+scharge_value_check,Populated when the soft validation and confirmed in the service
+pscharge_value_check,Populated when the soft validation and confirmed in the service
+supcharg_value_check,Populated when the soft validation and confirmed in the service
+hbrentshortfall,After the household has received any housing-related benefits, will they still need to pay for rent and charges?
+tshortfall_known,Can you estimate the outstanding amount?
+tshortfall,Estimated outstanding amount
+wtshortfall,Weekly total rent shortfall charge for tenant receiving housing benefit
+scheme_code,What scheme does this letting belong to?
+scheme_service_name,From scheme code, we map to the scheme name
+scheme_confidential,Does the scheme contain confidential information?
+SCHTYPE,What is this type of scheme? (Direct access hostel), Foyer, Housing for older people or Other supported housing
+scheme_registered_under_care_act,Is this scheme registered under the Care Standards Act 2000?
+scheme_owning_organisation_name,Which organisation owns the housing stock for this scheme?
+scheme_primary_client_group,What client group is this scheme intended for?
+scheme_has_other_client_group,Does this scheme provide for another client group?
+scheme_secondary_client_group,What is the other client group?
+scheme_support_type,What support does this scheme provide?
+scheme_intended_stay,Intended length of stay
+scheme_created_at,Date scheme was created
+location_code,Which location is this letting for?
+location_postcode,What is the postcode for this location?
+location_name,What is the name of this location?
+location_units,How many units are at this location?
+location_type_of_unit,What is the most common type of unit at this location?
+location_mobility_type,What are the mobility standards for the majority of the units in this location?
+location_local_authority,What is the local authority of this postcode?
+location_startdate,When did the first property in this location become available under this scheme?

spec/fixtures/variable_definitions/lettings_download_24_25.csv

@@ -0,0 +1,220 @@
+id,Log ID
+status,Status of log
+duplicate_set_id,ID of a set of duplicate logs
+created_by,User the log is created by
+is_dpo,Is the user in the assigned_to column the data protection officer?
+created_at,Time and date the log was created
+updated_by,User who last updated the log
+updated_at,Time and date the log was last updated
+creation_method,Was the log submitted in-service or via bulk upload?
+collection_start_year,Year collection period opened
+address_line1_as_entered,Address line 1 entered in bulk upload file
+address_line2_as_entered,Address line 2 entered in bulk upload file
+town_or_city_as_entered,Town or city entered in bulk upload file
+county_as_entered,County entered in bulk upload file
+postcode_full_as_entered,Postcode entered in bulk upload file
+la_as_entered,Local authority entered in bulk upload file
+bulk_upload_id,ID of a set of bulk uploaded logs
+assigned_to,User the log is assigned to
+owning_organisation_name,Which organisation owns this property?
+managing_organisation_name,Which organisation manages this letting?
+needstype,What is the needs type?
+lettype,What is the letting type?
+renewal,Is this letting a renewal?
+startdate,What is the tenancy start date?
+renttype,What is the rent type? (grouped into SR, IR or AR)
+renttype_detail,What is the rent type?
+irproduct,Which type of Intermediate Rent is this letting?
+irproduct_other,Which 'Other' type of Intermediate Rent is this letting?
+lar,Is this a London Affordable Rent letting?
+tenancycode,What is the tenant code?
+propcode,What is the property reference?
+uprn_known,Is the UPRN known?
+uprn,If known, property's UPRN
+address_line1_input,Address line 1 input from address matching feature
+postcode_full_input,Postcode input from address matching feature
+address_search_value_check,Was the 'No address found' page seen?
+uprn_selection,UPRN of the address selected
+address_line1,Address line 1
+address_line2,Address line 2
+town_or_city,Town or City
+county,County
+postcode_full,Postcode
+is_la_inferred,The internal value to indicate if the LA was inferred from the postcode
+la_label,What is the property's local authority?
+la,Local authority code
+first_time_property_let_as_social_housing,Is this the first time the property has been let as social housing?
+unitletas,What rent product was the property most recently let as?
+rsnvac,What is the reason for the property being vacant?
+newprop,Is this property new to the social rented sector?
+unittype_gn,What type of unit is the property?
+builtype,Which type of building is the property?
+wchair,Is the property built or adapted to wheelchair-user standards?
+beds,How many bedrooms does the property have?
+voiddate,What is the void date?
+vacdays,Number of days the property was vacant
+void_date_value_check,The following soft validation was confirmed: You told us that the property has been vacant for more than 2 years. This is higher than we would expect.
+majorrepairs,Were any major repairs carried out during the void period?
+mrcdate,What date were any major repairs completed on?
+major_repairs_date_value_check,The following soft validation was confirmed: You told us the property has been vacant for 2 years. This is higher than we would expect.
+joint,Is this a joint tenancy?
+startertenancy,Is this a starter tenancy?
+tenancy,What is the type of tenancy?
+tenancyother,If 'Other', what is the type of tenancy?
+tenancylength,What is the length of the fixed-term tenancy to the nearest year?
+sheltered,Is this letting in sheltered accommodation?
+declaration,Has the tenant seen the MHCLG privacy notice?
+hhmemb,How many people live in the household at this letting?
+pregnancy_value_check,The following soft validation was confirmed: You told us somebody in the household is pregnant. You also told us there are no female tenants living at the property.
+refused,Where household characteristics have a 'Refused' option for some or all of: AGE1-AGE8, SEX1-SEX8, RELAT2-RELAT8, ECSTAT1-ECSTAT8
+hhtype,Type of household 1 = 1 elder; 2 = 2 adults, including elder(s); 3 = 1 adult; 4 = 2 adults; 5 = 1 adult & 1+ children; 6 = 2+ adults & 1+ children; 9 = Other
+totchild,Total number of dependent children in the household (Sum of when RELAT2-8 = C)
+totelder,Total number of elders in household (Sum of when AGE1-8 >= 60)
+totadult,Total number of adults in household
+age1,What is the lead tenant's age?
+retirement_value_check,The following soft validation was confirmed: You told us this person is aged %{age} years and retired. The minimum expected retirement age for %{gender} in England is %{age}.
+sex1,Which of these best describes the lead tenant's gender identity?
+ethnic_group,What is the lead tenant's ethnic group?
+ethnic,Which of these best describes the lead tenant's ethnic background?
+nationality_all,What is the lead tenant's nationality?
+ecstat1,Which of these best describes the lead tenant's working situation?
+details_known_2,Are the details of tenant 2 known?
+relat2,What is person 2's relationship to the lead tenant?
+partner_under_16_value_check,The following soft validation was confirmed: You said that [person X]'s relationship to lead tenant is partner, and that their age is [AGEX]. Are you sure this is correct?
+multiple_partners_value_check,The following soft validation was confirmed: You said that more than one person in the household is the partner of the lead tenant. Are you sure this is correct?
+age2,What is person 2's age?
+sex2,Which of these best describes person 2's gender identity?
+ecstat2,Which of these best describes person 2's working situation?
+details_known_3,Are the details of tenant 3 known?
+relat3,What is person 3's relationship to the lead tenant?
+age3,What is person 3's age?
+sex3,Which of these best describes person 3's gender identity?
+ecstat3,Which of these best describes person 3's working situation?
+details_known_4,Are the details of tenant 4 known?
+relat4,What is person 4's relationship to the lead tenant?
+age4,What is person 4's age?
+sex4,Which of these best describes person 4's gender identity?
+ecstat4,Which of these best describes person 4's working situation?
+details_known_5,Are the details of tenant 5 known?
+relat5,What is person 5's relationship to the lead tenant?
+age5,What is person 5's age?
+sex5,Which of these best describes person 5's gender identity?
+ecstat5,Which of these best describes person 5's working situation?
+details_known_6,Are the details of tenant 6 known?
+relat6,What is person 6's relationship to the lead tenant?
+age6,What is person 6's age?
+sex6,Which of these best describes person 6's gender identity?
+ecstat6,Which of these best describes person 6's working situation?
+details_known_7,Are the details of tenant 7 known?
+relat7,What is person 7's relationship to the lead tenant?
+age7,What is person 7's age?
+sex7,Which of these best describes person 7's gender identity?
+ecstat7,Which of these best describes person 7's working situation?
+details_known_8,Are the details of tenant 8 known?
+relat8,What is person 8's relationship to the lead tenant?
+age8,What is person 8's age?
+sex8,Which of these best describes person 8's gender identity?
+ecstat8,Which of these best describes person 8's working situation?
+armedforces,Does anybody in the household have links to the UK armed forces?
+leftreg,Is this person still serving in the UK armed forces?
+reservist,Was this person seriously injured or ill as a result of serving in the UK armed forces?
+preg_occ,Is anybody in the household pregnant?
+housingneeds,Does anybody in the household have any disabled access needs?
+housingneeds_type,What access needs do they have? (Fully wheelchair-accessible housing, Level access housing or Wheelchair access to essential rooms)
+housingneeds_a,Disabled access needs a) Fully wheelchair-accessible housing
+housingneeds_b,Disabled access needs b) Wheelchair access to essential rooms
+housingneeds_c,Disabled access needs c) Level access housing
+housingneeds_f,Disabled access needs f) Other disabled access needs
+housingneeds_g,Disabled access needs g) No disabled access needs
+housingneeds_h,Disabled access needs h) Don't know
+housingneeds_other,Do they have any other disabled access needs?
+illness,Does anybody in the household have a physical or mental health condition (or other illness) expected to last 12 months or more?
+illness_type_4,Does this person's condition affect their dexterity?
+illness_type_5,Does this person's condition affect their learning or understanding or concentrating?
+illness_type_2,Does this person's condition affect their hearing?
+illness_type_6,Does this person's condition affect their memory?
+illness_type_7,Does this person's condition affect their mental health?
+illness_type_3,Does this person's condition affect their mobility?
+illness_type_9,Does this person's condition affect them socially or behaviourally?
+illness_type_8,Does this person's condition affect their stamina or breathing or fatigue?
+illness_type_1,Does this person's condition affect their vision?
+illness_type_10,Does this person's condition affect them in another way?
+layear,How long has the household continuously lived in the local authority area of the new letting?
+waityear,How long has the household been on the local authority waiting list for the new letting?
+reason,What is the tenant's main reason for the household leaving their last settled home?
+reasonother,If 'Other', what was the main reason for leaving their last settled home?
+reasonother_value_check,The soft validation was confirmed
+prevten,Where was the household immediately before this letting?
+homeless,Did the household experience homelessness immediately before this letting?
+ppcodenk,Previous postcode unknown or previous accommodation was temporary
+ppostcode_full,What is the postcode of the household's last settled home?
+previous_la_known,Was the local authority of the household's last settled home known?
+is_previous_la_inferred,The internal value to indicate if the previous LA was inferred from the postcode
+prevloc_label,Previous location LA name
+prevloc,Previous location's ONS LA Code
+reasonpref,Was the household given reasonable preference by the local authority?
+rp_homeless,Reasonable preference reason - They were homeless or about to lose their home (within 56 days)
+rp_insan_unsat,Reasonable preference reason - They were living in insanitary, overcrowded or unisatisfactory housing
+rp_medwel,Reasonable preference reason - They needed to move on medical and welfare reasons (including disability)
+rp_hardship,Reasonable preference reason - They needed to move to avoid hardship to themselves or others
+rp_dontknow,Reasonable preference reason - Don't Know
+cbl,Was the letting made under Choice-Based Lettings (CBL)?
+cap,Was the letting made under the Common Allocation Policy (CAP)?
+chr,Was the letting made under the Common Housing Register (CHR)?
+accessible_register,Was the letting made under the Accessible Register?
+letting_allocation_none,The letting was not allocated under CBL, CAP, CHR or Accessible Register.
+referral,What was the source of referral for this letting?
+referral_value_check,The following soft validation was confirmed: Are you sure? This is a general needs log, and this referral type is for supported housing.
+net_income_known,Do you know the household's combined income after tax?
+incref,Was the household income refused?
+earnings,How much income does the household have in total?
+incfreq,How often does the household receive income?
+net_income_value_check,Populated when someone hits the soft validation and confirmed in the service
+hb,Is the tenant likely to be receiving any of these housing-related benefits?
+has_benefits,Does the tenant receive housing-related benefits? Yes if hb = Universal Credit housing element or Housing benefit, No if hb =  Don't Know, Neither, Tenant prefers not to say or blank
+benefits,How much of the household's income is from Universal Credit, state pensions or benefits?
+household_charge,Does the household pay rent or other charges for the accommodation?
+nocharge,Does the household pay rent or other charges for the accommodation? - flag for when household_charge is answered no
+period,How often does the household pay rent and other charges?
+is_carehome,Is this accommodation a care home?
+chcharge,If this is a care home, how much does the household pay every [time period]?
+wchchrg,Weekly care home charge
+carehome_charges_value_check,Populated when the soft validation and confirmed in the service
+brent,What is the basic rent?
+wrent,Weekly rent
+rent_value_check,Populated when the soft validation and confirmed in the service
+scharge,What is the service charge?
+wscharge,Weekly service charge
+pscharge,What is the personal service charge?
+wpschrge,Weekly personal service charge
+supcharg,What is the support charge?
+wsupchrg,Weekly support charge
+tcharge,Total charge to the tenant
+wtcharge,Weekly total charge to the tenant
+scharge_value_check,Populated when the soft validation and confirmed in the service
+pscharge_value_check,Populated when the soft validation and confirmed in the service
+supcharg_value_check,Populated when the soft validation and confirmed in the service
+hbrentshortfall,After the household has received any housing-related benefits, will they still need to pay for rent and charges?
+tshortfall_known,Can you estimate the outstanding amount?
+tshortfall,Estimated outstanding amount
+wtshortfall,Weekly total rent shortfall charge for tenant receiving housing benefit
+scheme_code,What scheme does this letting belong to?
+scheme_service_name,From scheme code, we map to the scheme name
+scheme_confidential,Does the scheme contain confidential information?
+SCHTYPE,What is this type of scheme? (Direct access hostel), Foyer, Housing for older people or Other supported housing
+scheme_registered_under_care_act,Is this scheme registered under the Care Standards Act 2000?
+scheme_owning_organisation_name,Which organisation owns the housing stock for this scheme?
+scheme_primary_client_group,What client group is this scheme intended for?
+scheme_has_other_client_group,Does this scheme provide for another client group?
+scheme_secondary_client_group,What is the other client group?
+scheme_support_type,What support does this scheme provide?
+scheme_intended_stay,Intended length of stay
+scheme_created_at,Date scheme was created
+location_code,Which location is this letting for?
+location_postcode,What is the postcode for this location?
+location_name,What is the name of this location?
+location_units,How many units are at this location?
+location_type_of_unit,What is the most common type of unit at this location?
+location_mobility_type,What are the mobility standards for the majority of the units in this location?
+location_local_authority,What is the local authority of this postcode?
+location_startdate,When did the first property in this location become available under this scheme?

spec/fixtures/variable_definitions/sales_download_23_24.csv

@@ -0,0 +1,269 @@
+ID,Log ID
+STATUS,Status of log
+DUPLICATESET,ID of a set of duplicate logs
+CREATEDDATE,Time and date the log was created
+UPLOADDATE,Time and date the log was last updated
+FORM,The ID on the old service
+COLLECTIONYEAR,Year collection period opened
+CREATIONMETHOD,Was the log submitted in-service or via bulk upload?
+DATAPROTECT,Is the user in the created_by column the data protection officer?
+OWNINGORGNAME,Which organisation owned this property before the sale?
+MANINGORGNAME,Which organisation reported the sale?
+CREATEDBY,User that created the log
+USERNAME,User the log is assigned to
+DAY,Day of sale completion date
+MONTH,Month of sale completion date
+YEAR,Year of sale completion date
+PURCHID,What is the purchaser code?
+OWNERSHIP,Was this purchase made through an ownership scheme?
+TYPE,What is the type of shared ownership/discounted ownership/outright sale?
+OTHTYPE,If type = 'Other', what is the type of outright sale?
+COMPANY,Is the buyer a company?
+LIVEINBUYER,Will the buyer(s) live in the property?
+JOINT,Is this a joint purchase?
+JOINTMORE,Are there more than 2 joint buyers of this property?
+BEDS,How many bedrooms does the property have?
+PROPTYPE,What type of unit is the property?
+BUILTYPE,Which type of building is the property?
+UPRN,What is the UPRN of the property?
+UPRNCONFIRMED,We found an address that might be this property. Is this the property address?
+ADDRESS1,Address line 1
+ADDRESS2,Address line 2
+TOWNCITY,Town/City
+COUNTY,County
+PCODE1,Part 1 of the property's postcode
+PCODE2,Part 2 of the property's postcode
+LA,LA code
+LANAME,LA name
+WCHAIR,Is the property built or adapted to wheelchair-user standards?
+NOINT,Did you interview the buyer to answer these questions?
+PRIVACYNOTICE,Has the buyer seen the MHCLG privacy notice?
+AGE1,What is buyer 1's age?
+SEX1,Which of these best describes buyer 1's gender identity?
+ETHNICGROUP1,What is buyer 1's ethnic group?
+ETHNIC,Which of the following best describes buyer 1's ethnic background?
+NATIONAL,What is buyer 1's nationality?
+ECSTAT1,Which of these best describes buyer 1's working situation?
+LIVEINBUYER1,Will buyer 1 live in the property?
+RELAT2,What is buyer 2 or person 2's relationship to buyer 1?
+AGE2,What is buyer 2 or person 2's age?
+SEX2,Which of these best describes buyer 2 or person 2's gender identity?
+ETHNICGROUP2,What is buyer 2's ethnic group?
+ETHNIC2,Which of the following best describes buyer 2's ethnic background?
+NATIONAL2,What is buyer 2's nationality?
+ECSTAT2,What is buyer 2 or person 2's working situation?
+LIVEINBUYER2,Will buyer 2 live in the property?
+HHTYPE,Besides the buyer(s), how many other people live or will live in the property?
+RELAT3,What is person 3's relationship to buyer 1?
+AGE3,What is person 3's age?
+SEX3,What is person 3's gender identity?
+ECSTAT3,What is person 3's working situation?
+RELAT4,What is person 4's relationship to buyer 1?
+AGE4,What is person 4's age?
+SEX4,What is person 4's gender identity?
+ECSTAT4,What is person 4's working situation?
+RELAT5,What is person 5's relationship to buyer 1?
+AGE5,What is person 5's age?
+SEX5,What is person 5's gender identity?
+ECSTAT5,What is person 5's working situation?
+RELAT6,What is person 6's relationship to buyer 1?
+AGE6,What is person 6's age?
+SEX6,What is person 6's gender identity?
+ECSTAT6,What is person 6's working situation?
+PREVTEN,What was buyer 1's previous tenure?
+PPCODENK,Do you know the postcode of buyer 1's last settled accommodation?
+PPOSTC1,Part 1 of postcode of buyer 1's last settled accommodation
+PPOSTC2,Part 2 of postcode of buyer 1's last settled accommodation
+PREVIOUSLAKNOWN,Do you know the local authority of buyer 1's last settled accommodation?
+PREVLOC,The local authority code of buyer 1's last settled accommodation
+PREVLOCNAME,The local authority name of buyer 1's last settled accommodation
+PREGYRHA,Was the buyer registered with their PRP (HA)?
+PREGOTHER,Was the buyer registered with another PRP (HA)?
+PREGLA,Was the buyer registered with the local authority?
+PREGGHB,Was the buyer registered with a Help to Buy agent?
+PREGBLANK,Populated if pregyrha, pregother, pregla and pregghb are blank
+BUY2LIVING,At the time of purchase, was buyer 2 living at the same address as buyer 1?
+PREVTEN2,What was buyer 2's previous tenure?
+HHREGRES,Have any of the buyers ever served as a regular in the UK armed forces?
+HHREGRESSTILL,Is the buyer still serving in the UK armed forces?
+ARMEDFORCESSPOUSE,Are any of the buyers a spouse or civil partner of a UK armed forces regular who died in service within the last 2 years?
+DISABLED,Does anyone in the household consider themselves to have a disability?
+WHEEL,Does anyone in the household use a wheelchair?
+INC1NK,Is buyer 1's annual income known?
+INCOME1,What is buyer 1's annual income?
+INC1MORT,Was buyer 1's income used for a mortgage application?
+INC2NK,Is buyer 1's annual income known?
+INCOME2,What is buyer 2's annual income?
+INC2MORT,Was buyer 2's income used for a mortgage application?
+HB,Were the buyers receiving any of these housing-related benefits immediately before buying this property?
+SAVINGSNK,Is the the total amount the buyers had in savings known?
+SAVINGS,What is the total amount the buyers had in savings before they paid any deposit for the property?
+PREVOWN,Have any of the buyers previously owned a property?
+PREVSHARED,Was the previous property under shared ownership?
+PROPLEN,How long did the buyer(s) live in the property before purchasing it?
+STAIRCASE,Is this a staircasing transaction?
+STAIRBOUGHT,What percentage of the property has been bought in this staircasing transaction?
+STAIROWNED,What percentage of the property do the buyers now own in total?
+STAIRCASETOSALE,Was this transaction part of a back-to-back staircasing transaction to facilitate sale of the home on the open market?
+RESALE,Is this a resale?
+EXDAY,Day of the exchange of contracts
+EXMONTH,Month of the exchange of contracts
+EXYEAR,Year of the exchange of contracts
+HODAY,Day of the practical completion or handover date
+HOMONTH,Month of the practical completion or handover date
+HOYEAR,Year of the practical completion or handover date
+LANOMAGR,Was the household rehoused under a local authority nominations agreement?
+SOCTEN,Was the buyer a private registered provider, housing association or local authority tenant immediately before this sale?
+FROMBEDS,How many bedrooms did the buyer's previous property have?
+FROMPROP,What was the previous property type?
+SOCPREVTEN,What was the rent type of buyer's previous tenure?
+VALUE,What is the full purchase price?
+VALUE_VALUE_CHECK,Populated if a soft validation is confirmed.
+EQUITY,What was the initial percentage equity stake purchased?
+MORTGAGEUSED,Was a mortgage used to buy this property?
+MORTGAGE,What is the mortgage amount?
+MORTGAGELENDER,What is the name of the mortgage lender?
+MORTGAGELENDEROTHER,If mortgagelender = 'Other', what is the name of the mortgage lender?
+MORTLEN1,What is the length of the mortgage in years?
+EXTRABOR,Does this include any extra borrowing?
+DEPOSIT,How much was the cash deposit paid on the property?
+CASHDIS,How much cash discount was given through Social Homebuy?
+MRENT,What is the basic monthly rent?
+HASMSCHARGE,Does the property have any monthly leasehold charges?
+MSCHARGE,What are the total monthly leasehold charges for the property?
+MSCHARGE_VALUE_CHECK,Populated if a soft validation is confirmed.
+DISCOUNT,What was the percentage discount?
+GRANT,What was the amount of any loan, grant, discount or subsidy given?
+id,Log ID
+status,Status of log
+duplicate_set_id,ID of a set of duplicate logs
+created_at,Time and date the log was created
+updated_at,Time and date the log was last updated
+old_form_id,The ID on the old service
+collection_start_year,Year collection period opened
+creation_method,Was the log submitted in-service or via bulk upload?
+is_dpo,Is the user in the assigned_to column the data protection officer?
+owning_organisation_name,Which organisation owned this property before the sale?
+managing_organisation_name,Which organisation reported the sale?
+assigned_to,User the log is assigned to
+day,Day of sale completion date
+month,Month of sale completion date
+year,Year of sale completion date
+purchid,What is the purchaser code?
+ownershipsch,Was this purchase made through an ownership scheme?
+type,What is the type of shared ownership/discounted ownership/outright sale?
+othtype,If type = 'Other', what is the type of outright sale?
+companybuy,Is the buyer a company?
+buylivein,Will the buyer(s) live in the property?
+jointpur,Is this a joint purchase?
+jointmore,Are there more than 2 joint buyers of this property?
+beds,How many bedrooms does the property have?
+proptype,What type of unit is the property?
+builtype,Which type of building is the property?
+uprn,What is the UPRN of the property?
+uprn_confirmed,We found an address that might be this property. Is this the property address?
+address_line1,Address line 1
+address_line2,Address line 2
+town_or_city,Town/City
+county,County
+pcode1,Part 1 of the property's postcode
+pcode2,Part 2 of the property's postcode
+la,LA code
+la_label,LA name
+wchair,Is the property built or adapted to wheelchair-user standards?
+noint,Did you interview the buyer to answer these questions?
+privacynotice,Has the buyer seen the MHCLG privacy notice?
+age1,What is buyer 1's age?
+sex1,Which of these best describes buyer 1's gender identity?
+ethnic_group,What is buyer 1's ethnic group?
+ethnic,Which of the following best describes buyer 1's ethnic background?
+national,What is buyer 1's nationality?
+ecstat1,Which of these best describes buyer 1's working situation?
+buy1livein,Will buyer 1 live in the property?
+relat2,What is buyer 2 or person 2's relationship to buyer 1?
+age2,What is buyer 2 or person 2's age?
+sex2,Which of these best describes buyer 2 or person 2's gender identity?
+ethnic_group2,What is buyer 2's ethnic group?
+ethnicbuy2,Which of the following best describes buyer 2's ethnic background?
+nationalbuy2,What is buyer 2's nationality?
+ecstat2,What is buyer 2 or person 2's working situation?
+buy2livein,Will buyer 2 live in the property?
+hholdcount,Besides the buyer(s), how many other people live or will live in the property?
+relat3,What is person 3's relationship to buyer 1?
+age3,What is person 3's age?
+sex3,What is person 3's gender identity?
+ecstat3,What is person 3's working situation?
+relat4,What is person 4's relationship to buyer 1?
+age4,What is person 4's age?
+sex4,What is person 4's gender identity?
+ecstat4,What is person 4's working situation?
+relat5,What is person 5's relationship to buyer 1?
+age5,What is person 5's age?
+sex5,What is person 5's gender identity?
+ecstat5,What is person 5's working situation?
+relat6,What is person 6's relationship to buyer 1?
+age6,What is person 6's age?
+sex6,What is person 6's gender identity?
+ecstat6,What is person 6's working situation?
+prevten,What was buyer 1's previous tenure?
+ppcodenk,Do you know the postcode of buyer 1's last settled accommodation?
+ppostc1,Part 1 of postcode of buyer 1's last settled accommodation
+ppostc2,Part 2 of postcode of buyer 1's last settled accommodation
+previous_la_known,Do you know the local authority of buyer 1's last settled accommodation?
+prevloc,The local authority code of buyer 1's last settled accommodation
+prevloc_label,The local authority name of buyer 1's last settled accommodation
+pregyrha,Was the buyer registered with their PRP (HA)?
+pregother,Was the buyer registered with another PRP (HA)?
+pregla,Was the buyer registered with the local authority?
+pregghb,Was the buyer registered with a Help to Buy agent?
+pregblank,Populated if pregyrha, pregother, pregla and pregghb are blank
+buy2living,At the time of purchase, was buyer 2 living at the same address as buyer 1?
+prevtenbuy2,What was buyer 2's previous tenure?
+hhregres,Have any of the buyers ever served as a regular in the UK armed forces?
+hhregresstill,Is the buyer still serving in the UK armed forces?
+armedforcesspouse,Are any of the buyers a spouse or civil partner of a UK armed forces regular who died in service within the last 2 years?
+disabled,Does anyone in the household consider themselves to have a disability?
+wheel,Does anyone in the household use a wheelchair?
+income1nk,Is buyer 1's annual income known?
+income1,What is buyer 1's annual income?
+inc1mort,Was buyer 1's income used for a mortgage application?
+income2nk,Is buyer 2's annual income known?
+income2,What is buyer 2's annual income?
+inc2mort,Was buyer 2's income used for a mortgage application?
+hb,Were the buyers receiving any of these housing-related benefits immediately before buying this property?
+savingsnk,Is the the total amount the buyers had in savings known?
+savings,What is the total amount the buyers had in savings before they paid any deposit for the property?
+prevown,Have any of the buyers previously owned a property?
+prevshared,Was the previous property under shared ownership?
+proplen,How long did the buyer(s) live in the property before purchasing it?
+staircase,Is this a staircasing transaction?
+stairbought,What percentage of the property has been bought in this staircasing transaction?
+stairowned,What percentage of the property do the buyers now own in total?
+staircasesale,Was this transaction part of a back-to-back staircasing transaction to facilitate sale of the home on the open market?
+resale,Is this a resale?
+exday,Day of the exchange of contracts
+exmonth,Month of the exchange of contracts
+exyear,Year of the exchange of contracts
+hoday,Day of the practical completion or handover date
+homonth,Month of the practical completion or handover date
+hoyear,Year of the practical completion or handover date
+lanomagr,Was the household rehoused under a local authority nominations agreement?
+soctenant,Was the buyer a private registered provider, housing association or local authority tenant immediately before this sale?
+frombeds,How many bedrooms did the buyer's previous property have?
+fromprop,What was the previous property type?
+socprevten,What was the rent type of buyer's previous tenure?
+value,What is the full purchase price?
+equity,What was the initial percentage equity stake purchased?
+mortgageused,Was a mortgage used to buy this property?
+mortgage,What is the mortgage amount?
+mortgagelender,What is the name of the mortgage lender?
+mortgagelenderother,If mortgagelender = 'Other', what is the name of the mortgage lender?
+mortlen,What is the length of the mortgage in years?
+extrabor,Does this include any extra borrowing?
+deposit,How much was the cash deposit paid on the property?
+cashdis,How much cash discount was given through Social Homebuy?
+mrent,What is the basic monthly rent?
+has_mscharge,Does the property have any monthly leasehold charges?
+mscharge,What are the total monthly leasehold charges for the property?
+discount,What was the percentage discount?
+grant,What was the amount of any loan, grant, discount or subsidy given?

spec/fixtures/variable_definitions/sales_download_24_25.csv

@@ -0,0 +1,281 @@
+ID,Log ID
+STATUS,Status of log
+DUPLICATESET,ID of a set of duplicate logs
+CREATEDDATE,Time and date the log was created
+UPLOADDATE,Time and date the log was last updated
+COLLECTIONYEAR,Year collection period opened
+CREATIONMETHOD,Was the log submitted in-service or via bulk upload?
+BULKUPLOADID,ID of a set of bulk uploaded logs
+DATAPROTECT,Is the user in the created_by column the data protection officer?
+OWNINGORGNAME,Which organisation owned this property before the sale?
+MANINGORGNAME,Which organisation reported the sale?
+CREATEDBY,User that created the log
+USERNAME,User the log is assigned to
+DAY,Day of sale completion date
+MONTH,Month of sale completion date
+YEAR,Year of sale completion date
+PURCHID,What is the purchaser code?
+OWNERSHIP,Was this purchase made through an ownership scheme?
+TYPE,What is the type of shared ownership/discounted ownership/outright sale?
+OTHTYPE,If type = 'Other', what is the type of outright sale?
+COMPANY,Is the buyer a company?
+LIVEINBUYER,Will the buyer(s) live in the property?
+JOINT,Is this a joint purchase?
+JOINTMORE,Are there more than 2 joint buyers of this property?
+NOINT,Did you interview the buyer to answer these questions?
+PRIVACYNOTICE,Has the buyer seen the MHCLG privacy notice?
+UPRN,What is the UPRN of the property?
+ADDRESS1,Address line 1
+ADDRESS2,Address line 2
+TOWNCITY,Town/City
+COUNTY,County
+POSTCODE,Postcode
+ISLAINFERRED,The internal value to indicate if the LA was inferred from the postcode
+LANAME,LA name
+LA,LA code
+UPRNSELECTED,UPRN of the address selected
+ADDRESS_SEARCH_VALUE_CHECK,Was the 'No address found' page seen?
+ADDRESS1INPUT,Address line 1 input from address matching feature
+POSTCODEINPUT,Postcode input from address matching feature
+BULKADDRESS1,Address line 1 entered in bulk upload file
+BULKADDRESS2,Address line 2 entered in bulk upload file
+BULKTOWNCITY,Town or city entered in bulk upload file
+BULKCOUNTY,County entered in bulk upload file
+BULKPOSTCODE,Postcode entered in bulk upload file
+BULKLA,Local authority entered in bulk upload file
+BEDS,How many bedrooms does the property have?
+PROPTYPE,What type of unit is the property?
+BUILTYPE,Which type of building is the property?
+WCHAIR,Is the property built or adapted to wheelchair-user standards?
+AGE1,What is buyer 1's age?
+SEX1,Which of these best describes buyer 1's gender identity?
+ETHNICGROUP1,What is buyer 1's ethnic group?
+ETHNIC,Which of the following best describes buyer 1's ethnic background?
+NATIONALITYALL1,What is buyer 1's nationality?
+ECSTAT1,Which of these best describes buyer 1's working situation?
+LIVEINBUYER1,Will buyer 1 live in the property?
+RELAT2,What is buyer 2 or person 2's relationship to buyer 1?
+AGE2,What is buyer 2 or person 2's age?
+SEX2,Which of these best describes buyer 2 or person 2's gender identity?
+ETHNICGROUP2,What is buyer 2's ethnic group?
+ETHNIC2,Which of the following best describes buyer 2's ethnic background?
+NATIONALITYALL2,What is buyer 2's nationality?
+ECSTAT2,What is buyer 2 or person 2's working situation?
+LIVEINBUYER2,Will buyer 2 live in the property?
+HHTYPE,Besides the buyer(s), how many other people live or will live in the property?
+RELAT3,What is person 3's relationship to buyer 1?
+AGE3,What is person 3's age?
+SEX3,What is person 3's gender identity?
+ECSTAT3,What is person 3's working situation?
+RELAT4,What is person 4's relationship to buyer 1?
+AGE4,What is person 4's age?
+SEX4,What is person 4's gender identity?
+ECSTAT4,What is person 4's working situation?
+RELAT5,What is person 5's relationship to buyer 1?
+AGE5,What is person 5's age?
+SEX5,What is person 5's gender identity?
+ECSTAT5,What is person 5's working situation?
+RELAT6,What is person 6's relationship to buyer 1?
+AGE6,What is person 6's age?
+SEX6,What is person 6's gender identity?
+ECSTAT6,What is person 6's working situation?
+PREVTEN,What was buyer 1's previous tenure?
+PPCODENK,Do you know the postcode of buyer 1's last settled accommodation?
+PPOSTC1,Part 1 of postcode of buyer 1's last settled accommodation
+PPOSTC2,Part 2 of postcode of buyer 1's last settled accommodation
+PREVIOUSLAKNOWN,Do you know the local authority of buyer 1's last settled accommodation?
+PREVLOC,The local authority code of buyer 1's last settled accommodation
+PREVLOCNAME,The local authority name of buyer 1's last settled accommodation
+PREGYRHA,Was the buyer registered with their PRP (HA)?
+PREGOTHER,Was the buyer registered with another PRP (HA)?
+PREGLA,Was the buyer registered with the local authority?
+PREGGHB,Was the buyer registered with a Help to Buy agent?
+PREGBLANK,Populated if pregyrha, pregother, pregla and pregghb are blank
+BUY2LIVING,At the time of purchase, was buyer 2 living at the same address as buyer 1?
+PREVTEN2,What was buyer 2's previous tenure?
+HHREGRES,Have any of the buyers ever served as a regular in the UK armed forces?
+HHREGRESSTILL,Is the buyer still serving in the UK armed forces?
+ARMEDFORCESSPOUSE,Are any of the buyers a spouse or civil partner of a UK armed forces regular who died in service within the last 2 years?
+DISABLED,Does anyone in the household consider themselves to have a disability?
+WHEEL,Does anyone in the household use a wheelchair?
+INC1NK,Is buyer 1's annual income known?
+INCOME1,What is buyer 1's annual income?
+INC1MORT,Was buyer 1's income used for a mortgage application?
+INC2NK,Is buyer 1's annual income known?
+INCOME2,What is buyer 2's annual income?
+INC2MORT,Was buyer 2's income used for a mortgage application?
+HB,Were the buyers receiving any of these housing-related benefits immediately before buying this property?
+SAVINGSNK,Is the the total amount the buyers had in savings known?
+SAVINGS,What is the total amount the buyers had in savings before they paid any deposit for the property?
+PREVOWN,Have any of the buyers previously owned a property?
+PREVSHARED,Was the previous property under shared ownership?
+PROPLEN,How long did the buyer(s) live in the property before purchasing it?
+STAIRCASE,Is this a staircasing transaction?
+STAIRBOUGHT,What percentage of the property has been bought in this staircasing transaction?
+STAIROWNED,What percentage of the property do the buyers now own in total?
+STAIRCASETOSALE,Was this transaction part of a back-to-back staircasing transaction to facilitate sale of the home on the open market?
+RESALE,Is this a resale?
+EXDAY,Day of the exchange of contracts
+EXMONTH,Month of the exchange of contracts
+EXYEAR,Year of the exchange of contracts
+HODAY,Day of the practical completion or handover date
+HOMONTH,Month of the practical completion or handover date
+HOYEAR,Year of the practical completion or handover date
+LANOMAGR,Was the household rehoused under a local authority nominations agreement?
+SOCTEN,Was the buyer a private registered provider, housing association or local authority tenant immediately before this sale?
+FROMBEDS,How many bedrooms did the buyer's previous property have?
+FROMPROP,What was the previous property type?
+SOCPREVTEN,What was the rent type of buyer's previous tenure?
+VALUE,What is the full purchase price?
+VALUE_VALUE_CHECK,Populated if a soft validation is confirmed.
+EQUITY,What was the initial percentage equity stake purchased?
+MORTGAGEUSED,Was a mortgage used to buy this property?
+MORTGAGE,What is the mortgage amount?
+MORTGAGELENDER,What is the name of the mortgage lender?
+MORTGAGELENDEROTHER,If mortgagelender = 'Other', what is the name of the mortgage lender?
+MORTLEN1,What is the length of the mortgage in years?
+EXTRABOR,Does this include any extra borrowing?
+DEPOSIT,How much was the cash deposit paid on the property?
+CASHDIS,How much cash discount was given through Social Homebuy?
+MRENT,What is the basic monthly rent?
+HASMSCHARGE,Does the property have any monthly leasehold charges?
+MSCHARGE,What are the total monthly leasehold charges for the property?
+MSCHARGE_VALUE_CHECK,Populated if a soft validation is confirmed.
+DISCOUNT,What was the percentage discount?
+GRANT,What was the amount of any loan, grant, discount or subsidy given?
+id,Log ID
+status,Status of log
+duplicate_set_id,ID of a set of duplicate logs
+created_at,Time and date the log was created
+updated_at,Time and date the log was last updated
+old_form_id,The ID on the old service
+collection_start_year,Year collection period opened
+creation_method,Was the log submitted in-service or via bulk upload?
+is_dpo,Is the user in the assigned_to column the data protection officer?
+owning_organisation_name,Which organisation owned this property before the sale?
+managing_organisation_name,Which organisation reported the sale?
+assigned_to,User the log is assigned to
+day,Day of sale completion date
+month,Month of sale completion date
+year,Year of sale completion date
+purchid,What is the purchaser code?
+ownershipsch,Was this purchase made through an ownership scheme?
+type,What is the type of shared ownership/discounted ownership/outright sale?
+othtype,If type = 'Other', what is the type of outright sale?
+companybuy,Is the buyer a company?
+buylivein,Will the buyer(s) live in the property?
+jointpur,Is this a joint purchase?
+jointmore,Are there more than 2 joint buyers of this property?
+beds,How many bedrooms does the property have?
+proptype,What type of unit is the property?
+builtype,Which type of building is the property?
+uprn,What is the UPRN of the property?
+uprn_confirmed,We found an address that might be this property. Is this the property address?
+address_line1_input,Address line 1 input from address matching feature
+postcode_full_input,Postcode input from address matching feature
+uprn_selection,UPRN of the address selected
+address_line1,Address line 1
+address_line2,Address line 2
+town_or_city,Town/City
+county,County
+pcode1,Part 1 of the property's postcode
+pcode2,Part 2 of the property's postcode
+la,LA code
+la_label,LA name
+wchair,Is the property built or adapted to wheelchair-user standards?
+noint,Did you interview the buyer to answer these questions?
+privacynotice,Has the buyer seen the MHCLG privacy notice?
+age1,What is buyer 1's age?
+sex1,Which of these best describes buyer 1's gender identity?
+ethnic_group,What is buyer 1's ethnic group?
+ethnic,Which of the following best describes buyer 1's ethnic background?
+nationality_all,What is buyer 1's nationality?
+ecstat1,Which of these best describes buyer 1's working situation?
+buy1livein,Will buyer 1 live in the property?
+relat2,What is buyer 2 or person 2's relationship to buyer 1?
+age2,What is buyer 2 or person 2's age?
+sex2,Which of these best describes buyer 2 or person 2's gender identity?
+ethnic_group2,What is buyer 2's ethnic group?
+ethnicbuy2,Which of the following best describes buyer 2's ethnic background?
+nationality_all_buyer2,What is buyer 2's nationality?
+ecstat2,What is buyer 2 or person 2's working situation?
+buy2livein,Will buyer 2 live in the property?
+hholdcount,Besides the buyer(s), how many other people live or will live in the property?
+relat3,What is person 3's relationship to buyer 1?
+age3,What is person 3's age?
+sex3,What is person 3's gender identity?
+ecstat3,What is person 3's working situation?
+relat4,What is person 4's relationship to buyer 1?
+age4,What is person 4's age?
+sex4,What is person 4's gender identity?
+ecstat4,What is person 4's working situation?
+relat5,What is person 5's relationship to buyer 1?
+age5,What is person 5's age?
+sex5,What is person 5's gender identity?
+ecstat5,What is person 5's working situation?
+relat6,What is person 6's relationship to buyer 1?
+age6,What is person 6's age?
+sex6,What is person 6's gender identity?
+ecstat6,What is person 6's working situation?
+prevten,What was buyer 1's previous tenure?
+ppcodenk,Do you know the postcode of buyer 1's last settled accommodation?
+ppostc1,Part 1 of postcode of buyer 1's last settled accommodation
+ppostc2,Part 2 of postcode of buyer 1's last settled accommodation
+previous_la_known,Do you know the local authority of buyer 1's last settled accommodation?
+prevloc,The local authority code of buyer 1's last settled accommodation
+prevloc_label,The local authority name of buyer 1's last settled accommodation
+pregyrha,Was the buyer registered with their PRP (HA)?
+pregother,Was the buyer registered with another PRP (HA)?
+pregla,Was the buyer registered with the local authority?
+pregghb,Was the buyer registered with a Help to Buy agent?
+pregblank,Populated if pregyrha, pregother, pregla and pregghb are blank
+buy2living,At the time of purchase, was buyer 2 living at the same address as buyer 1?
+prevtenbuy2,What was buyer 2's previous tenure?
+hhregres,Have any of the buyers ever served as a regular in the UK armed forces?
+hhregresstill,Is the buyer still serving in the UK armed forces?
+armedforcesspouse,Are any of the buyers a spouse or civil partner of a UK armed forces regular who died in service within the last 2 years?
+disabled,Does anyone in the household consider themselves to have a disability?
+wheel,Does anyone in the household use a wheelchair?
+income1nk,Is buyer 1's annual income known?
+income1,What is buyer 1's annual income?
+inc1mort,Was buyer 1's income used for a mortgage application?
+income2nk,Is buyer 2's annual income known?
+income2,What is buyer 2's annual income?
+inc2mort,Was buyer 2's income used for a mortgage application?
+hb,Were the buyers receiving any of these housing-related benefits immediately before buying this property?
+savingsnk,Is the the total amount the buyers had in savings known?
+savings,What is the total amount the buyers had in savings before they paid any deposit for the property?
+prevown,Have any of the buyers previously owned a property?
+prevshared,Was the previous property under shared ownership?
+proplen,How long did the buyer(s) live in the property before purchasing it?
+staircase,Is this a staircasing transaction?
+stairbought,What percentage of the property has been bought in this staircasing transaction?
+stairowned,What percentage of the property do the buyers now own in total?
+staircasesale,Was this transaction part of a back-to-back staircasing transaction to facilitate sale of the home on the open market?
+resale,Is this a resale?
+exday,Day of the exchange of contracts
+exmonth,Month of the exchange of contracts
+exyear,Year of the exchange of contracts
+hoday,Day of the practical completion or handover date
+homonth,Month of the practical completion or handover date
+hoyear,Year of the practical completion or handover date
+lanomagr,Was the household rehoused under a local authority nominations agreement?
+soctenant,Was the buyer a private registered provider, housing association or local authority tenant immediately before this sale?
+frombeds,How many bedrooms did the buyer's previous property have?
+fromprop,What was the previous property type?
+socprevten,What was the rent type of buyer's previous tenure?
+value,What is the full purchase price?
+equity,What was the initial percentage equity stake purchased?
+mortgageused,Was a mortgage used to buy this property?
+mortgage,What is the mortgage amount?
+mortgagelender,What is the name of the mortgage lender?
+mortgagelenderother,If mortgagelender = 'Other', what is the name of the mortgage lender?
+mortlen,What is the length of the mortgage in years?
+extrabor,Does this include any extra borrowing?
+deposit,How much was the cash deposit paid on the property?
+cashdis,How much cash discount was given through Social Homebuy?
+mrent,What is the basic monthly rent?
+has_mscharge,Does the property have any monthly leasehold charges?
+mscharge,What are the total monthly leasehold charges for the property?
+discount,What was the percentage discount?
+grant,What was the amount of any loan, grant, discount or subsidy given?

spec/helpers/filters_helper_spec.rb

@@ -168,7 +168,7 @@ RSpec.describe FiltersHelper do
 
     before do
       FactoryBot.create(:organisation_relationship, parent_organisation:, child_organisation:)
-      FactoryBot.create(:organisation, name: "Other organisation", id: 99)
+      FactoryBot.create(:organisation, name: "Other organisation", id: 9_999_999)
       user.organisation.reload
     end
 
@@ -326,7 +326,7 @@ RSpec.describe FiltersHelper do
 
     before do
       FactoryBot.create(:organisation_relationship, parent_organisation:, child_organisation:)
-      FactoryBot.create(:organisation, name: "Other organisation", id: 99)
+      FactoryBot.create(:organisation, name: "Other organisation", id: 9_999_999)
       user.organisation.reload
     end
 

spec/lib/tasks/log_variable_definitions_spec.rb

@@ -0,0 +1,41 @@
+require "rails_helper"
+require "rake"
+
+RSpec.describe "log_variable_definitions" do
+  describe ":add_variable_definitions", type: :task do
+    subject(:task) { Rake::Task["data_import:add_variable_definitions"] }
+
+    let(:path) { "spec/fixtures/variable_definitions" }
+
+    before do
+      Rake.application.rake_require("tasks/log_variable_definitions")
+      Rake::Task.define_task(:environment)
+      task.reenable
+    end
+
+    it "adds CsvVariableDefinition records from each file in the specified directory" do
+      expect { task.invoke(path) }.to change(CsvVariableDefinition, :count).by(416)
+    end
+
+    it "handles an empty directory without errors" do
+      empty_path = "spec/fixtures/empty_directory"
+      FileUtils.mkdir_p(empty_path)
+      expect { task.invoke(empty_path) }.not_to raise_error
+      expect(CsvVariableDefinition.count).to eq(0)
+    end
+
+    it "does not create duplicate records if run multiple times" do
+      CsvVariableDefinition.delete_all
+      initial_count = CsvVariableDefinition.count
+
+      task.invoke(path)
+      first_run_count = CsvVariableDefinition.count
+
+      task.invoke(path)
+      second_run_count = CsvVariableDefinition.count
+
+      expect(first_run_count).to eq(initial_count + 416)
+      expect(second_run_count).to eq(first_run_count)
+    end
+  end
+end

spec/services/csv/scheme_csv_service_spec.rb

@@ -13,12 +13,14 @@ RSpec.describe Csv::SchemeCsvService do
 
   before do
     Timecop.freeze(fixed_time)
+    Singleton.__init__(FormHandler)
     create(:scheme_deactivation_period, scheme:, deactivation_date: scheme.created_at + 1.year, reactivation_date: scheme.created_at + 2.years)
     create(:location_deactivation_period, location:, deactivation_date: location.created_at + 6.months)
   end
 
   after do
     Timecop.return
+    Singleton.__init__(FormHandler)
   end
 
   it "returns a string" do

spec/services/imports/variable_definitions_service_spec.rb

@@ -0,0 +1,44 @@
+require "rails_helper"
+
+RSpec.describe Imports::VariableDefinitionsService, type: :service do
+  let(:path) { "spec/fixtures/variable_definitions" }
+  let(:service) { described_class.new(path:) }
+
+  describe "#initialize" do
+    it "initializes with the correct path and count" do
+      expect(service.path).to eq(path)
+      expect(service.count).to eq(0)
+    end
+  end
+
+  describe "#call" do
+    before do
+      allow(Dir).to receive(:glob).and_return(%w[lettings_download_23_24.csv lettings_download_24_25.csv sales_download_23_24.csv sales_download_24_25.csv])
+      allow(service).to receive(:process_file)
+    end
+
+    it "processes each file in the directory" do
+      service.call
+      %w[lettings_download_23_24.csv lettings_download_24_25.csv sales_download_23_24.csv sales_download_24_25.csv].each do |file|
+        expect(service).to have_received(:process_file).with(file)
+      end
+    end
+  end
+
+  describe "#process_file" do
+    let(:file) { "spec/fixtures/variable_definitions/lettings_download_23_24.csv" }
+    let(:csv_content) { [["id", "Log ID"], ["status", "Status of log"], ["duplicate_set_id", "ID of a set of duplicate logs"]] }
+
+    before do
+      allow(CSV).to receive(:foreach).and_yield(csv_content[0]).and_yield(csv_content[1]).and_yield(csv_content[2])
+    end
+
+    context "when no existing record" do
+      it "creates new records" do
+        expect {
+          service.send(:process_file, file)
+        }.to change(CsvVariableDefinition, :count).by(3)
+      end
+    end
+  end
+end