CLDC-3241: Part 1 - Explanation of variables on CSVs (#2585)

* Create CsvVariableDefinition migration and model * Add definition files * Add CsvVariableDefinitions to rails admin * Create rake and service * Create tests * Additional fixes * Rollback unwanted changes to schema * Update Gemfile.lock
5 months ago · 4746161488
18 changed files with 2165 additions and 4 deletions
--- a/app/models/csv_variable_definition.rb
+++ b/app/models/csv_variable_definition.rb
@ -0,0 +1,10 @@
 class CsvVariableDefinition < ApplicationRecord
  validates :variable, presence: true
  validates :definition, presence: true
  validates :log_type, presence: true, inclusion: { in: %w[lettings sales] }
  validates :year, presence: true, numericality: { only_integer: true, greater_than_or_equal_to: 2000, less_than_or_equal_to: 2099 }
  attribute :last_accessed, :datetime
  scope :lettings, -> { where(log_type: "lettings") }
  scope :sales, -> { where(log_type: "sales") }
 end
--- a/app/services/imports/variable_definitions_service.rb
+++ b/app/services/imports/variable_definitions_service.rb
@ -0,0 +1,53 @@
 require "csv"
 module Imports
  class VariableDefinitionsService
    attr_reader :path, :count
    def initialize(path:)
      @path = path
      @count = 0
    end
    def call
      files = Dir.glob(File.join(@path, "*.csv"))
      files.each do |file|
        process_file(file)
      end
    end
  private
    def process_file(file)
      file_name = File.basename(file, ".csv")
      parsed_file_name = file_name.split("_")
      log_type = parsed_file_name[0]
      year = "20#{parsed_file_name[2]}".to_i
      records_added = 0
      CSV.foreach(file) do |row|
        next if row.empty?
        variable = row[0].downcase
        definition = row[1..].join(",")
        next if variable.nil? || definition.nil?
        existing_record = CsvVariableDefinition.find_by(variable: variable.strip, definition: definition.strip, log_type:)
        if existing_record.nil?
          CsvVariableDefinition.create!(
            variable: variable.strip,
            definition: definition.strip,
            log_type:,
            year:,
          )
          records_added += 1
        end
      end
      Rails.logger.debug "Added #{records_added} variable/definition records for file: #{file_name}. Duplicates excluded."
      @count += records_added
    end
  end
 end
--- a/config/csv/definitions/lettings_download_23_24.csv
+++ b/config/csv/definitions/lettings_download_23_24.csv
@ -0,0 +1,209 @@
 id,Log ID
 status,Status of log
 duplicate_set_id,ID of a set of duplicate logs
 created_by,User the log is created by
 is_dpo,Is the user in the assigned_to column the data protection officer?
 created_at,Time and date the log was created
 updated_by,User who last updated the log
 updated_at,Time and date the log was last updated
 creation_method,Was the log submitted in-service or via bulk upload?
 old_id,The (internal) ID on the old service
 old_form_id,The ID the users saw on the old service
 collection_start_year,Year collection period opened
 assigned_to,User the log is assigned to
 owning_organisation_name,Which organisation owns this property?
 managing_organisation_name,Which organisation manages this letting?
 needstype,What is the needs type?
 lettype,What is the letting type?
 renewal,Is this letting a renewal?
 startdate,What is the tenancy start date?
 renttype,What is the rent type? (grouped into SR, IR or AR)
 renttype_detail,What is the rent type?
 irproduct,Which type of Intermediate Rent is this letting?
 irproduct_other,Which 'Other' type of Intermediate Rent is this letting?
 lar,Is this a London Affordable Rent letting?
 tenancycode,What is the tenant code?
 propcode,What is the property reference?
 uprn_known,Is the UPRN known?
 uprn,If known, property's UPRN
 uprn_confirmed,We found an address that might be this property. Is this the property address?
 address_line1,Address line 1
 address_line2,Address line 2
 town_or_city,Town or City
 county,County
 postcode_full,Postcode
 is_la_inferred,The internal value to indicate if the LA was inferred from the postcode
 la_label,What is the property's local authority?
 la,Local authority code
 first_time_property_let_as_social_housing,Is this the first time the property has been let as social housing?
 unitletas,What rent product was the property most recently let as?
 rsnvac,What is the reason for the property being vacant?
 newprop,Is this property new to the social rented sector?
 offered,How many times was the property offered between becoming vacant and this letting?
 unittype_gn,What type of unit is the property?
 builtype,Which type of building is the property?
 wchair,Is the property built or adapted to wheelchair-user standards?
 beds,How many bedrooms does the property have?
 voiddate,What is the void date?
 vacdays,Number of days the property was vacant
 void_date_value_check,The following soft validation was confirmed: You told us that the property has been vacant for more than 2 years. This is higher than we would expect.
 majorrepairs,Were any major repairs carried out during the void period?
 mrcdate,What date were any major repairs completed on?
 major_repairs_date_value_check,The following soft validation was confirmed: You told us the property has been vacant for 2 years. This is higher than we would expect.
 joint,Is this a joint tenancy?
 startertenancy,Is this a starter tenancy?
 tenancy,What is the type of tenancy?
 tenancyother,If 'Other', what is the type of tenancy?
 tenancylength,What is the length of the fixed-term tenancy to the nearest year?
 sheltered,Is this letting in sheltered accommodation?
 declaration,Has the tenant seen the MHCLG privacy notice?
 hhmemb,How many people live in the household at this letting?
 pregnancy_value_check,The following soft validation was confirmed: You told us somebody in the household is pregnant. You also told us there are no female tenants living at the property.
 refused,Where household characteristics have a 'Refused' option for some or all of: AGE1-AGE8, SEX1-SEX8, RELAT2-RELAT8, ECSTAT1-ECSTAT8
 hhtype,Type of household 1 = 1 elder; 2 = 2 adults, including elder(s); 3 = 1 adult; 4 = 2 adults; 5 = 1 adult & 1+ children; 6 = 2+ adults & 1+ children; 9 = Other
 totchild,Total number of dependent children in the household (Sum of when RELAT2-8 = C)
 totelder,Total number of elders in household (Sum of when AGE1-8 >= 60)
 totadult,Total number of adults in household
 age1,What is the lead tenant's age?
 retirement_value_check,The following soft validation was confirmed: You told us this person is aged %{age} years and retired. The minimum expected retirement age for %{gender} in England is %{age}.
 sex1,Which of these best describes the lead tenant's gender identity?
 ethnic_group,What is the lead tenant's ethnic group?
 ethnic,Which of these best describes the lead tenant's ethnic background?
 national,What is the lead tenant's nationality?
 ecstat1,Which of these best describes the lead tenant's working situation?
 details_known_2,Are the details of tenant 2 known?
 relat2,What is person 2's relationship to the lead tenant?
 age2,What is person 2's age?
 sex2,Which of these best describes person 2's gender identity?
 ecstat2,Which of these best describes person 2's working situation?
 details_known_3,Are the details of tenant 3 known?
 relat3,What is person 3's relationship to the lead tenant?
 age3,What is person 3's age?
 sex3,Which of these best describes person 3's gender identity?
 ecstat3,Which of these best describes person 3's working situation?
 details_known_4,Are the details of tenant 4 known?
 relat4,What is person 4's relationship to the lead tenant?
 age4,What is person 4's age?
 sex4,Which of these best describes person 4's gender identity?
 ecstat4,Which of these best describes person 4's working situation?
 details_known_5,Are the details of tenant 5 known?
 relat5,What is person 5's relationship to the lead tenant?
 age5,What is person 5's age?
 sex5,Which of these best describes person 5's gender identity?
 ecstat5,Which of these best describes person 5's working situation?
 details_known_6,Are the details of tenant 6 known?
 relat6,What is person 6's relationship to the lead tenant?
 age6,What is person 6's age?
 sex6,Which of these best describes person 6's gender identity?
 ecstat6,Which of these best describes person 6's working situation?
 details_known_7,Are the details of tenant 7 known?
 relat7,What is person 7's relationship to the lead tenant?
 age7,What is person 7's age?
 sex7,Which of these best describes person 7's gender identity?
 ecstat7,Which of these best describes person 7's working situation?
 details_known_8,Are the details of tenant 8 known?
 relat8,What is person 8's relationship to the lead tenant?
 age8,What is person 8's age?
 sex8,Which of these best describes person 8's gender identity?
 ecstat8,Which of these best describes person 8's working situation?
 armedforces,Does anybody in the household have links to the UK armed forces?
 leftreg,Is this person still serving in the UK armed forces?
 reservist,Was this person seriously injured or ill as a result of serving in the UK armed forces?
 preg_occ,Is anybody in the household pregnant?
 housingneeds,Does anybody in the household have any disabled access needs?
 housingneeds_type,What access needs do they have? (Fully wheelchair-accessible housing, Level access housing or Wheelchair access to essential rooms)
 housingneeds_a,Disabled access needs a) Fully wheelchair-accessible housing
 housingneeds_b,Disabled access needs b) Wheelchair access to essential rooms
 housingneeds_c,Disabled access needs c) Level access housing
 housingneeds_f,Disabled access needs f) Other disabled access needs
 housingneeds_g,Disabled access needs g) No disabled access needs
 housingneeds_h,Disabled access needs h) Don't know
 housingneeds_other,Do they have any other disabled access needs?
 illness,Does anybody in the household have a physical or mental health condition (or other illness) expected to last 12 months or more?
 illness_type_4,Does this person's condition affect their dexterity?
 illness_type_5,Does this person's condition affect their learning or understanding or concentrating?
 illness_type_2,Does this person's condition affect their hearing?
 illness_type_6,Does this person's condition affect their memory?
 illness_type_7,Does this person's condition affect their mental health?
 illness_type_3,Does this person's condition affect their mobility?
 illness_type_9,Does this person's condition affect them socially or behaviourally?
 illness_type_8,Does this person's condition affect their stamina or breathing or fatigue?
 illness_type_1,Does this person's condition affect their vision?
 illness_type_10,Does this person's condition affect them in another way?
 layear,How long has the household continuously lived in the local authority area of the new letting?
 waityear,How long has the household been on the local authority waiting list for the new letting?
 reason,What is the tenant's main reason for the household leaving their last settled home?
 reasonother,If 'Other', what was the main reason for leaving their last settled home?
 prevten,Where was the household immediately before this letting?
 homeless,Did the household experience homelessness immediately before this letting?
 ppcodenk,Previous postcode unknown or previous accommodation was temporary
 ppostcode_full,What is the postcode of the household's last settled home?
 previous_la_known,Was the local authority of the household's last settled home known?
 is_previous_la_inferred,The internal value to indicate if the previous LA was inferred from the postcode
 prevloc_label,Previous location LA name
 prevloc,Previous location's ONS LA Code
 reasonpref,Was the household given reasonable preference by the local authority?
 rp_homeless,Reasonable preference reason - They were homeless or about to lose their home (within 56 days)
 rp_insan_unsat,Reasonable preference reason - They were living in insanitary, overcrowded or unisatisfactory housing
 rp_medwel,Reasonable preference reason - They needed to move on medical and welfare reasons (including disability)
 rp_hardship,Reasonable preference reason - They needed to move to avoid hardship to themselves or others
 rp_dontknow,Reasonable preference reason - Don't Know
 cbl,Was the letting made under Choice-Based Lettings (CBL)?
 cap,Was the letting made under the Common Allocation Policy (CAP)?
 chr,Was the letting made under the Common Housing Register (CHR)?
 letting_allocation_none,The letting was not allocated under CBL, CAP, CHR or Accessible Register.
 referral,What was the source of referral for this letting?
 referral_value_check,The following soft validation was confirmed: Are you sure? This is a general needs log, and this referral type is for supported housing.
 net_income_known,Do you know the household's combined income after tax?
 incref,Was the household income refused?
 earnings,How much income does the household have in total?
 incfreq,How often does the household receive income?
 net_income_value_check,Populated when someone hits the soft validation and confirmed in the service
 hb,Is the tenant likely to be receiving any of these housing-related benefits?
 has_benefits,Does the tenant receive housing-related benefits? Yes if hb = Universal Credit housing element or Housing benefit, No if hb =  Don't Know, Neither, Tenant prefers not to say or blank
 benefits,How much of the household's income is from Universal Credit, state pensions or benefits?
 household_charge,Does the household pay rent or other charges for the accommodation?
 nocharge,Does the household pay rent or other charges for the accommodation? - flag for when household_charge is answered no
 period,How often does the household pay rent and other charges?
 is_carehome,Is this accommodation a care home?
 chcharge,If this is a care home, how much does the household pay every [time period]?
 wchchrg,Weekly care home charge
 carehome_charges_value_check,Populated when the soft validation and confirmed in the service
 brent,What is the basic rent?
 wrent,Weekly rent
 rent_value_check,Populated when the soft validation and confirmed in the service
 scharge,What is the service charge?
 wscharge,Weekly service charge
 pscharge,What is the personal service charge?
 wpschrge,Weekly personal service charge
 supcharg,What is the support charge?
 wsupchrg,Weekly support charge
 tcharge,Total charge to the tenant
 wtcharge,Weekly total charge to the tenant
 scharge_value_check,Populated when the soft validation and confirmed in the service
 pscharge_value_check,Populated when the soft validation and confirmed in the service
 supcharg_value_check,Populated when the soft validation and confirmed in the service
 hbrentshortfall,After the household has received any housing-related benefits, will they still need to pay for rent and charges?
 tshortfall_known,Can you estimate the outstanding amount?
 tshortfall,Estimated outstanding amount
 wtshortfall,Weekly total rent shortfall charge for tenant receiving housing benefit
 scheme_code,What scheme does this letting belong to?
 scheme_service_name,From scheme code, we map to the scheme name
 scheme_confidential,Does the scheme contain confidential information?
 SCHTYPE,What is this type of scheme? (Direct access hostel), Foyer, Housing for older people or Other supported housing
 scheme_registered_under_care_act,Is this scheme registered under the Care Standards Act 2000?
 scheme_owning_organisation_name,Which organisation owns the housing stock for this scheme?
 scheme_primary_client_group,What client group is this scheme intended for?
 scheme_has_other_client_group,Does this scheme provide for another client group?
 scheme_secondary_client_group,What is the other client group?
 scheme_support_type,What support does this scheme provide?
 scheme_intended_stay,Intended length of stay
 scheme_created_at,Date scheme was created
 location_code,Which location is this letting for?
 location_postcode,What is the postcode for this location?
 location_name,What is the name of this location?
 location_units,How many units are at this location?
 location_type_of_unit,What is the most common type of unit at this location?
 location_mobility_type,What are the mobility standards for the majority of the units in this location?
 location_local_authority,What is the local authority of this postcode?
 location_startdate,When did the first property in this location become available under this scheme?
--- a/config/csv/definitions/lettings_download_24_25.csv
+++ b/config/csv/definitions/lettings_download_24_25.csv
@ -0,0 +1,220 @@
 id,Log ID
 status,Status of log
 duplicate_set_id,ID of a set of duplicate logs
 created_by,User the log is created by
 is_dpo,Is the user in the assigned_to column the data protection officer?
 created_at,Time and date the log was created
 updated_by,User who last updated the log
 updated_at,Time and date the log was last updated
 creation_method,Was the log submitted in-service or via bulk upload?
 collection_start_year,Year collection period opened
 address_line1_as_entered,Address line 1 entered in bulk upload file
 address_line2_as_entered,Address line 2 entered in bulk upload file
 town_or_city_as_entered,Town or city entered in bulk upload file
 county_as_entered,County entered in bulk upload file
 postcode_full_as_entered,Postcode entered in bulk upload file
 la_as_entered,Local authority entered in bulk upload file
 bulk_upload_id,ID of a set of bulk uploaded logs
 assigned_to,User the log is assigned to
 owning_organisation_name,Which organisation owns this property?
 managing_organisation_name,Which organisation manages this letting?
 needstype,What is the needs type?
 lettype,What is the letting type?
 renewal,Is this letting a renewal?
 startdate,What is the tenancy start date?
 renttype,What is the rent type? (grouped into SR, IR or AR)
 renttype_detail,What is the rent type?
 irproduct,Which type of Intermediate Rent is this letting?
 irproduct_other,Which 'Other' type of Intermediate Rent is this letting?
 lar,Is this a London Affordable Rent letting?
 tenancycode,What is the tenant code?
 propcode,What is the property reference?
 uprn_known,Is the UPRN known?
 uprn,If known, property's UPRN
 address_line1_input,Address line 1 input from address matching feature
 postcode_full_input,Postcode input from address matching feature
 address_search_value_check,Was the 'No address found' page seen?
 uprn_selection,UPRN of the address selected
 address_line1,Address line 1
 address_line2,Address line 2
 town_or_city,Town or City
 county,County
 postcode_full,Postcode
 is_la_inferred,The internal value to indicate if the LA was inferred from the postcode
 la_label,What is the property's local authority?
 la,Local authority code
 first_time_property_let_as_social_housing,Is this the first time the property has been let as social housing?
 unitletas,What rent product was the property most recently let as?
 rsnvac,What is the reason for the property being vacant?
 newprop,Is this property new to the social rented sector?
 unittype_gn,What type of unit is the property?
 builtype,Which type of building is the property?
 wchair,Is the property built or adapted to wheelchair-user standards?
 beds,How many bedrooms does the property have?
 voiddate,What is the void date?
 vacdays,Number of days the property was vacant
 void_date_value_check,The following soft validation was confirmed: You told us that the property has been vacant for more than 2 years. This is higher than we would expect.
 majorrepairs,Were any major repairs carried out during the void period?
 mrcdate,What date were any major repairs completed on?
 major_repairs_date_value_check,The following soft validation was confirmed: You told us the property has been vacant for 2 years. This is higher than we would expect.
 joint,Is this a joint tenancy?
 startertenancy,Is this a starter tenancy?
 tenancy,What is the type of tenancy?
 tenancyother,If 'Other', what is the type of tenancy?
 tenancylength,What is the length of the fixed-term tenancy to the nearest year?
 sheltered,Is this letting in sheltered accommodation?
 declaration,Has the tenant seen the MHCLG privacy notice?
 hhmemb,How many people live in the household at this letting?
 pregnancy_value_check,The following soft validation was confirmed: You told us somebody in the household is pregnant. You also told us there are no female tenants living at the property.
 refused,Where household characteristics have a 'Refused' option for some or all of: AGE1-AGE8, SEX1-SEX8, RELAT2-RELAT8, ECSTAT1-ECSTAT8
 hhtype,Type of household 1 = 1 elder; 2 = 2 adults, including elder(s); 3 = 1 adult; 4 = 2 adults; 5 = 1 adult & 1+ children; 6 = 2+ adults & 1+ children; 9 = Other
 totchild,Total number of dependent children in the household (Sum of when RELAT2-8 = C)
 totelder,Total number of elders in household (Sum of when AGE1-8 >= 60)
 totadult,Total number of adults in household
 age1,What is the lead tenant's age?
 retirement_value_check,The following soft validation was confirmed: You told us this person is aged %{age} years and retired. The minimum expected retirement age for %{gender} in England is %{age}.
 sex1,Which of these best describes the lead tenant's gender identity?
 ethnic_group,What is the lead tenant's ethnic group?
 ethnic,Which of these best describes the lead tenant's ethnic background?
 nationality_all,What is the lead tenant's nationality?
 ecstat1,Which of these best describes the lead tenant's working situation?
 details_known_2,Are the details of tenant 2 known?
 relat2,What is person 2's relationship to the lead tenant?
 partner_under_16_value_check,The following soft validation was confirmed: You said that [person X]'s relationship to lead tenant is partner, and that their age is [AGEX]. Are you sure this is correct?
 multiple_partners_value_check,The following soft validation was confirmed: You said that more than one person in the household is the partner of the lead tenant. Are you sure this is correct?
 age2,What is person 2's age?
 sex2,Which of these best describes person 2's gender identity?
 ecstat2,Which of these best describes person 2's working situation?
 details_known_3,Are the details of tenant 3 known?
 relat3,What is person 3's relationship to the lead tenant?
 age3,What is person 3's age?
 sex3,Which of these best describes person 3's gender identity?
 ecstat3,Which of these best describes person 3's working situation?
 details_known_4,Are the details of tenant 4 known?
 relat4,What is person 4's relationship to the lead tenant?
 age4,What is person 4's age?
 sex4,Which of these best describes person 4's gender identity?
 ecstat4,Which of these best describes person 4's working situation?
 details_known_5,Are the details of tenant 5 known?
 relat5,What is person 5's relationship to the lead tenant?
 age5,What is person 5's age?
 sex5,Which of these best describes person 5's gender identity?
 ecstat5,Which of these best describes person 5's working situation?
 details_known_6,Are the details of tenant 6 known?
 relat6,What is person 6's relationship to the lead tenant?
 age6,What is person 6's age?
 sex6,Which of these best describes person 6's gender identity?
 ecstat6,Which of these best describes person 6's working situation?
 details_known_7,Are the details of tenant 7 known?
 relat7,What is person 7's relationship to the lead tenant?
 age7,What is person 7's age?
 sex7,Which of these best describes person 7's gender identity?
 ecstat7,Which of these best describes person 7's working situation?
 details_known_8,Are the details of tenant 8 known?
 relat8,What is person 8's relationship to the lead tenant?
 age8,What is person 8's age?
 sex8,Which of these best describes person 8's gender identity?
 ecstat8,Which of these best describes person 8's working situation?
 armedforces,Does anybody in the household have links to the UK armed forces?
 leftreg,Is this person still serving in the UK armed forces?
 reservist,Was this person seriously injured or ill as a result of serving in the UK armed forces?
 preg_occ,Is anybody in the household pregnant?
 housingneeds,Does anybody in the household have any disabled access needs?
 housingneeds_type,What access needs do they have? (Fully wheelchair-accessible housing, Level access housing or Wheelchair access to essential rooms)
 housingneeds_a,Disabled access needs a) Fully wheelchair-accessible housing
 housingneeds_b,Disabled access needs b) Wheelchair access to essential rooms
 housingneeds_c,Disabled access needs c) Level access housing
 housingneeds_f,Disabled access needs f) Other disabled access needs
 housingneeds_g,Disabled access needs g) No disabled access needs
 housingneeds_h,Disabled access needs h) Don't know
 housingneeds_other,Do they have any other disabled access needs?
 illness,Does anybody in the household have a physical or mental health condition (or other illness) expected to last 12 months or more?
 illness_type_4,Does this person's condition affect their dexterity?
 illness_type_5,Does this person's condition affect their learning or understanding or concentrating?
 illness_type_2,Does this person's condition affect their hearing?
 illness_type_6,Does this person's condition affect their memory?
 illness_type_7,Does this person's condition affect their mental health?
 illness_type_3,Does this person's condition affect their mobility?
 illness_type_9,Does this person's condition affect them socially or behaviourally?
 illness_type_8,Does this person's condition affect their stamina or breathing or fatigue?
 illness_type_1,Does this person's condition affect their vision?
 illness_type_10,Does this person's condition affect them in another way?
 layear,How long has the household continuously lived in the local authority area of the new letting?
 waityear,How long has the household been on the local authority waiting list for the new letting?
 reason,What is the tenant's main reason for the household leaving their last settled home?
 reasonother,If 'Other', what was the main reason for leaving their last settled home?
 reasonother_value_check,The soft validation was confirmed
 prevten,Where was the household immediately before this letting?
 homeless,Did the household experience homelessness immediately before this letting?
 ppcodenk,Previous postcode unknown or previous accommodation was temporary
 ppostcode_full,What is the postcode of the household's last settled home?
 previous_la_known,Was the local authority of the household's last settled home known?
 is_previous_la_inferred,The internal value to indicate if the previous LA was inferred from the postcode
 prevloc_label,Previous location LA name
 prevloc,Previous location's ONS LA Code
 reasonpref,Was the household given reasonable preference by the local authority?
 rp_homeless,Reasonable preference reason - They were homeless or about to lose their home (within 56 days)
 rp_insan_unsat,Reasonable preference reason - They were living in insanitary, overcrowded or unisatisfactory housing
 rp_medwel,Reasonable preference reason - They needed to move on medical and welfare reasons (including disability)
 rp_hardship,Reasonable preference reason - They needed to move to avoid hardship to themselves or others
 rp_dontknow,Reasonable preference reason - Don't Know
 cbl,Was the letting made under Choice-Based Lettings (CBL)?
 cap,Was the letting made under the Common Allocation Policy (CAP)?
 chr,Was the letting made under the Common Housing Register (CHR)?
 accessible_register,Was the letting made under the Accessible Register?
 letting_allocation_none,The letting was not allocated under CBL, CAP, CHR or Accessible Register.
 referral,What was the source of referral for this letting?
 referral_value_check,The following soft validation was confirmed: Are you sure? This is a general needs log, and this referral type is for supported housing.
 net_income_known,Do you know the household's combined income after tax?
 incref,Was the household income refused?
 earnings,How much income does the household have in total?
 incfreq,How often does the household receive income?
 net_income_value_check,Populated when someone hits the soft validation and confirmed in the service
 hb,Is the tenant likely to be receiving any of these housing-related benefits?
 has_benefits,Does the tenant receive housing-related benefits? Yes if hb = Universal Credit housing element or Housing benefit, No if hb =  Don't Know, Neither, Tenant prefers not to say or blank
 benefits,How much of the household's income is from Universal Credit, state pensions or benefits?
 household_charge,Does the household pay rent or other charges for the accommodation?
 nocharge,Does the household pay rent or other charges for the accommodation? - flag for when household_charge is answered no
 period,How often does the household pay rent and other charges?
 is_carehome,Is this accommodation a care home?
 chcharge,If this is a care home, how much does the household pay every [time period]?
 wchchrg,Weekly care home charge
 carehome_charges_value_check,Populated when the soft validation and confirmed in the service
 brent,What is the basic rent?
 wrent,Weekly rent
 rent_value_check,Populated when the soft validation and confirmed in the service
 scharge,What is the service charge?
 wscharge,Weekly service charge
 pscharge,What is the personal service charge?
 wpschrge,Weekly personal service charge
 supcharg,What is the support charge?
 wsupchrg,Weekly support charge
 tcharge,Total charge to the tenant
 wtcharge,Weekly total charge to the tenant
 scharge_value_check,Populated when the soft validation and confirmed in the service
 pscharge_value_check,Populated when the soft validation and confirmed in the service
 supcharg_value_check,Populated when the soft validation and confirmed in the service
 hbrentshortfall,After the household has received any housing-related benefits, will they still need to pay for rent and charges?
 tshortfall_known,Can you estimate the outstanding amount?
 tshortfall,Estimated outstanding amount
 wtshortfall,Weekly total rent shortfall charge for tenant receiving housing benefit
 scheme_code,What scheme does this letting belong to?
 scheme_service_name,From scheme code, we map to the scheme name
 scheme_confidential,Does the scheme contain confidential information?
 SCHTYPE,What is this type of scheme? (Direct access hostel), Foyer, Housing for older people or Other supported housing
 scheme_registered_under_care_act,Is this scheme registered under the Care Standards Act 2000?
 scheme_owning_organisation_name,Which organisation owns the housing stock for this scheme?
 scheme_primary_client_group,What client group is this scheme intended for?
 scheme_has_other_client_group,Does this scheme provide for another client group?
 scheme_secondary_client_group,What is the other client group?
 scheme_support_type,What support does this scheme provide?
 scheme_intended_stay,Intended length of stay
 scheme_created_at,Date scheme was created
 location_code,Which location is this letting for?
 location_postcode,What is the postcode for this location?
 location_name,What is the name of this location?
 location_units,How many units are at this location?
 location_type_of_unit,What is the most common type of unit at this location?
 location_mobility_type,What are the mobility standards for the majority of the units in this location?
 location_local_authority,What is the local authority of this postcode?
 location_startdate,When did the first property in this location become available under this scheme?
--- a/config/csv/definitions/sales_download_23_24.csv
+++ b/config/csv/definitions/sales_download_23_24.csv
@ -0,0 +1,269 @@
 ID,Log ID
 STATUS,Status of log
 DUPLICATESET,ID of a set of duplicate logs
 CREATEDDATE,Time and date the log was created
 UPLOADDATE,Time and date the log was last updated
 FORM,The ID on the old service
 COLLECTIONYEAR,Year collection period opened
 CREATIONMETHOD,Was the log submitted in-service or via bulk upload?
 DATAPROTECT,Is the user in the created_by column the data protection officer?
 OWNINGORGNAME,Which organisation owned this property before the sale?
 MANINGORGNAME,Which organisation reported the sale?
 CREATEDBY,User that created the log
 USERNAME,User the log is assigned to
 DAY,Day of sale completion date
 MONTH,Month of sale completion date
 YEAR,Year of sale completion date
 PURCHID,What is the purchaser code?
 OWNERSHIP,Was this purchase made through an ownership scheme?
 TYPE,What is the type of shared ownership/discounted ownership/outright sale?
 OTHTYPE,If type = 'Other', what is the type of outright sale?
 COMPANY,Is the buyer a company?
 LIVEINBUYER,Will the buyer(s) live in the property?
 JOINT,Is this a joint purchase?
 JOINTMORE,Are there more than 2 joint buyers of this property?
 BEDS,How many bedrooms does the property have?
 PROPTYPE,What type of unit is the property?
 BUILTYPE,Which type of building is the property?
 UPRN,What is the UPRN of the property?
 UPRNCONFIRMED,We found an address that might be this property. Is this the property address?
 ADDRESS1,Address line 1
 ADDRESS2,Address line 2
 TOWNCITY,Town/City
 COUNTY,County
 PCODE1,Part 1 of the property's postcode
 PCODE2,Part 2 of the property's postcode
 LA,LA code
 LANAME,LA name
 WCHAIR,Is the property built or adapted to wheelchair-user standards?
 NOINT,Did you interview the buyer to answer these questions?
 PRIVACYNOTICE,Has the buyer seen the MHCLG privacy notice?
 AGE1,What is buyer 1's age?
 SEX1,Which of these best describes buyer 1's gender identity?
 ETHNICGROUP1,What is buyer 1's ethnic group?
 ETHNIC,Which of the following best describes buyer 1's ethnic background?
 NATIONAL,What is buyer 1's nationality?
 ECSTAT1,Which of these best describes buyer 1's working situation?
 LIVEINBUYER1,Will buyer 1 live in the property?
 RELAT2,What is buyer 2 or person 2's relationship to buyer 1?
 AGE2,What is buyer 2 or person 2's age?
 SEX2,Which of these best describes buyer 2 or person 2's gender identity?
 ETHNICGROUP2,What is buyer 2's ethnic group?
 ETHNIC2,Which of the following best describes buyer 2's ethnic background?
 NATIONAL2,What is buyer 2's nationality?
 ECSTAT2,What is buyer 2 or person 2's working situation?
 LIVEINBUYER2,Will buyer 2 live in the property?
 HHTYPE,Besides the buyer(s), how many other people live or will live in the property?
 RELAT3,What is person 3's relationship to buyer 1?
 AGE3,What is person 3's age?
 SEX3,What is person 3's gender identity?
 ECSTAT3,What is person 3's working situation?
 RELAT4,What is person 4's relationship to buyer 1?
 AGE4,What is person 4's age?
 SEX4,What is person 4's gender identity?
 ECSTAT4,What is person 4's working situation?
 RELAT5,What is person 5's relationship to buyer 1?
 AGE5,What is person 5's age?
 SEX5,What is person 5's gender identity?
 ECSTAT5,What is person 5's working situation?
 RELAT6,What is person 6's relationship to buyer 1?
 AGE6,What is person 6's age?
 SEX6,What is person 6's gender identity?
 ECSTAT6,What is person 6's working situation?
 PREVTEN,What was buyer 1's previous tenure?
 PPCODENK,Do you know the postcode of buyer 1's last settled accommodation?
 PPOSTC1,Part 1 of postcode of buyer 1's last settled accommodation
 PPOSTC2,Part 2 of postcode of buyer 1's last settled accommodation
 PREVIOUSLAKNOWN,Do you know the local authority of buyer 1's last settled accommodation?
 PREVLOC,The local authority code of buyer 1's last settled accommodation
 PREVLOCNAME,The local authority name of buyer 1's last settled accommodation
 PREGYRHA,Was the buyer registered with their PRP (HA)?
 PREGOTHER,Was the buyer registered with another PRP (HA)?
 PREGLA,Was the buyer registered with the local authority?
 PREGGHB,Was the buyer registered with a Help to Buy agent?
 PREGBLANK,Populated if pregyrha, pregother, pregla and pregghb are blank
 BUY2LIVING,At the time of purchase, was buyer 2 living at the same address as buyer 1?
 PREVTEN2,What was buyer 2's previous tenure?
 HHREGRES,Have any of the buyers ever served as a regular in the UK armed forces?
 HHREGRESSTILL,Is the buyer still serving in the UK armed forces?
 ARMEDFORCESSPOUSE,Are any of the buyers a spouse or civil partner of a UK armed forces regular who died in service within the last 2 years?
 DISABLED,Does anyone in the household consider themselves to have a disability?
 WHEEL,Does anyone in the household use a wheelchair?
 INC1NK,Is buyer 1's annual income known?
 INCOME1,What is buyer 1's annual income?
 INC1MORT,Was buyer 1's income used for a mortgage application?
 INC2NK,Is buyer 1's annual income known?
 INCOME2,What is buyer 2's annual income?
 INC2MORT,Was buyer 2's income used for a mortgage application?
 HB,Were the buyers receiving any of these housing-related benefits immediately before buying this property?
 SAVINGSNK,Is the the total amount the buyers had in savings known?
 SAVINGS,What is the total amount the buyers had in savings before they paid any deposit for the property?
 PREVOWN,Have any of the buyers previously owned a property?
 PREVSHARED,Was the previous property under shared ownership?
 PROPLEN,How long did the buyer(s) live in the property before purchasing it?
 STAIRCASE,Is this a staircasing transaction?
 STAIRBOUGHT,What percentage of the property has been bought in this staircasing transaction?
 STAIROWNED,What percentage of the property do the buyers now own in total?
 STAIRCASETOSALE,Was this transaction part of a back-to-back staircasing transaction to facilitate sale of the home on the open market?
 RESALE,Is this a resale?
 EXDAY,Day of the exchange of contracts
 EXMONTH,Month of the exchange of contracts
 EXYEAR,Year of the exchange of contracts
 HODAY,Day of the practical completion or handover date
 HOMONTH,Month of the practical completion or handover date
 HOYEAR,Year of the practical completion or handover date
 LANOMAGR,Was the household rehoused under a local authority nominations agreement?
 SOCTEN,Was the buyer a private registered provider, housing association or local authority tenant immediately before this sale?
 FROMBEDS,How many bedrooms did the buyer's previous property have?
 FROMPROP,What was the previous property type?
 SOCPREVTEN,What was the rent type of buyer's previous tenure?
 VALUE,What is the full purchase price?
 VALUE_VALUE_CHECK,Populated if a soft validation is confirmed.
 EQUITY,What was the initial percentage equity stake purchased?
 MORTGAGEUSED,Was a mortgage used to buy this property?
 MORTGAGE,What is the mortgage amount?
 MORTGAGELENDER,What is the name of the mortgage lender?
 MORTGAGELENDEROTHER,If mortgagelender = 'Other', what is the name of the mortgage lender?
 MORTLEN1,What is the length of the mortgage in years?
 EXTRABOR,Does this include any extra borrowing?
 DEPOSIT,How much was the cash deposit paid on the property?
 CASHDIS,How much cash discount was given through Social Homebuy?
 MRENT,What is the basic monthly rent?
 HASMSCHARGE,Does the property have any monthly leasehold charges?
 MSCHARGE,What are the total monthly leasehold charges for the property?
 MSCHARGE_VALUE_CHECK,Populated if a soft validation is confirmed.
 DISCOUNT,What was the percentage discount?
 GRANT,What was the amount of any loan, grant, discount or subsidy given?
 id,Log ID
 status,Status of log
 duplicate_set_id,ID of a set of duplicate logs
 created_at,Time and date the log was created
 updated_at,Time and date the log was last updated
 old_form_id,The ID on the old service
 collection_start_year,Year collection period opened
 creation_method,Was the log submitted in-service or via bulk upload?
 is_dpo,Is the user in the assigned_to column the data protection officer?
 owning_organisation_name,Which organisation owned this property before the sale?
 managing_organisation_name,Which organisation reported the sale?
 assigned_to,User the log is assigned to
 day,Day of sale completion date
 month,Month of sale completion date
 year,Year of sale completion date
 purchid,What is the purchaser code?
 ownershipsch,Was this purchase made through an ownership scheme?
 type,What is the type of shared ownership/discounted ownership/outright sale?
 othtype,If type = 'Other', what is the type of outright sale?
 companybuy,Is the buyer a company?
 buylivein,Will the buyer(s) live in the property?
 jointpur,Is this a joint purchase?
 jointmore,Are there more than 2 joint buyers of this property?
 beds,How many bedrooms does the property have?
 proptype,What type of unit is the property?
 builtype,Which type of building is the property?
 uprn,What is the UPRN of the property?
 uprn_confirmed,We found an address that might be this property. Is this the property address?
 address_line1,Address line 1
 address_line2,Address line 2
 town_or_city,Town/City
 county,County
 pcode1,Part 1 of the property's postcode
 pcode2,Part 2 of the property's postcode
 la,LA code
 la_label,LA name
 wchair,Is the property built or adapted to wheelchair-user standards?
 noint,Did you interview the buyer to answer these questions?
 privacynotice,Has the buyer seen the MHCLG privacy notice?
 age1,What is buyer 1's age?
 sex1,Which of these best describes buyer 1's gender identity?
 ethnic_group,What is buyer 1's ethnic group?
 ethnic,Which of the following best describes buyer 1's ethnic background?
 national,What is buyer 1's nationality?
 ecstat1,Which of these best describes buyer 1's working situation?
 buy1livein,Will buyer 1 live in the property?
 relat2,What is buyer 2 or person 2's relationship to buyer 1?
 age2,What is buyer 2 or person 2's age?
 sex2,Which of these best describes buyer 2 or person 2's gender identity?
 ethnic_group2,What is buyer 2's ethnic group?
 ethnicbuy2,Which of the following best describes buyer 2's ethnic background?
 nationalbuy2,What is buyer 2's nationality?
 ecstat2,What is buyer 2 or person 2's working situation?
 buy2livein,Will buyer 2 live in the property?
 hholdcount,Besides the buyer(s), how many other people live or will live in the property?
 relat3,What is person 3's relationship to buyer 1?
 age3,What is person 3's age?
 sex3,What is person 3's gender identity?
 ecstat3,What is person 3's working situation?
 relat4,What is person 4's relationship to buyer 1?
 age4,What is person 4's age?
 sex4,What is person 4's gender identity?
 ecstat4,What is person 4's working situation?
 relat5,What is person 5's relationship to buyer 1?
 age5,What is person 5's age?
 sex5,What is person 5's gender identity?
 ecstat5,What is person 5's working situation?
 relat6,What is person 6's relationship to buyer 1?
 age6,What is person 6's age?
 sex6,What is person 6's gender identity?
 ecstat6,What is person 6's working situation?
 prevten,What was buyer 1's previous tenure?
 ppcodenk,Do you know the postcode of buyer 1's last settled accommodation?
 ppostc1,Part 1 of postcode of buyer 1's last settled accommodation
 ppostc2,Part 2 of postcode of buyer 1's last settled accommodation
 previous_la_known,Do you know the local authority of buyer 1's last settled accommodation?
 prevloc,The local authority code of buyer 1's last settled accommodation
 prevloc_label,The local authority name of buyer 1's last settled accommodation
 pregyrha,Was the buyer registered with their PRP (HA)?
 pregother,Was the buyer registered with another PRP (HA)?
 pregla,Was the buyer registered with the local authority?
 pregghb,Was the buyer registered with a Help to Buy agent?
 pregblank,Populated if pregyrha, pregother, pregla and pregghb are blank
 buy2living,At the time of purchase, was buyer 2 living at the same address as buyer 1?
 prevtenbuy2,What was buyer 2's previous tenure?
 hhregres,Have any of the buyers ever served as a regular in the UK armed forces?
 hhregresstill,Is the buyer still serving in the UK armed forces?
 armedforcesspouse,Are any of the buyers a spouse or civil partner of a UK armed forces regular who died in service within the last 2 years?
 disabled,Does anyone in the household consider themselves to have a disability?
 wheel,Does anyone in the household use a wheelchair?
 income1nk,Is buyer 1's annual income known?
 income1,What is buyer 1's annual income?
 inc1mort,Was buyer 1's income used for a mortgage application?
 income2nk,Is buyer 2's annual income known?
 income2,What is buyer 2's annual income?
 inc2mort,Was buyer 2's income used for a mortgage application?
 hb,Were the buyers receiving any of these housing-related benefits immediately before buying this property?
 savingsnk,Is the the total amount the buyers had in savings known?
 savings,What is the total amount the buyers had in savings before they paid any deposit for the property?
 prevown,Have any of the buyers previously owned a property?
 prevshared,Was the previous property under shared ownership?
 proplen,How long did the buyer(s) live in the property before purchasing it?
 staircase,Is this a staircasing transaction?
 stairbought,What percentage of the property has been bought in this staircasing transaction?
 stairowned,What percentage of the property do the buyers now own in total?
 staircasesale,Was this transaction part of a back-to-back staircasing transaction to facilitate sale of the home on the open market?
 resale,Is this a resale?
 exday,Day of the exchange of contracts
 exmonth,Month of the exchange of contracts
 exyear,Year of the exchange of contracts
 hoday,Day of the practical completion or handover date
 homonth,Month of the practical completion or handover date
 hoyear,Year of the practical completion or handover date
 lanomagr,Was the household rehoused under a local authority nominations agreement?
 soctenant,Was the buyer a private registered provider, housing association or local authority tenant immediately before this sale?
 frombeds,How many bedrooms did the buyer's previous property have?
 fromprop,What was the previous property type?
 socprevten,What was the rent type of buyer's previous tenure?
 value,What is the full purchase price?
 equity,What was the initial percentage equity stake purchased?
 mortgageused,Was a mortgage used to buy this property?
 mortgage,What is the mortgage amount?
 mortgagelender,What is the name of the mortgage lender?
 mortgagelenderother,If mortgagelender = 'Other', what is the name of the mortgage lender?
 mortlen,What is the length of the mortgage in years?
 extrabor,Does this include any extra borrowing?
 deposit,How much was the cash deposit paid on the property?
 cashdis,How much cash discount was given through Social Homebuy?
 mrent,What is the basic monthly rent?
 has_mscharge,Does the property have any monthly leasehold charges?
 mscharge,What are the total monthly leasehold charges for the property?
 discount,What was the percentage discount?
 grant,What was the amount of any loan, grant, discount or subsidy given?
--- a/config/csv/definitions/sales_download_24_25.csv
+++ b/config/csv/definitions/sales_download_24_25.csv
@ -0,0 +1,281 @@
 ID,Log ID
 STATUS,Status of log
 DUPLICATESET,ID of a set of duplicate logs
 CREATEDDATE,Time and date the log was created
 UPLOADDATE,Time and date the log was last updated
 COLLECTIONYEAR,Year collection period opened
 CREATIONMETHOD,Was the log submitted in-service or via bulk upload?
 BULKUPLOADID,ID of a set of bulk uploaded logs
 DATAPROTECT,Is the user in the created_by column the data protection officer?
 OWNINGORGNAME,Which organisation owned this property before the sale?
 MANINGORGNAME,Which organisation reported the sale?
 CREATEDBY,User that created the log
 USERNAME,User the log is assigned to
 DAY,Day of sale completion date
 MONTH,Month of sale completion date
 YEAR,Year of sale completion date
 PURCHID,What is the purchaser code?
 OWNERSHIP,Was this purchase made through an ownership scheme?
 TYPE,What is the type of shared ownership/discounted ownership/outright sale?
 OTHTYPE,If type = 'Other', what is the type of outright sale?
 COMPANY,Is the buyer a company?
 LIVEINBUYER,Will the buyer(s) live in the property?
 JOINT,Is this a joint purchase?
 JOINTMORE,Are there more than 2 joint buyers of this property?
 NOINT,Did you interview the buyer to answer these questions?
 PRIVACYNOTICE,Has the buyer seen the MHCLG privacy notice?
 UPRN,What is the UPRN of the property?
 ADDRESS1,Address line 1
 ADDRESS2,Address line 2
 TOWNCITY,Town/City
 COUNTY,County
 POSTCODE,Postcode
 ISLAINFERRED,The internal value to indicate if the LA was inferred from the postcode
 LANAME,LA name
 LA,LA code
 UPRNSELECTED,UPRN of the address selected
 ADDRESS_SEARCH_VALUE_CHECK,Was the 'No address found' page seen?
 ADDRESS1INPUT,Address line 1 input from address matching feature
 POSTCODEINPUT,Postcode input from address matching feature
 BULKADDRESS1,Address line 1 entered in bulk upload file
 BULKADDRESS2,Address line 2 entered in bulk upload file
 BULKTOWNCITY,Town or city entered in bulk upload file
 BULKCOUNTY,County entered in bulk upload file
 BULKPOSTCODE,Postcode entered in bulk upload file
 BULKLA,Local authority entered in bulk upload file
 BEDS,How many bedrooms does the property have?
 PROPTYPE,What type of unit is the property?
 BUILTYPE,Which type of building is the property?
 WCHAIR,Is the property built or adapted to wheelchair-user standards?
 AGE1,What is buyer 1's age?
 SEX1,Which of these best describes buyer 1's gender identity?
 ETHNICGROUP1,What is buyer 1's ethnic group?
 ETHNIC,Which of the following best describes buyer 1's ethnic background?
 NATIONALITYALL1,What is buyer 1's nationality?
 ECSTAT1,Which of these best describes buyer 1's working situation?
 LIVEINBUYER1,Will buyer 1 live in the property?
 RELAT2,What is buyer 2 or person 2's relationship to buyer 1?
 AGE2,What is buyer 2 or person 2's age?
 SEX2,Which of these best describes buyer 2 or person 2's gender identity?
 ETHNICGROUP2,What is buyer 2's ethnic group?
 ETHNIC2,Which of the following best describes buyer 2's ethnic background?
 NATIONALITYALL2,What is buyer 2's nationality?
 ECSTAT2,What is buyer 2 or person 2's working situation?
 LIVEINBUYER2,Will buyer 2 live in the property?
 HHTYPE,Besides the buyer(s), how many other people live or will live in the property?
 RELAT3,What is person 3's relationship to buyer 1?
 AGE3,What is person 3's age?
 SEX3,What is person 3's gender identity?
 ECSTAT3,What is person 3's working situation?
 RELAT4,What is person 4's relationship to buyer 1?
 AGE4,What is person 4's age?
 SEX4,What is person 4's gender identity?
 ECSTAT4,What is person 4's working situation?
 RELAT5,What is person 5's relationship to buyer 1?
 AGE5,What is person 5's age?
 SEX5,What is person 5's gender identity?
 ECSTAT5,What is person 5's working situation?
 RELAT6,What is person 6's relationship to buyer 1?
 AGE6,What is person 6's age?
 SEX6,What is person 6's gender identity?
 ECSTAT6,What is person 6's working situation?
 PREVTEN,What was buyer 1's previous tenure?
 PPCODENK,Do you know the postcode of buyer 1's last settled accommodation?
 PPOSTC1,Part 1 of postcode of buyer 1's last settled accommodation
 PPOSTC2,Part 2 of postcode of buyer 1's last settled accommodation
 PREVIOUSLAKNOWN,Do you know the local authority of buyer 1's last settled accommodation?
 PREVLOC,The local authority code of buyer 1's last settled accommodation
 PREVLOCNAME,The local authority name of buyer 1's last settled accommodation
 PREGYRHA,Was the buyer registered with their PRP (HA)?
 PREGOTHER,Was the buyer registered with another PRP (HA)?
 PREGLA,Was the buyer registered with the local authority?
 PREGGHB,Was the buyer registered with a Help to Buy agent?
 PREGBLANK,Populated if pregyrha, pregother, pregla and pregghb are blank
 BUY2LIVING,At the time of purchase, was buyer 2 living at the same address as buyer 1?
 PREVTEN2,What was buyer 2's previous tenure?
 HHREGRES,Have any of the buyers ever served as a regular in the UK armed forces?
 HHREGRESSTILL,Is the buyer still serving in the UK armed forces?
 ARMEDFORCESSPOUSE,Are any of the buyers a spouse or civil partner of a UK armed forces regular who died in service within the last 2 years?
 DISABLED,Does anyone in the household consider themselves to have a disability?
 WHEEL,Does anyone in the household use a wheelchair?
 INC1NK,Is buyer 1's annual income known?
 INCOME1,What is buyer 1's annual income?
 INC1MORT,Was buyer 1's income used for a mortgage application?
 INC2NK,Is buyer 1's annual income known?
 INCOME2,What is buyer 2's annual income?
 INC2MORT,Was buyer 2's income used for a mortgage application?
 HB,Were the buyers receiving any of these housing-related benefits immediately before buying this property?
 SAVINGSNK,Is the the total amount the buyers had in savings known?
 SAVINGS,What is the total amount the buyers had in savings before they paid any deposit for the property?
 PREVOWN,Have any of the buyers previously owned a property?
 PREVSHARED,Was the previous property under shared ownership?
 PROPLEN,How long did the buyer(s) live in the property before purchasing it?
 STAIRCASE,Is this a staircasing transaction?
 STAIRBOUGHT,What percentage of the property has been bought in this staircasing transaction?
 STAIROWNED,What percentage of the property do the buyers now own in total?
 STAIRCASETOSALE,Was this transaction part of a back-to-back staircasing transaction to facilitate sale of the home on the open market?
 RESALE,Is this a resale?
 EXDAY,Day of the exchange of contracts
 EXMONTH,Month of the exchange of contracts
 EXYEAR,Year of the exchange of contracts
 HODAY,Day of the practical completion or handover date
 HOMONTH,Month of the practical completion or handover date
 HOYEAR,Year of the practical completion or handover date
 LANOMAGR,Was the household rehoused under a local authority nominations agreement?
 SOCTEN,Was the buyer a private registered provider, housing association or local authority tenant immediately before this sale?
 FROMBEDS,How many bedrooms did the buyer's previous property have?
 FROMPROP,What was the previous property type?
 SOCPREVTEN,What was the rent type of buyer's previous tenure?
 VALUE,What is the full purchase price?
 VALUE_VALUE_CHECK,Populated if a soft validation is confirmed.
 EQUITY,What was the initial percentage equity stake purchased?
 MORTGAGEUSED,Was a mortgage used to buy this property?
 MORTGAGE,What is the mortgage amount?
 MORTGAGELENDER,What is the name of the mortgage lender?
 MORTGAGELENDEROTHER,If mortgagelender = 'Other', what is the name of the mortgage lender?
 MORTLEN1,What is the length of the mortgage in years?
 EXTRABOR,Does this include any extra borrowing?
 DEPOSIT,How much was the cash deposit paid on the property?
 CASHDIS,How much cash discount was given through Social Homebuy?
 MRENT,What is the basic monthly rent?
 HASMSCHARGE,Does the property have any monthly leasehold charges?
 MSCHARGE,What are the total monthly leasehold charges for the property?
 MSCHARGE_VALUE_CHECK,Populated if a soft validation is confirmed.
 DISCOUNT,What was the percentage discount?
 GRANT,What was the amount of any loan, grant, discount or subsidy given?
 id,Log ID
 status,Status of log
 duplicate_set_id,ID of a set of duplicate logs
 created_at,Time and date the log was created
 updated_at,Time and date the log was last updated
 old_form_id,The ID on the old service
 collection_start_year,Year collection period opened
 creation_method,Was the log submitted in-service or via bulk upload?
 is_dpo,Is the user in the assigned_to column the data protection officer?
 owning_organisation_name,Which organisation owned this property before the sale?
 managing_organisation_name,Which organisation reported the sale?
 assigned_to,User the log is assigned to
 day,Day of sale completion date
 month,Month of sale completion date
 year,Year of sale completion date
 purchid,What is the purchaser code?
 ownershipsch,Was this purchase made through an ownership scheme?
 type,What is the type of shared ownership/discounted ownership/outright sale?
 othtype,If type = 'Other', what is the type of outright sale?
 companybuy,Is the buyer a company?
 buylivein,Will the buyer(s) live in the property?
 jointpur,Is this a joint purchase?
 jointmore,Are there more than 2 joint buyers of this property?
 beds,How many bedrooms does the property have?
 proptype,What type of unit is the property?
 builtype,Which type of building is the property?
 uprn,What is the UPRN of the property?
 uprn_confirmed,We found an address that might be this property. Is this the property address?
 address_line1_input,Address line 1 input from address matching feature
 postcode_full_input,Postcode input from address matching feature
 uprn_selection,UPRN of the address selected
 address_line1,Address line 1
 address_line2,Address line 2
 town_or_city,Town/City
 county,County
 pcode1,Part 1 of the property's postcode
 pcode2,Part 2 of the property's postcode
 la,LA code
 la_label,LA name
 wchair,Is the property built or adapted to wheelchair-user standards?
 noint,Did you interview the buyer to answer these questions?
 privacynotice,Has the buyer seen the MHCLG privacy notice?
 age1,What is buyer 1's age?
 sex1,Which of these best describes buyer 1's gender identity?
 ethnic_group,What is buyer 1's ethnic group?
 ethnic,Which of the following best describes buyer 1's ethnic background?
 nationality_all,What is buyer 1's nationality?
 ecstat1,Which of these best describes buyer 1's working situation?
 buy1livein,Will buyer 1 live in the property?
 relat2,What is buyer 2 or person 2's relationship to buyer 1?
 age2,What is buyer 2 or person 2's age?
 sex2,Which of these best describes buyer 2 or person 2's gender identity?
 ethnic_group2,What is buyer 2's ethnic group?
 ethnicbuy2,Which of the following best describes buyer 2's ethnic background?
 nationality_all_buyer2,What is buyer 2's nationality?
 ecstat2,What is buyer 2 or person 2's working situation?
 buy2livein,Will buyer 2 live in the property?
 hholdcount,Besides the buyer(s), how many other people live or will live in the property?
 relat3,What is person 3's relationship to buyer 1?
 age3,What is person 3's age?
 sex3,What is person 3's gender identity?
 ecstat3,What is person 3's working situation?
 relat4,What is person 4's relationship to buyer 1?
 age4,What is person 4's age?
 sex4,What is person 4's gender identity?
 ecstat4,What is person 4's working situation?
 relat5,What is person 5's relationship to buyer 1?
 age5,What is person 5's age?
 sex5,What is person 5's gender identity?
 ecstat5,What is person 5's working situation?
 relat6,What is person 6's relationship to buyer 1?
 age6,What is person 6's age?
 sex6,What is person 6's gender identity?
 ecstat6,What is person 6's working situation?
 prevten,What was buyer 1's previous tenure?
 ppcodenk,Do you know the postcode of buyer 1's last settled accommodation?
 ppostc1,Part 1 of postcode of buyer 1's last settled accommodation
 ppostc2,Part 2 of postcode of buyer 1's last settled accommodation
 previous_la_known,Do you know the local authority of buyer 1's last settled accommodation?
 prevloc,The local authority code of buyer 1's last settled accommodation
 prevloc_label,The local authority name of buyer 1's last settled accommodation
 pregyrha,Was the buyer registered with their PRP (HA)?
 pregother,Was the buyer registered with another PRP (HA)?
 pregla,Was the buyer registered with the local authority?
 pregghb,Was the buyer registered with a Help to Buy agent?
 pregblank,Populated if pregyrha, pregother, pregla and pregghb are blank
 buy2living,At the time of purchase, was buyer 2 living at the same address as buyer 1?
 prevtenbuy2,What was buyer 2's previous tenure?
 hhregres,Have any of the buyers ever served as a regular in the UK armed forces?
 hhregresstill,Is the buyer still serving in the UK armed forces?
 armedforcesspouse,Are any of the buyers a spouse or civil partner of a UK armed forces regular who died in service within the last 2 years?
 disabled,Does anyone in the household consider themselves to have a disability?
 wheel,Does anyone in the household use a wheelchair?
 income1nk,Is buyer 1's annual income known?
 income1,What is buyer 1's annual income?
 inc1mort,Was buyer 1's income used for a mortgage application?
 income2nk,Is buyer 2's annual income known?
 income2,What is buyer 2's annual income?
 inc2mort,Was buyer 2's income used for a mortgage application?
 hb,Were the buyers receiving any of these housing-related benefits immediately before buying this property?
 savingsnk,Is the the total amount the buyers had in savings known?
 savings,What is the total amount the buyers had in savings before they paid any deposit for the property?
 prevown,Have any of the buyers previously owned a property?
 prevshared,Was the previous property under shared ownership?
 proplen,How long did the buyer(s) live in the property before purchasing it?
 staircase,Is this a staircasing transaction?
 stairbought,What percentage of the property has been bought in this staircasing transaction?
 stairowned,What percentage of the property do the buyers now own in total?
 staircasesale,Was this transaction part of a back-to-back staircasing transaction to facilitate sale of the home on the open market?
 resale,Is this a resale?
 exday,Day of the exchange of contracts
 exmonth,Month of the exchange of contracts
 exyear,Year of the exchange of contracts
 hoday,Day of the practical completion or handover date
 homonth,Month of the practical completion or handover date
 hoyear,Year of the practical completion or handover date
 lanomagr,Was the household rehoused under a local authority nominations agreement?
 soctenant,Was the buyer a private registered provider, housing association or local authority tenant immediately before this sale?
 frombeds,How many bedrooms did the buyer's previous property have?
 fromprop,What was the previous property type?
 socprevten,What was the rent type of buyer's previous tenure?
 value,What is the full purchase price?
 equity,What was the initial percentage equity stake purchased?
 mortgageused,Was a mortgage used to buy this property?
 mortgage,What is the mortgage amount?
 mortgagelender,What is the name of the mortgage lender?
 mortgagelenderother,If mortgagelender = 'Other', what is the name of the mortgage lender?
 mortlen,What is the length of the mortgage in years?
 extrabor,Does this include any extra borrowing?
 deposit,How much was the cash deposit paid on the property?
 cashdis,How much cash discount was given through Social Homebuy?
 mrent,What is the basic monthly rent?
 has_mscharge,Does the property have any monthly leasehold charges?
 mscharge,What are the total monthly leasehold charges for the property?
 discount,What was the percentage discount?
 grant,What was the amount of any loan, grant, discount or subsidy given?
--- a/config/initializers/rails_admin.rb
+++ b/config/initializers/rails_admin.rb
@ -25,8 +25,25 @@ RailsAdmin.config do |config|
  ## == Gravatar integration ==
  ## To disable Gravatar integration in Navigation Bar set to false
-  # config.show_gravatar = true
+  config.show_gravatar = false
-  config.included_models = %w[LogValidation]
+  config.included_models = %w[LogValidation CsvVariableDefinition]
  config.model "LogValidation" do
    label "Log Validation"
  end
  config.model "CsvVariableDefinition" do
    label "CSV Variable Definition"
    edit do
      exclude_fields :last_accessed
      field :log_type do
        help "Required. Specify the type of log associated with this variable: 'lettings' or 'sales'."
      end
      field :year do
        help "Required. Specify the year this definition should be available from. This definition will be used in subsequent years unless superseded by a newer definition."
      end
    end
  end
  config.actions do
    dashboard                     # mandatory
--- a/db/migrate/20240726152326_create_csv_variable_definitions.rb
+++ b/db/migrate/20240726152326_create_csv_variable_definitions.rb
@ -0,0 +1,15 @@
 class CreateCsvVariableDefinitions < ActiveRecord::Migration[7.0]
  def change
    create_table :csv_variable_definitions do |t|
      t.string :variable, null: false
      t.string :definition, null: false
      t.string :log_type, null: false
      t.integer :year, null: false
      t.datetime :last_accessed
      t.timestamps
    end
    add_check_constraint :csv_variable_definitions, "log_type IN ('lettings', 'sales')", name: "log_type_check"
    add_check_constraint :csv_variable_definitions, "year BETWEEN 2000 AND 2099", name: "year_check"
  end
 end
--- a/db/schema.rb
+++ b/db/schema.rb
@ -46,6 +46,18 @@ ActiveRecord::Schema[7.0].define(version: 2024_08_19_143150) do
    t.index ["user_id"], name: "index_bulk_uploads_on_user_id"
  end
  create_table "csv_variable_definitions", force: :cascade do |t|
    t.string "variable", null: false
    t.string "definition", null: false
    t.string "log_type", null: false
    t.integer "year", null: false
    t.datetime "last_accessed"
    t.datetime "created_at", null: false
    t.datetime "updated_at", null: false
    t.check_constraint "log_type::text = ANY (ARRAY['lettings'::character varying::text, 'sales'::character varying::text])", name: "log_type_check"
    t.check_constraint "year >= 2000 AND year <= 2099", name: "year_check"
  end
  create_table "data_protection_confirmations", force: :cascade do |t|
    t.bigint "organisation_id"
    t.bigint "data_protection_officer_id"
--- a/lib/tasks/log_variable_definitions.rake
+++ b/lib/tasks/log_variable_definitions.rake
@ -0,0 +1,9 @@
 namespace :data_import do
  desc "Add CsvVariableDefinition records for each file in the specified directory"
  task :add_variable_definitions, [:path] => :environment do |_task, args|
    path = Rails.root.join(args[:path])
    service = Imports::VariableDefinitionsService.new(path:)
    service.call
    Rails.logger.info "CSV Variable Definitions: #{service.count} total records added"
  end
 end
--- a/spec/fixtures/variable_definitions/lettings_download_23_24.csv
+++ b/spec/fixtures/variable_definitions/lettings_download_23_24.csv
@ -0,0 +1,209 @@
 id,Log ID
 status,Status of log
 duplicate_set_id,ID of a set of duplicate logs
 created_by,User the log is created by
 is_dpo,Is the user in the assigned_to column the data protection officer?
 created_at,Time and date the log was created
 updated_by,User who last updated the log
 updated_at,Time and date the log was last updated
 creation_method,Was the log submitted in-service or via bulk upload?
 old_id,The (internal) ID on the old service
 old_form_id,The ID the users saw on the old service
 collection_start_year,Year collection period opened
 assigned_to,User the log is assigned to
 owning_organisation_name,Which organisation owns this property?
 managing_organisation_name,Which organisation manages this letting?
 needstype,What is the needs type?
 lettype,What is the letting type?
 renewal,Is this letting a renewal?
 startdate,What is the tenancy start date?
 renttype,What is the rent type? (grouped into SR, IR or AR)
 renttype_detail,What is the rent type?
 irproduct,Which type of Intermediate Rent is this letting?
 irproduct_other,Which 'Other' type of Intermediate Rent is this letting?
 lar,Is this a London Affordable Rent letting?
 tenancycode,What is the tenant code?
 propcode,What is the property reference?
 uprn_known,Is the UPRN known?
 uprn,If known, property's UPRN
 uprn_confirmed,We found an address that might be this property. Is this the property address?
 address_line1,Address line 1
 address_line2,Address line 2
 town_or_city,Town or City
 county,County
 postcode_full,Postcode
 is_la_inferred,The internal value to indicate if the LA was inferred from the postcode
 la_label,What is the property's local authority?
 la,Local authority code
 first_time_property_let_as_social_housing,Is this the first time the property has been let as social housing?
 unitletas,What rent product was the property most recently let as?
 rsnvac,What is the reason for the property being vacant?
 newprop,Is this property new to the social rented sector?
 offered,How many times was the property offered between becoming vacant and this letting?
 unittype_gn,What type of unit is the property?
 builtype,Which type of building is the property?
 wchair,Is the property built or adapted to wheelchair-user standards?
 beds,How many bedrooms does the property have?
 voiddate,What is the void date?
 vacdays,Number of days the property was vacant
 void_date_value_check,The following soft validation was confirmed: You told us that the property has been vacant for more than 2 years. This is higher than we would expect.
 majorrepairs,Were any major repairs carried out during the void period?
 mrcdate,What date were any major repairs completed on?
 major_repairs_date_value_check,The following soft validation was confirmed: You told us the property has been vacant for 2 years. This is higher than we would expect.
 joint,Is this a joint tenancy?
 startertenancy,Is this a starter tenancy?
 tenancy,What is the type of tenancy?
 tenancyother,If 'Other', what is the type of tenancy?
 tenancylength,What is the length of the fixed-term tenancy to the nearest year?
 sheltered,Is this letting in sheltered accommodation?
 declaration,Has the tenant seen the MHCLG privacy notice?
 hhmemb,How many people live in the household at this letting?
 pregnancy_value_check,The following soft validation was confirmed: You told us somebody in the household is pregnant. You also told us there are no female tenants living at the property.
 refused,Where household characteristics have a 'Refused' option for some or all of: AGE1-AGE8, SEX1-SEX8, RELAT2-RELAT8, ECSTAT1-ECSTAT8
 hhtype,Type of household 1 = 1 elder; 2 = 2 adults, including elder(s); 3 = 1 adult; 4 = 2 adults; 5 = 1 adult & 1+ children; 6 = 2+ adults & 1+ children; 9 = Other
 totchild,Total number of dependent children in the household (Sum of when RELAT2-8 = C)
 totelder,Total number of elders in household (Sum of when AGE1-8 >= 60)
 totadult,Total number of adults in household
 age1,What is the lead tenant's age?
 retirement_value_check,The following soft validation was confirmed: You told us this person is aged %{age} years and retired. The minimum expected retirement age for %{gender} in England is %{age}.
 sex1,Which of these best describes the lead tenant's gender identity?
 ethnic_group,What is the lead tenant's ethnic group?
 ethnic,Which of these best describes the lead tenant's ethnic background?
 national,What is the lead tenant's nationality?
 ecstat1,Which of these best describes the lead tenant's working situation?
 details_known_2,Are the details of tenant 2 known?
 relat2,What is person 2's relationship to the lead tenant?
 age2,What is person 2's age?
 sex2,Which of these best describes person 2's gender identity?
 ecstat2,Which of these best describes person 2's working situation?
 details_known_3,Are the details of tenant 3 known?
 relat3,What is person 3's relationship to the lead tenant?
 age3,What is person 3's age?
 sex3,Which of these best describes person 3's gender identity?
 ecstat3,Which of these best describes person 3's working situation?
 details_known_4,Are the details of tenant 4 known?
 relat4,What is person 4's relationship to the lead tenant?
 age4,What is person 4's age?
 sex4,Which of these best describes person 4's gender identity?
 ecstat4,Which of these best describes person 4's working situation?
 details_known_5,Are the details of tenant 5 known?
 relat5,What is person 5's relationship to the lead tenant?
 age5,What is person 5's age?
 sex5,Which of these best describes person 5's gender identity?
 ecstat5,Which of these best describes person 5's working situation?
 details_known_6,Are the details of tenant 6 known?
 relat6,What is person 6's relationship to the lead tenant?
 age6,What is person 6's age?
 sex6,Which of these best describes person 6's gender identity?
 ecstat6,Which of these best describes person 6's working situation?
 details_known_7,Are the details of tenant 7 known?
 relat7,What is person 7's relationship to the lead tenant?
 age7,What is person 7's age?
 sex7,Which of these best describes person 7's gender identity?
 ecstat7,Which of these best describes person 7's working situation?
 details_known_8,Are the details of tenant 8 known?
 relat8,What is person 8's relationship to the lead tenant?
 age8,What is person 8's age?
 sex8,Which of these best describes person 8's gender identity?
 ecstat8,Which of these best describes person 8's working situation?
 armedforces,Does anybody in the household have links to the UK armed forces?
 leftreg,Is this person still serving in the UK armed forces?
 reservist,Was this person seriously injured or ill as a result of serving in the UK armed forces?
 preg_occ,Is anybody in the household pregnant?
 housingneeds,Does anybody in the household have any disabled access needs?
 housingneeds_type,What access needs do they have? (Fully wheelchair-accessible housing, Level access housing or Wheelchair access to essential rooms)
 housingneeds_a,Disabled access needs a) Fully wheelchair-accessible housing
 housingneeds_b,Disabled access needs b) Wheelchair access to essential rooms
 housingneeds_c,Disabled access needs c) Level access housing
 housingneeds_f,Disabled access needs f) Other disabled access needs
 housingneeds_g,Disabled access needs g) No disabled access needs
 housingneeds_h,Disabled access needs h) Don't know
 housingneeds_other,Do they have any other disabled access needs?
 illness,Does anybody in the household have a physical or mental health condition (or other illness) expected to last 12 months or more?
 illness_type_4,Does this person's condition affect their dexterity?
 illness_type_5,Does this person's condition affect their learning or understanding or concentrating?
 illness_type_2,Does this person's condition affect their hearing?
 illness_type_6,Does this person's condition affect their memory?
 illness_type_7,Does this person's condition affect their mental health?
 illness_type_3,Does this person's condition affect their mobility?
 illness_type_9,Does this person's condition affect them socially or behaviourally?
 illness_type_8,Does this person's condition affect their stamina or breathing or fatigue?
 illness_type_1,Does this person's condition affect their vision?
 illness_type_10,Does this person's condition affect them in another way?
 layear,How long has the household continuously lived in the local authority area of the new letting?
 waityear,How long has the household been on the local authority waiting list for the new letting?
 reason,What is the tenant's main reason for the household leaving their last settled home?
 reasonother,If 'Other', what was the main reason for leaving their last settled home?
 prevten,Where was the household immediately before this letting?
 homeless,Did the household experience homelessness immediately before this letting?
 ppcodenk,Previous postcode unknown or previous accommodation was temporary
 ppostcode_full,What is the postcode of the household's last settled home?
 previous_la_known,Was the local authority of the household's last settled home known?
 is_previous_la_inferred,The internal value to indicate if the previous LA was inferred from the postcode
 prevloc_label,Previous location LA name
 prevloc,Previous location's ONS LA Code
 reasonpref,Was the household given reasonable preference by the local authority?
 rp_homeless,Reasonable preference reason - They were homeless or about to lose their home (within 56 days)
 rp_insan_unsat,Reasonable preference reason - They were living in insanitary, overcrowded or unisatisfactory housing
 rp_medwel,Reasonable preference reason - They needed to move on medical and welfare reasons (including disability)
 rp_hardship,Reasonable preference reason - They needed to move to avoid hardship to themselves or others
 rp_dontknow,Reasonable preference reason - Don't Know
 cbl,Was the letting made under Choice-Based Lettings (CBL)?
 cap,Was the letting made under the Common Allocation Policy (CAP)?
 chr,Was the letting made under the Common Housing Register (CHR)?
 letting_allocation_none,The letting was not allocated under CBL, CAP, CHR or Accessible Register.
 referral,What was the source of referral for this letting?
 referral_value_check,The following soft validation was confirmed: Are you sure? This is a general needs log, and this referral type is for supported housing.
 net_income_known,Do you know the household's combined income after tax?
 incref,Was the household income refused?
 earnings,How much income does the household have in total?
 incfreq,How often does the household receive income?
 net_income_value_check,Populated when someone hits the soft validation and confirmed in the service
 hb,Is the tenant likely to be receiving any of these housing-related benefits?
 has_benefits,Does the tenant receive housing-related benefits? Yes if hb = Universal Credit housing element or Housing benefit, No if hb =  Don't Know, Neither, Tenant prefers not to say or blank
 benefits,How much of the household's income is from Universal Credit, state pensions or benefits?
 household_charge,Does the household pay rent or other charges for the accommodation?
 nocharge,Does the household pay rent or other charges for the accommodation? - flag for when household_charge is answered no
 period,How often does the household pay rent and other charges?
 is_carehome,Is this accommodation a care home?
 chcharge,If this is a care home, how much does the household pay every [time period]?
 wchchrg,Weekly care home charge
 carehome_charges_value_check,Populated when the soft validation and confirmed in the service
 brent,What is the basic rent?
 wrent,Weekly rent
 rent_value_check,Populated when the soft validation and confirmed in the service
 scharge,What is the service charge?
 wscharge,Weekly service charge
 pscharge,What is the personal service charge?
 wpschrge,Weekly personal service charge
 supcharg,What is the support charge?
 wsupchrg,Weekly support charge
 tcharge,Total charge to the tenant
 wtcharge,Weekly total charge to the tenant
 scharge_value_check,Populated when the soft validation and confirmed in the service
 pscharge_value_check,Populated when the soft validation and confirmed in the service
 supcharg_value_check,Populated when the soft validation and confirmed in the service
 hbrentshortfall,After the household has received any housing-related benefits, will they still need to pay for rent and charges?
 tshortfall_known,Can you estimate the outstanding amount?
 tshortfall,Estimated outstanding amount
 wtshortfall,Weekly total rent shortfall charge for tenant receiving housing benefit
 scheme_code,What scheme does this letting belong to?
 scheme_service_name,From scheme code, we map to the scheme name
 scheme_confidential,Does the scheme contain confidential information?
 SCHTYPE,What is this type of scheme? (Direct access hostel), Foyer, Housing for older people or Other supported housing
 scheme_registered_under_care_act,Is this scheme registered under the Care Standards Act 2000?
 scheme_owning_organisation_name,Which organisation owns the housing stock for this scheme?
 scheme_primary_client_group,What client group is this scheme intended for?
 scheme_has_other_client_group,Does this scheme provide for another client group?
 scheme_secondary_client_group,What is the other client group?
 scheme_support_type,What support does this scheme provide?
 scheme_intended_stay,Intended length of stay
 scheme_created_at,Date scheme was created
 location_code,Which location is this letting for?
 location_postcode,What is the postcode for this location?
 location_name,What is the name of this location?
 location_units,How many units are at this location?
 location_type_of_unit,What is the most common type of unit at this location?
 location_mobility_type,What are the mobility standards for the majority of the units in this location?
 location_local_authority,What is the local authority of this postcode?
 location_startdate,When did the first property in this location become available under this scheme?
--- a/spec/fixtures/variable_definitions/lettings_download_24_25.csv
+++ b/spec/fixtures/variable_definitions/lettings_download_24_25.csv
@ -0,0 +1,220 @@
 id,Log ID
 status,Status of log
 duplicate_set_id,ID of a set of duplicate logs
 created_by,User the log is created by
 is_dpo,Is the user in the assigned_to column the data protection officer?
 created_at,Time and date the log was created
 updated_by,User who last updated the log
 updated_at,Time and date the log was last updated
 creation_method,Was the log submitted in-service or via bulk upload?
 collection_start_year,Year collection period opened
 address_line1_as_entered,Address line 1 entered in bulk upload file
 address_line2_as_entered,Address line 2 entered in bulk upload file
 town_or_city_as_entered,Town or city entered in bulk upload file
 county_as_entered,County entered in bulk upload file
 postcode_full_as_entered,Postcode entered in bulk upload file
 la_as_entered,Local authority entered in bulk upload file
 bulk_upload_id,ID of a set of bulk uploaded logs
 assigned_to,User the log is assigned to
 owning_organisation_name,Which organisation owns this property?
 managing_organisation_name,Which organisation manages this letting?
 needstype,What is the needs type?
 lettype,What is the letting type?
 renewal,Is this letting a renewal?
 startdate,What is the tenancy start date?
 renttype,What is the rent type? (grouped into SR, IR or AR)
 renttype_detail,What is the rent type?
 irproduct,Which type of Intermediate Rent is this letting?
 irproduct_other,Which 'Other' type of Intermediate Rent is this letting?
 lar,Is this a London Affordable Rent letting?
 tenancycode,What is the tenant code?
 propcode,What is the property reference?
 uprn_known,Is the UPRN known?
 uprn,If known, property's UPRN
 address_line1_input,Address line 1 input from address matching feature
 postcode_full_input,Postcode input from address matching feature
 address_search_value_check,Was the 'No address found' page seen?
 uprn_selection,UPRN of the address selected
 address_line1,Address line 1
 address_line2,Address line 2
 town_or_city,Town or City
 county,County
 postcode_full,Postcode
 is_la_inferred,The internal value to indicate if the LA was inferred from the postcode
 la_label,What is the property's local authority?
 la,Local authority code
 first_time_property_let_as_social_housing,Is this the first time the property has been let as social housing?
 unitletas,What rent product was the property most recently let as?
 rsnvac,What is the reason for the property being vacant?
 newprop,Is this property new to the social rented sector?
 unittype_gn,What type of unit is the property?
 builtype,Which type of building is the property?
 wchair,Is the property built or adapted to wheelchair-user standards?
 beds,How many bedrooms does the property have?
 voiddate,What is the void date?
 vacdays,Number of days the property was vacant
 void_date_value_check,The following soft validation was confirmed: You told us that the property has been vacant for more than 2 years. This is higher than we would expect.
 majorrepairs,Were any major repairs carried out during the void period?
 mrcdate,What date were any major repairs completed on?
 major_repairs_date_value_check,The following soft validation was confirmed: You told us the property has been vacant for 2 years. This is higher than we would expect.
 joint,Is this a joint tenancy?
 startertenancy,Is this a starter tenancy?
 tenancy,What is the type of tenancy?
 tenancyother,If 'Other', what is the type of tenancy?
 tenancylength,What is the length of the fixed-term tenancy to the nearest year?
 sheltered,Is this letting in sheltered accommodation?
 declaration,Has the tenant seen the MHCLG privacy notice?
 hhmemb,How many people live in the household at this letting?
 pregnancy_value_check,The following soft validation was confirmed: You told us somebody in the household is pregnant. You also told us there are no female tenants living at the property.
 refused,Where household characteristics have a 'Refused' option for some or all of: AGE1-AGE8, SEX1-SEX8, RELAT2-RELAT8, ECSTAT1-ECSTAT8
 hhtype,Type of household 1 = 1 elder; 2 = 2 adults, including elder(s); 3 = 1 adult; 4 = 2 adults; 5 = 1 adult & 1+ children; 6 = 2+ adults & 1+ children; 9 = Other
 totchild,Total number of dependent children in the household (Sum of when RELAT2-8 = C)
 totelder,Total number of elders in household (Sum of when AGE1-8 >= 60)
 totadult,Total number of adults in household
 age1,What is the lead tenant's age?
 retirement_value_check,The following soft validation was confirmed: You told us this person is aged %{age} years and retired. The minimum expected retirement age for %{gender} in England is %{age}.
 sex1,Which of these best describes the lead tenant's gender identity?
 ethnic_group,What is the lead tenant's ethnic group?
 ethnic,Which of these best describes the lead tenant's ethnic background?
 nationality_all,What is the lead tenant's nationality?
 ecstat1,Which of these best describes the lead tenant's working situation?
 details_known_2,Are the details of tenant 2 known?
 relat2,What is person 2's relationship to the lead tenant?
 partner_under_16_value_check,The following soft validation was confirmed: You said that [person X]'s relationship to lead tenant is partner, and that their age is [AGEX]. Are you sure this is correct?
 multiple_partners_value_check,The following soft validation was confirmed: You said that more than one person in the household is the partner of the lead tenant. Are you sure this is correct?
 age2,What is person 2's age?
 sex2,Which of these best describes person 2's gender identity?
 ecstat2,Which of these best describes person 2's working situation?
 details_known_3,Are the details of tenant 3 known?
 relat3,What is person 3's relationship to the lead tenant?
 age3,What is person 3's age?
 sex3,Which of these best describes person 3's gender identity?
 ecstat3,Which of these best describes person 3's working situation?
 details_known_4,Are the details of tenant 4 known?
 relat4,What is person 4's relationship to the lead tenant?
 age4,What is person 4's age?
 sex4,Which of these best describes person 4's gender identity?
 ecstat4,Which of these best describes person 4's working situation?
 details_known_5,Are the details of tenant 5 known?
 relat5,What is person 5's relationship to the lead tenant?
 age5,What is person 5's age?
 sex5,Which of these best describes person 5's gender identity?
 ecstat5,Which of these best describes person 5's working situation?
 details_known_6,Are the details of tenant 6 known?
 relat6,What is person 6's relationship to the lead tenant?
 age6,What is person 6's age?
 sex6,Which of these best describes person 6's gender identity?
 ecstat6,Which of these best describes person 6's working situation?
 details_known_7,Are the details of tenant 7 known?
 relat7,What is person 7's relationship to the lead tenant?
 age7,What is person 7's age?
 sex7,Which of these best describes person 7's gender identity?
 ecstat7,Which of these best describes person 7's working situation?
 details_known_8,Are the details of tenant 8 known?
 relat8,What is person 8's relationship to the lead tenant?
 age8,What is person 8's age?
 sex8,Which of these best describes person 8's gender identity?
 ecstat8,Which of these best describes person 8's working situation?
 armedforces,Does anybody in the household have links to the UK armed forces?
 leftreg,Is this person still serving in the UK armed forces?
 reservist,Was this person seriously injured or ill as a result of serving in the UK armed forces?
 preg_occ,Is anybody in the household pregnant?
 housingneeds,Does anybody in the household have any disabled access needs?
 housingneeds_type,What access needs do they have? (Fully wheelchair-accessible housing, Level access housing or Wheelchair access to essential rooms)
 housingneeds_a,Disabled access needs a) Fully wheelchair-accessible housing
 housingneeds_b,Disabled access needs b) Wheelchair access to essential rooms
 housingneeds_c,Disabled access needs c) Level access housing
 housingneeds_f,Disabled access needs f) Other disabled access needs
 housingneeds_g,Disabled access needs g) No disabled access needs
 housingneeds_h,Disabled access needs h) Don't know
 housingneeds_other,Do they have any other disabled access needs?
 illness,Does anybody in the household have a physical or mental health condition (or other illness) expected to last 12 months or more?
 illness_type_4,Does this person's condition affect their dexterity?
 illness_type_5,Does this person's condition affect their learning or understanding or concentrating?
 illness_type_2,Does this person's condition affect their hearing?
 illness_type_6,Does this person's condition affect their memory?
 illness_type_7,Does this person's condition affect their mental health?
 illness_type_3,Does this person's condition affect their mobility?
 illness_type_9,Does this person's condition affect them socially or behaviourally?
 illness_type_8,Does this person's condition affect their stamina or breathing or fatigue?
 illness_type_1,Does this person's condition affect their vision?
 illness_type_10,Does this person's condition affect them in another way?
 layear,How long has the household continuously lived in the local authority area of the new letting?
 waityear,How long has the household been on the local authority waiting list for the new letting?
 reason,What is the tenant's main reason for the household leaving their last settled home?
 reasonother,If 'Other', what was the main reason for leaving their last settled home?
 reasonother_value_check,The soft validation was confirmed
 prevten,Where was the household immediately before this letting?
 homeless,Did the household experience homelessness immediately before this letting?
 ppcodenk,Previous postcode unknown or previous accommodation was temporary
 ppostcode_full,What is the postcode of the household's last settled home?
 previous_la_known,Was the local authority of the household's last settled home known?
 is_previous_la_inferred,The internal value to indicate if the previous LA was inferred from the postcode
 prevloc_label,Previous location LA name
 prevloc,Previous location's ONS LA Code
 reasonpref,Was the household given reasonable preference by the local authority?
 rp_homeless,Reasonable preference reason - They were homeless or about to lose their home (within 56 days)
 rp_insan_unsat,Reasonable preference reason - They were living in insanitary, overcrowded or unisatisfactory housing
 rp_medwel,Reasonable preference reason - They needed to move on medical and welfare reasons (including disability)
 rp_hardship,Reasonable preference reason - They needed to move to avoid hardship to themselves or others
 rp_dontknow,Reasonable preference reason - Don't Know
 cbl,Was the letting made under Choice-Based Lettings (CBL)?
 cap,Was the letting made under the Common Allocation Policy (CAP)?
 chr,Was the letting made under the Common Housing Register (CHR)?
 accessible_register,Was the letting made under the Accessible Register?
 letting_allocation_none,The letting was not allocated under CBL, CAP, CHR or Accessible Register.
 referral,What was the source of referral for this letting?
 referral_value_check,The following soft validation was confirmed: Are you sure? This is a general needs log, and this referral type is for supported housing.
 net_income_known,Do you know the household's combined income after tax?
 incref,Was the household income refused?
 earnings,How much income does the household have in total?
 incfreq,How often does the household receive income?
 net_income_value_check,Populated when someone hits the soft validation and confirmed in the service
 hb,Is the tenant likely to be receiving any of these housing-related benefits?
 has_benefits,Does the tenant receive housing-related benefits? Yes if hb = Universal Credit housing element or Housing benefit, No if hb =  Don't Know, Neither, Tenant prefers not to say or blank
 benefits,How much of the household's income is from Universal Credit, state pensions or benefits?
 household_charge,Does the household pay rent or other charges for the accommodation?
 nocharge,Does the household pay rent or other charges for the accommodation? - flag for when household_charge is answered no
 period,How often does the household pay rent and other charges?
 is_carehome,Is this accommodation a care home?
 chcharge,If this is a care home, how much does the household pay every [time period]?
 wchchrg,Weekly care home charge
 carehome_charges_value_check,Populated when the soft validation and confirmed in the service
 brent,What is the basic rent?
 wrent,Weekly rent
 rent_value_check,Populated when the soft validation and confirmed in the service
 scharge,What is the service charge?
 wscharge,Weekly service charge
 pscharge,What is the personal service charge?
 wpschrge,Weekly personal service charge
 supcharg,What is the support charge?
 wsupchrg,Weekly support charge
 tcharge,Total charge to the tenant
 wtcharge,Weekly total charge to the tenant
 scharge_value_check,Populated when the soft validation and confirmed in the service
 pscharge_value_check,Populated when the soft validation and confirmed in the service
 supcharg_value_check,Populated when the soft validation and confirmed in the service
 hbrentshortfall,After the household has received any housing-related benefits, will they still need to pay for rent and charges?
 tshortfall_known,Can you estimate the outstanding amount?
 tshortfall,Estimated outstanding amount
 wtshortfall,Weekly total rent shortfall charge for tenant receiving housing benefit
 scheme_code,What scheme does this letting belong to?
 scheme_service_name,From scheme code, we map to the scheme name
 scheme_confidential,Does the scheme contain confidential information?
 SCHTYPE,What is this type of scheme? (Direct access hostel), Foyer, Housing for older people or Other supported housing
 scheme_registered_under_care_act,Is this scheme registered under the Care Standards Act 2000?
 scheme_owning_organisation_name,Which organisation owns the housing stock for this scheme?
 scheme_primary_client_group,What client group is this scheme intended for?
 scheme_has_other_client_group,Does this scheme provide for another client group?
 scheme_secondary_client_group,What is the other client group?
 scheme_support_type,What support does this scheme provide?
 scheme_intended_stay,Intended length of stay
 scheme_created_at,Date scheme was created
 location_code,Which location is this letting for?
 location_postcode,What is the postcode for this location?
 location_name,What is the name of this location?
 location_units,How many units are at this location?
 location_type_of_unit,What is the most common type of unit at this location?
 location_mobility_type,What are the mobility standards for the majority of the units in this location?
 location_local_authority,What is the local authority of this postcode?
 location_startdate,When did the first property in this location become available under this scheme?
--- a/spec/fixtures/variable_definitions/sales_download_23_24.csv
+++ b/spec/fixtures/variable_definitions/sales_download_23_24.csv
@ -0,0 +1,269 @@
 ID,Log ID
 STATUS,Status of log
 DUPLICATESET,ID of a set of duplicate logs
 CREATEDDATE,Time and date the log was created
 UPLOADDATE,Time and date the log was last updated
 FORM,The ID on the old service
 COLLECTIONYEAR,Year collection period opened
 CREATIONMETHOD,Was the log submitted in-service or via bulk upload?
 DATAPROTECT,Is the user in the created_by column the data protection officer?
 OWNINGORGNAME,Which organisation owned this property before the sale?
 MANINGORGNAME,Which organisation reported the sale?
 CREATEDBY,User that created the log
 USERNAME,User the log is assigned to
 DAY,Day of sale completion date
 MONTH,Month of sale completion date
 YEAR,Year of sale completion date
 PURCHID,What is the purchaser code?
 OWNERSHIP,Was this purchase made through an ownership scheme?
 TYPE,What is the type of shared ownership/discounted ownership/outright sale?
 OTHTYPE,If type = 'Other', what is the type of outright sale?
 COMPANY,Is the buyer a company?
 LIVEINBUYER,Will the buyer(s) live in the property?
 JOINT,Is this a joint purchase?
 JOINTMORE,Are there more than 2 joint buyers of this property?
 BEDS,How many bedrooms does the property have?
 PROPTYPE,What type of unit is the property?
 BUILTYPE,Which type of building is the property?
 UPRN,What is the UPRN of the property?
 UPRNCONFIRMED,We found an address that might be this property. Is this the property address?
 ADDRESS1,Address line 1
 ADDRESS2,Address line 2
 TOWNCITY,Town/City
 COUNTY,County
 PCODE1,Part 1 of the property's postcode
 PCODE2,Part 2 of the property's postcode
 LA,LA code
 LANAME,LA name
 WCHAIR,Is the property built or adapted to wheelchair-user standards?
 NOINT,Did you interview the buyer to answer these questions?
 PRIVACYNOTICE,Has the buyer seen the MHCLG privacy notice?
 AGE1,What is buyer 1's age?
 SEX1,Which of these best describes buyer 1's gender identity?
 ETHNICGROUP1,What is buyer 1's ethnic group?
 ETHNIC,Which of the following best describes buyer 1's ethnic background?
 NATIONAL,What is buyer 1's nationality?
 ECSTAT1,Which of these best describes buyer 1's working situation?
 LIVEINBUYER1,Will buyer 1 live in the property?
 RELAT2,What is buyer 2 or person 2's relationship to buyer 1?
 AGE2,What is buyer 2 or person 2's age?
 SEX2,Which of these best describes buyer 2 or person 2's gender identity?
 ETHNICGROUP2,What is buyer 2's ethnic group?
 ETHNIC2,Which of the following best describes buyer 2's ethnic background?
 NATIONAL2,What is buyer 2's nationality?
 ECSTAT2,What is buyer 2 or person 2's working situation?
 LIVEINBUYER2,Will buyer 2 live in the property?
 HHTYPE,Besides the buyer(s), how many other people live or will live in the property?
 RELAT3,What is person 3's relationship to buyer 1?
 AGE3,What is person 3's age?
 SEX3,What is person 3's gender identity?
 ECSTAT3,What is person 3's working situation?
 RELAT4,What is person 4's relationship to buyer 1?
 AGE4,What is person 4's age?
 SEX4,What is person 4's gender identity?
 ECSTAT4,What is person 4's working situation?
 RELAT5,What is person 5's relationship to buyer 1?
 AGE5,What is person 5's age?
 SEX5,What is person 5's gender identity?
 ECSTAT5,What is person 5's working situation?
 RELAT6,What is person 6's relationship to buyer 1?
 AGE6,What is person 6's age?
 SEX6,What is person 6's gender identity?
 ECSTAT6,What is person 6's working situation?
 PREVTEN,What was buyer 1's previous tenure?
 PPCODENK,Do you know the postcode of buyer 1's last settled accommodation?
 PPOSTC1,Part 1 of postcode of buyer 1's last settled accommodation
 PPOSTC2,Part 2 of postcode of buyer 1's last settled accommodation
 PREVIOUSLAKNOWN,Do you know the local authority of buyer 1's last settled accommodation?
 PREVLOC,The local authority code of buyer 1's last settled accommodation
 PREVLOCNAME,The local authority name of buyer 1's last settled accommodation
 PREGYRHA,Was the buyer registered with their PRP (HA)?
 PREGOTHER,Was the buyer registered with another PRP (HA)?
 PREGLA,Was the buyer registered with the local authority?
 PREGGHB,Was the buyer registered with a Help to Buy agent?
 PREGBLANK,Populated if pregyrha, pregother, pregla and pregghb are blank
 BUY2LIVING,At the time of purchase, was buyer 2 living at the same address as buyer 1?
 PREVTEN2,What was buyer 2's previous tenure?
 HHREGRES,Have any of the buyers ever served as a regular in the UK armed forces?
 HHREGRESSTILL,Is the buyer still serving in the UK armed forces?
 ARMEDFORCESSPOUSE,Are any of the buyers a spouse or civil partner of a UK armed forces regular who died in service within the last 2 years?
 DISABLED,Does anyone in the household consider themselves to have a disability?
 WHEEL,Does anyone in the household use a wheelchair?
 INC1NK,Is buyer 1's annual income known?
 INCOME1,What is buyer 1's annual income?
 INC1MORT,Was buyer 1's income used for a mortgage application?
 INC2NK,Is buyer 1's annual income known?
 INCOME2,What is buyer 2's annual income?
 INC2MORT,Was buyer 2's income used for a mortgage application?
 HB,Were the buyers receiving any of these housing-related benefits immediately before buying this property?
 SAVINGSNK,Is the the total amount the buyers had in savings known?
 SAVINGS,What is the total amount the buyers had in savings before they paid any deposit for the property?
 PREVOWN,Have any of the buyers previously owned a property?
 PREVSHARED,Was the previous property under shared ownership?
 PROPLEN,How long did the buyer(s) live in the property before purchasing it?
 STAIRCASE,Is this a staircasing transaction?
 STAIRBOUGHT,What percentage of the property has been bought in this staircasing transaction?
 STAIROWNED,What percentage of the property do the buyers now own in total?
 STAIRCASETOSALE,Was this transaction part of a back-to-back staircasing transaction to facilitate sale of the home on the open market?
 RESALE,Is this a resale?
 EXDAY,Day of the exchange of contracts
 EXMONTH,Month of the exchange of contracts
 EXYEAR,Year of the exchange of contracts
 HODAY,Day of the practical completion or handover date
 HOMONTH,Month of the practical completion or handover date
 HOYEAR,Year of the practical completion or handover date
 LANOMAGR,Was the household rehoused under a local authority nominations agreement?
 SOCTEN,Was the buyer a private registered provider, housing association or local authority tenant immediately before this sale?
 FROMBEDS,How many bedrooms did the buyer's previous property have?
 FROMPROP,What was the previous property type?
 SOCPREVTEN,What was the rent type of buyer's previous tenure?
 VALUE,What is the full purchase price?
 VALUE_VALUE_CHECK,Populated if a soft validation is confirmed.
 EQUITY,What was the initial percentage equity stake purchased?
 MORTGAGEUSED,Was a mortgage used to buy this property?
 MORTGAGE,What is the mortgage amount?
 MORTGAGELENDER,What is the name of the mortgage lender?
 MORTGAGELENDEROTHER,If mortgagelender = 'Other', what is the name of the mortgage lender?
 MORTLEN1,What is the length of the mortgage in years?
 EXTRABOR,Does this include any extra borrowing?
 DEPOSIT,How much was the cash deposit paid on the property?
 CASHDIS,How much cash discount was given through Social Homebuy?
 MRENT,What is the basic monthly rent?
 HASMSCHARGE,Does the property have any monthly leasehold charges?
 MSCHARGE,What are the total monthly leasehold charges for the property?
 MSCHARGE_VALUE_CHECK,Populated if a soft validation is confirmed.
 DISCOUNT,What was the percentage discount?
 GRANT,What was the amount of any loan, grant, discount or subsidy given?
 id,Log ID
 status,Status of log
 duplicate_set_id,ID of a set of duplicate logs
 created_at,Time and date the log was created
 updated_at,Time and date the log was last updated
 old_form_id,The ID on the old service
 collection_start_year,Year collection period opened
 creation_method,Was the log submitted in-service or via bulk upload?
 is_dpo,Is the user in the assigned_to column the data protection officer?
 owning_organisation_name,Which organisation owned this property before the sale?
 managing_organisation_name,Which organisation reported the sale?
 assigned_to,User the log is assigned to
 day,Day of sale completion date
 month,Month of sale completion date
 year,Year of sale completion date
 purchid,What is the purchaser code?
 ownershipsch,Was this purchase made through an ownership scheme?
 type,What is the type of shared ownership/discounted ownership/outright sale?
 othtype,If type = 'Other', what is the type of outright sale?
 companybuy,Is the buyer a company?
 buylivein,Will the buyer(s) live in the property?
 jointpur,Is this a joint purchase?
 jointmore,Are there more than 2 joint buyers of this property?
 beds,How many bedrooms does the property have?
 proptype,What type of unit is the property?
 builtype,Which type of building is the property?
 uprn,What is the UPRN of the property?
 uprn_confirmed,We found an address that might be this property. Is this the property address?
 address_line1,Address line 1
 address_line2,Address line 2
 town_or_city,Town/City
 county,County
 pcode1,Part 1 of the property's postcode
 pcode2,Part 2 of the property's postcode
 la,LA code
 la_label,LA name
 wchair,Is the property built or adapted to wheelchair-user standards?
 noint,Did you interview the buyer to answer these questions?
 privacynotice,Has the buyer seen the MHCLG privacy notice?
 age1,What is buyer 1's age?
 sex1,Which of these best describes buyer 1's gender identity?
 ethnic_group,What is buyer 1's ethnic group?
 ethnic,Which of the following best describes buyer 1's ethnic background?
 national,What is buyer 1's nationality?
 ecstat1,Which of these best describes buyer 1's working situation?
 buy1livein,Will buyer 1 live in the property?
 relat2,What is buyer 2 or person 2's relationship to buyer 1?
 age2,What is buyer 2 or person 2's age?
 sex2,Which of these best describes buyer 2 or person 2's gender identity?
 ethnic_group2,What is buyer 2's ethnic group?
 ethnicbuy2,Which of the following best describes buyer 2's ethnic background?
 nationalbuy2,What is buyer 2's nationality?
 ecstat2,What is buyer 2 or person 2's working situation?
 buy2livein,Will buyer 2 live in the property?
 hholdcount,Besides the buyer(s), how many other people live or will live in the property?
 relat3,What is person 3's relationship to buyer 1?
 age3,What is person 3's age?
 sex3,What is person 3's gender identity?
 ecstat3,What is person 3's working situation?
 relat4,What is person 4's relationship to buyer 1?
 age4,What is person 4's age?
 sex4,What is person 4's gender identity?
 ecstat4,What is person 4's working situation?
 relat5,What is person 5's relationship to buyer 1?
 age5,What is person 5's age?
 sex5,What is person 5's gender identity?
 ecstat5,What is person 5's working situation?
 relat6,What is person 6's relationship to buyer 1?
 age6,What is person 6's age?
 sex6,What is person 6's gender identity?
 ecstat6,What is person 6's working situation?
 prevten,What was buyer 1's previous tenure?
 ppcodenk,Do you know the postcode of buyer 1's last settled accommodation?
 ppostc1,Part 1 of postcode of buyer 1's last settled accommodation
 ppostc2,Part 2 of postcode of buyer 1's last settled accommodation
 previous_la_known,Do you know the local authority of buyer 1's last settled accommodation?
 prevloc,The local authority code of buyer 1's last settled accommodation
 prevloc_label,The local authority name of buyer 1's last settled accommodation
 pregyrha,Was the buyer registered with their PRP (HA)?
 pregother,Was the buyer registered with another PRP (HA)?
 pregla,Was the buyer registered with the local authority?
 pregghb,Was the buyer registered with a Help to Buy agent?
 pregblank,Populated if pregyrha, pregother, pregla and pregghb are blank
 buy2living,At the time of purchase, was buyer 2 living at the same address as buyer 1?
 prevtenbuy2,What was buyer 2's previous tenure?
 hhregres,Have any of the buyers ever served as a regular in the UK armed forces?
 hhregresstill,Is the buyer still serving in the UK armed forces?
 armedforcesspouse,Are any of the buyers a spouse or civil partner of a UK armed forces regular who died in service within the last 2 years?
 disabled,Does anyone in the household consider themselves to have a disability?
 wheel,Does anyone in the household use a wheelchair?
 income1nk,Is buyer 1's annual income known?
 income1,What is buyer 1's annual income?
 inc1mort,Was buyer 1's income used for a mortgage application?
 income2nk,Is buyer 2's annual income known?
 income2,What is buyer 2's annual income?
 inc2mort,Was buyer 2's income used for a mortgage application?
 hb,Were the buyers receiving any of these housing-related benefits immediately before buying this property?
 savingsnk,Is the the total amount the buyers had in savings known?
 savings,What is the total amount the buyers had in savings before they paid any deposit for the property?
 prevown,Have any of the buyers previously owned a property?
 prevshared,Was the previous property under shared ownership?
 proplen,How long did the buyer(s) live in the property before purchasing it?
 staircase,Is this a staircasing transaction?
 stairbought,What percentage of the property has been bought in this staircasing transaction?
 stairowned,What percentage of the property do the buyers now own in total?
 staircasesale,Was this transaction part of a back-to-back staircasing transaction to facilitate sale of the home on the open market?
 resale,Is this a resale?
 exday,Day of the exchange of contracts
 exmonth,Month of the exchange of contracts
 exyear,Year of the exchange of contracts
 hoday,Day of the practical completion or handover date
 homonth,Month of the practical completion or handover date
 hoyear,Year of the practical completion or handover date
 lanomagr,Was the household rehoused under a local authority nominations agreement?
 soctenant,Was the buyer a private registered provider, housing association or local authority tenant immediately before this sale?
 frombeds,How many bedrooms did the buyer's previous property have?
 fromprop,What was the previous property type?
 socprevten,What was the rent type of buyer's previous tenure?
 value,What is the full purchase price?
 equity,What was the initial percentage equity stake purchased?
 mortgageused,Was a mortgage used to buy this property?
 mortgage,What is the mortgage amount?
 mortgagelender,What is the name of the mortgage lender?
 mortgagelenderother,If mortgagelender = 'Other', what is the name of the mortgage lender?
 mortlen,What is the length of the mortgage in years?
 extrabor,Does this include any extra borrowing?
 deposit,How much was the cash deposit paid on the property?
 cashdis,How much cash discount was given through Social Homebuy?
 mrent,What is the basic monthly rent?
 has_mscharge,Does the property have any monthly leasehold charges?
 mscharge,What are the total monthly leasehold charges for the property?
 discount,What was the percentage discount?
 grant,What was the amount of any loan, grant, discount or subsidy given?
--- a/spec/fixtures/variable_definitions/sales_download_24_25.csv
+++ b/spec/fixtures/variable_definitions/sales_download_24_25.csv
@ -0,0 +1,281 @@
 ID,Log ID
 STATUS,Status of log
 DUPLICATESET,ID of a set of duplicate logs
 CREATEDDATE,Time and date the log was created
 UPLOADDATE,Time and date the log was last updated
 COLLECTIONYEAR,Year collection period opened
 CREATIONMETHOD,Was the log submitted in-service or via bulk upload?
 BULKUPLOADID,ID of a set of bulk uploaded logs
 DATAPROTECT,Is the user in the created_by column the data protection officer?
 OWNINGORGNAME,Which organisation owned this property before the sale?
 MANINGORGNAME,Which organisation reported the sale?
 CREATEDBY,User that created the log
 USERNAME,User the log is assigned to
 DAY,Day of sale completion date
 MONTH,Month of sale completion date
 YEAR,Year of sale completion date
 PURCHID,What is the purchaser code?
 OWNERSHIP,Was this purchase made through an ownership scheme?
 TYPE,What is the type of shared ownership/discounted ownership/outright sale?
 OTHTYPE,If type = 'Other', what is the type of outright sale?
 COMPANY,Is the buyer a company?
 LIVEINBUYER,Will the buyer(s) live in the property?
 JOINT,Is this a joint purchase?
 JOINTMORE,Are there more than 2 joint buyers of this property?
 NOINT,Did you interview the buyer to answer these questions?
 PRIVACYNOTICE,Has the buyer seen the MHCLG privacy notice?
 UPRN,What is the UPRN of the property?
 ADDRESS1,Address line 1
 ADDRESS2,Address line 2
 TOWNCITY,Town/City
 COUNTY,County
 POSTCODE,Postcode
 ISLAINFERRED,The internal value to indicate if the LA was inferred from the postcode
 LANAME,LA name
 LA,LA code
 UPRNSELECTED,UPRN of the address selected
 ADDRESS_SEARCH_VALUE_CHECK,Was the 'No address found' page seen?
 ADDRESS1INPUT,Address line 1 input from address matching feature
 POSTCODEINPUT,Postcode input from address matching feature
 BULKADDRESS1,Address line 1 entered in bulk upload file
 BULKADDRESS2,Address line 2 entered in bulk upload file
 BULKTOWNCITY,Town or city entered in bulk upload file
 BULKCOUNTY,County entered in bulk upload file
 BULKPOSTCODE,Postcode entered in bulk upload file
 BULKLA,Local authority entered in bulk upload file
 BEDS,How many bedrooms does the property have?
 PROPTYPE,What type of unit is the property?
 BUILTYPE,Which type of building is the property?
 WCHAIR,Is the property built or adapted to wheelchair-user standards?
 AGE1,What is buyer 1's age?
 SEX1,Which of these best describes buyer 1's gender identity?
 ETHNICGROUP1,What is buyer 1's ethnic group?
 ETHNIC,Which of the following best describes buyer 1's ethnic background?
 NATIONALITYALL1,What is buyer 1's nationality?
 ECSTAT1,Which of these best describes buyer 1's working situation?
 LIVEINBUYER1,Will buyer 1 live in the property?
 RELAT2,What is buyer 2 or person 2's relationship to buyer 1?
 AGE2,What is buyer 2 or person 2's age?
 SEX2,Which of these best describes buyer 2 or person 2's gender identity?
 ETHNICGROUP2,What is buyer 2's ethnic group?
 ETHNIC2,Which of the following best describes buyer 2's ethnic background?
 NATIONALITYALL2,What is buyer 2's nationality?
 ECSTAT2,What is buyer 2 or person 2's working situation?
 LIVEINBUYER2,Will buyer 2 live in the property?
 HHTYPE,Besides the buyer(s), how many other people live or will live in the property?
 RELAT3,What is person 3's relationship to buyer 1?
 AGE3,What is person 3's age?
 SEX3,What is person 3's gender identity?
 ECSTAT3,What is person 3's working situation?
 RELAT4,What is person 4's relationship to buyer 1?
 AGE4,What is person 4's age?
 SEX4,What is person 4's gender identity?
 ECSTAT4,What is person 4's working situation?
 RELAT5,What is person 5's relationship to buyer 1?
 AGE5,What is person 5's age?
 SEX5,What is person 5's gender identity?
 ECSTAT5,What is person 5's working situation?
 RELAT6,What is person 6's relationship to buyer 1?
 AGE6,What is person 6's age?
 SEX6,What is person 6's gender identity?
 ECSTAT6,What is person 6's working situation?
 PREVTEN,What was buyer 1's previous tenure?
 PPCODENK,Do you know the postcode of buyer 1's last settled accommodation?
 PPOSTC1,Part 1 of postcode of buyer 1's last settled accommodation
 PPOSTC2,Part 2 of postcode of buyer 1's last settled accommodation
 PREVIOUSLAKNOWN,Do you know the local authority of buyer 1's last settled accommodation?
 PREVLOC,The local authority code of buyer 1's last settled accommodation
 PREVLOCNAME,The local authority name of buyer 1's last settled accommodation
 PREGYRHA,Was the buyer registered with their PRP (HA)?
 PREGOTHER,Was the buyer registered with another PRP (HA)?
 PREGLA,Was the buyer registered with the local authority?
 PREGGHB,Was the buyer registered with a Help to Buy agent?
 PREGBLANK,Populated if pregyrha, pregother, pregla and pregghb are blank
 BUY2LIVING,At the time of purchase, was buyer 2 living at the same address as buyer 1?
 PREVTEN2,What was buyer 2's previous tenure?
 HHREGRES,Have any of the buyers ever served as a regular in the UK armed forces?
 HHREGRESSTILL,Is the buyer still serving in the UK armed forces?
 ARMEDFORCESSPOUSE,Are any of the buyers a spouse or civil partner of a UK armed forces regular who died in service within the last 2 years?
 DISABLED,Does anyone in the household consider themselves to have a disability?
 WHEEL,Does anyone in the household use a wheelchair?
 INC1NK,Is buyer 1's annual income known?
 INCOME1,What is buyer 1's annual income?
 INC1MORT,Was buyer 1's income used for a mortgage application?
 INC2NK,Is buyer 1's annual income known?
 INCOME2,What is buyer 2's annual income?
 INC2MORT,Was buyer 2's income used for a mortgage application?
 HB,Were the buyers receiving any of these housing-related benefits immediately before buying this property?
 SAVINGSNK,Is the the total amount the buyers had in savings known?
 SAVINGS,What is the total amount the buyers had in savings before they paid any deposit for the property?
 PREVOWN,Have any of the buyers previously owned a property?
 PREVSHARED,Was the previous property under shared ownership?
 PROPLEN,How long did the buyer(s) live in the property before purchasing it?
 STAIRCASE,Is this a staircasing transaction?
 STAIRBOUGHT,What percentage of the property has been bought in this staircasing transaction?
 STAIROWNED,What percentage of the property do the buyers now own in total?
 STAIRCASETOSALE,Was this transaction part of a back-to-back staircasing transaction to facilitate sale of the home on the open market?
 RESALE,Is this a resale?
 EXDAY,Day of the exchange of contracts
 EXMONTH,Month of the exchange of contracts
 EXYEAR,Year of the exchange of contracts
 HODAY,Day of the practical completion or handover date
 HOMONTH,Month of the practical completion or handover date
 HOYEAR,Year of the practical completion or handover date
 LANOMAGR,Was the household rehoused under a local authority nominations agreement?
 SOCTEN,Was the buyer a private registered provider, housing association or local authority tenant immediately before this sale?
 FROMBEDS,How many bedrooms did the buyer's previous property have?
 FROMPROP,What was the previous property type?
 SOCPREVTEN,What was the rent type of buyer's previous tenure?
 VALUE,What is the full purchase price?
 VALUE_VALUE_CHECK,Populated if a soft validation is confirmed.
 EQUITY,What was the initial percentage equity stake purchased?
 MORTGAGEUSED,Was a mortgage used to buy this property?
 MORTGAGE,What is the mortgage amount?
 MORTGAGELENDER,What is the name of the mortgage lender?
 MORTGAGELENDEROTHER,If mortgagelender = 'Other', what is the name of the mortgage lender?
 MORTLEN1,What is the length of the mortgage in years?
 EXTRABOR,Does this include any extra borrowing?
 DEPOSIT,How much was the cash deposit paid on the property?
 CASHDIS,How much cash discount was given through Social Homebuy?
 MRENT,What is the basic monthly rent?
 HASMSCHARGE,Does the property have any monthly leasehold charges?
 MSCHARGE,What are the total monthly leasehold charges for the property?
 MSCHARGE_VALUE_CHECK,Populated if a soft validation is confirmed.
 DISCOUNT,What was the percentage discount?
 GRANT,What was the amount of any loan, grant, discount or subsidy given?
 id,Log ID
 status,Status of log
 duplicate_set_id,ID of a set of duplicate logs
 created_at,Time and date the log was created
 updated_at,Time and date the log was last updated
 old_form_id,The ID on the old service
 collection_start_year,Year collection period opened
 creation_method,Was the log submitted in-service or via bulk upload?
 is_dpo,Is the user in the assigned_to column the data protection officer?
 owning_organisation_name,Which organisation owned this property before the sale?
 managing_organisation_name,Which organisation reported the sale?
 assigned_to,User the log is assigned to
 day,Day of sale completion date
 month,Month of sale completion date
 year,Year of sale completion date
 purchid,What is the purchaser code?
 ownershipsch,Was this purchase made through an ownership scheme?
 type,What is the type of shared ownership/discounted ownership/outright sale?
 othtype,If type = 'Other', what is the type of outright sale?
 companybuy,Is the buyer a company?
 buylivein,Will the buyer(s) live in the property?
 jointpur,Is this a joint purchase?
 jointmore,Are there more than 2 joint buyers of this property?
 beds,How many bedrooms does the property have?
 proptype,What type of unit is the property?
 builtype,Which type of building is the property?
 uprn,What is the UPRN of the property?
 uprn_confirmed,We found an address that might be this property. Is this the property address?
 address_line1_input,Address line 1 input from address matching feature
 postcode_full_input,Postcode input from address matching feature
 uprn_selection,UPRN of the address selected
 address_line1,Address line 1
 address_line2,Address line 2
 town_or_city,Town/City
 county,County
 pcode1,Part 1 of the property's postcode
 pcode2,Part 2 of the property's postcode
 la,LA code
 la_label,LA name
 wchair,Is the property built or adapted to wheelchair-user standards?
 noint,Did you interview the buyer to answer these questions?
 privacynotice,Has the buyer seen the MHCLG privacy notice?
 age1,What is buyer 1's age?
 sex1,Which of these best describes buyer 1's gender identity?
 ethnic_group,What is buyer 1's ethnic group?
 ethnic,Which of the following best describes buyer 1's ethnic background?
 nationality_all,What is buyer 1's nationality?
 ecstat1,Which of these best describes buyer 1's working situation?
 buy1livein,Will buyer 1 live in the property?
 relat2,What is buyer 2 or person 2's relationship to buyer 1?
 age2,What is buyer 2 or person 2's age?
 sex2,Which of these best describes buyer 2 or person 2's gender identity?
 ethnic_group2,What is buyer 2's ethnic group?
 ethnicbuy2,Which of the following best describes buyer 2's ethnic background?
 nationality_all_buyer2,What is buyer 2's nationality?
 ecstat2,What is buyer 2 or person 2's working situation?
 buy2livein,Will buyer 2 live in the property?
 hholdcount,Besides the buyer(s), how many other people live or will live in the property?
 relat3,What is person 3's relationship to buyer 1?
 age3,What is person 3's age?
 sex3,What is person 3's gender identity?
 ecstat3,What is person 3's working situation?
 relat4,What is person 4's relationship to buyer 1?
 age4,What is person 4's age?
 sex4,What is person 4's gender identity?
 ecstat4,What is person 4's working situation?
 relat5,What is person 5's relationship to buyer 1?
 age5,What is person 5's age?
 sex5,What is person 5's gender identity?
 ecstat5,What is person 5's working situation?
 relat6,What is person 6's relationship to buyer 1?
 age6,What is person 6's age?
 sex6,What is person 6's gender identity?
 ecstat6,What is person 6's working situation?
 prevten,What was buyer 1's previous tenure?
 ppcodenk,Do you know the postcode of buyer 1's last settled accommodation?
 ppostc1,Part 1 of postcode of buyer 1's last settled accommodation
 ppostc2,Part 2 of postcode of buyer 1's last settled accommodation
 previous_la_known,Do you know the local authority of buyer 1's last settled accommodation?
 prevloc,The local authority code of buyer 1's last settled accommodation
 prevloc_label,The local authority name of buyer 1's last settled accommodation
 pregyrha,Was the buyer registered with their PRP (HA)?
 pregother,Was the buyer registered with another PRP (HA)?
 pregla,Was the buyer registered with the local authority?
 pregghb,Was the buyer registered with a Help to Buy agent?
 pregblank,Populated if pregyrha, pregother, pregla and pregghb are blank
 buy2living,At the time of purchase, was buyer 2 living at the same address as buyer 1?
 prevtenbuy2,What was buyer 2's previous tenure?
 hhregres,Have any of the buyers ever served as a regular in the UK armed forces?
 hhregresstill,Is the buyer still serving in the UK armed forces?
 armedforcesspouse,Are any of the buyers a spouse or civil partner of a UK armed forces regular who died in service within the last 2 years?
 disabled,Does anyone in the household consider themselves to have a disability?
 wheel,Does anyone in the household use a wheelchair?
 income1nk,Is buyer 1's annual income known?
 income1,What is buyer 1's annual income?
 inc1mort,Was buyer 1's income used for a mortgage application?
 income2nk,Is buyer 2's annual income known?
 income2,What is buyer 2's annual income?
 inc2mort,Was buyer 2's income used for a mortgage application?
 hb,Were the buyers receiving any of these housing-related benefits immediately before buying this property?
 savingsnk,Is the the total amount the buyers had in savings known?
 savings,What is the total amount the buyers had in savings before they paid any deposit for the property?
 prevown,Have any of the buyers previously owned a property?
 prevshared,Was the previous property under shared ownership?
 proplen,How long did the buyer(s) live in the property before purchasing it?
 staircase,Is this a staircasing transaction?
 stairbought,What percentage of the property has been bought in this staircasing transaction?
 stairowned,What percentage of the property do the buyers now own in total?
 staircasesale,Was this transaction part of a back-to-back staircasing transaction to facilitate sale of the home on the open market?
 resale,Is this a resale?
 exday,Day of the exchange of contracts
 exmonth,Month of the exchange of contracts
 exyear,Year of the exchange of contracts
 hoday,Day of the practical completion or handover date
 homonth,Month of the practical completion or handover date
 hoyear,Year of the practical completion or handover date
 lanomagr,Was the household rehoused under a local authority nominations agreement?
 soctenant,Was the buyer a private registered provider, housing association or local authority tenant immediately before this sale?
 frombeds,How many bedrooms did the buyer's previous property have?
 fromprop,What was the previous property type?
 socprevten,What was the rent type of buyer's previous tenure?
 value,What is the full purchase price?
 equity,What was the initial percentage equity stake purchased?
 mortgageused,Was a mortgage used to buy this property?
 mortgage,What is the mortgage amount?
 mortgagelender,What is the name of the mortgage lender?
 mortgagelenderother,If mortgagelender = 'Other', what is the name of the mortgage lender?
 mortlen,What is the length of the mortgage in years?
 extrabor,Does this include any extra borrowing?
 deposit,How much was the cash deposit paid on the property?
 cashdis,How much cash discount was given through Social Homebuy?
 mrent,What is the basic monthly rent?
 has_mscharge,Does the property have any monthly leasehold charges?
 mscharge,What are the total monthly leasehold charges for the property?
 discount,What was the percentage discount?
 grant,What was the amount of any loan, grant, discount or subsidy given?
--- a/spec/helpers/filters_helper_spec.rb
+++ b/spec/helpers/filters_helper_spec.rb
@ -168,7 +168,7 @@ RSpec.describe FiltersHelper do
    before do
      FactoryBot.create(:organisation_relationship, parent_organisation:, child_organisation:)
-      FactoryBot.create(:organisation, name: "Other organisation", id: 99)
+      FactoryBot.create(:organisation, name: "Other organisation", id: 9_999_999)
      user.organisation.reload
    end
@ -326,7 +326,7 @@ RSpec.describe FiltersHelper do
    before do
      FactoryBot.create(:organisation_relationship, parent_organisation:, child_organisation:)
-      FactoryBot.create(:organisation, name: "Other organisation", id: 99)
+      FactoryBot.create(:organisation, name: "Other organisation", id: 9_999_999)
      user.organisation.reload
    end
--- a/spec/lib/tasks/log_variable_definitions_spec.rb
+++ b/spec/lib/tasks/log_variable_definitions_spec.rb
@ -0,0 +1,41 @@
 require "rails_helper"
 require "rake"
 RSpec.describe "log_variable_definitions" do
  describe ":add_variable_definitions", type: :task do
    subject(:task) { Rake::Task["data_import:add_variable_definitions"] }
    let(:path) { "spec/fixtures/variable_definitions" }
    before do
      Rake.application.rake_require("tasks/log_variable_definitions")
      Rake::Task.define_task(:environment)
      task.reenable
    end
    it "adds CsvVariableDefinition records from each file in the specified directory" do
      expect { task.invoke(path) }.to change(CsvVariableDefinition, :count).by(416)
    end
    it "handles an empty directory without errors" do
      empty_path = "spec/fixtures/empty_directory"
      FileUtils.mkdir_p(empty_path)
      expect { task.invoke(empty_path) }.not_to raise_error
      expect(CsvVariableDefinition.count).to eq(0)
    end
    it "does not create duplicate records if run multiple times" do
      CsvVariableDefinition.delete_all
      initial_count = CsvVariableDefinition.count
      task.invoke(path)
      first_run_count = CsvVariableDefinition.count
      task.invoke(path)
      second_run_count = CsvVariableDefinition.count
      expect(first_run_count).to eq(initial_count + 416)
      expect(second_run_count).to eq(first_run_count)
    end
  end
 end
--- a/spec/services/csv/scheme_csv_service_spec.rb
+++ b/spec/services/csv/scheme_csv_service_spec.rb
@ -13,12 +13,14 @@ RSpec.describe Csv::SchemeCsvService do
  before do
    Timecop.freeze(fixed_time)
    Singleton.__init__(FormHandler)
    create(:scheme_deactivation_period, scheme:, deactivation_date: scheme.created_at + 1.year, reactivation_date: scheme.created_at + 2.years)
    create(:location_deactivation_period, location:, deactivation_date: location.created_at + 6.months)
  end
  after do
    Timecop.return
    Singleton.__init__(FormHandler)
  end
  it "returns a string" do
--- a/spec/services/imports/variable_definitions_service_spec.rb
+++ b/spec/services/imports/variable_definitions_service_spec.rb
@ -0,0 +1,44 @@
 require "rails_helper"
 RSpec.describe Imports::VariableDefinitionsService, type: :service do
  let(:path) { "spec/fixtures/variable_definitions" }
  let(:service) { described_class.new(path:) }
  describe "#initialize" do
    it "initializes with the correct path and count" do
      expect(service.path).to eq(path)
      expect(service.count).to eq(0)
    end
  end
  describe "#call" do
    before do
      allow(Dir).to receive(:glob).and_return(%w[lettings_download_23_24.csv lettings_download_24_25.csv sales_download_23_24.csv sales_download_24_25.csv])
      allow(service).to receive(:process_file)
    end
    it "processes each file in the directory" do
      service.call
      %w[lettings_download_23_24.csv lettings_download_24_25.csv sales_download_23_24.csv sales_download_24_25.csv].each do |file|
        expect(service).to have_received(:process_file).with(file)
      end
    end
  end
  describe "#process_file" do
    let(:file) { "spec/fixtures/variable_definitions/lettings_download_23_24.csv" }
    let(:csv_content) { [["id", "Log ID"], ["status", "Status of log"], ["duplicate_set_id", "ID of a set of duplicate logs"]] }
    before do
      allow(CSV).to receive(:foreach).and_yield(csv_content[0]).and_yield(csv_content[1]).and_yield(csv_content[2])
    end
    context "when no existing record" do
      it "creates new records" do
        expect {
          service.send(:process_file, file)
        }.to change(CsvVariableDefinition, :count).by(3)
      end
    end
  end
 end