Browse Source
* Create CsvVariableDefinition migration and model * Add definition files * Add CsvVariableDefinitions to rails admin * Create rake and service * Create tests * Additional fixes * Rollback unwanted changes to schema * Add variable definitions as the first row of csv downloads * Add and update tests * Fix lint offences * Update Gemfile.lockpull/2609/head
Manny Dinssa
4 months ago
committed by
GitHub
37 changed files with 2362 additions and 118 deletions
@ -0,0 +1,10 @@ |
|||||||
|
class CsvVariableDefinition < ApplicationRecord |
||||||
|
validates :variable, presence: true |
||||||
|
validates :definition, presence: true |
||||||
|
validates :log_type, presence: true, inclusion: { in: %w[lettings sales] } |
||||||
|
validates :year, presence: true, numericality: { only_integer: true, greater_than_or_equal_to: 2000, less_than_or_equal_to: 2099 } |
||||||
|
attribute :last_accessed, :datetime |
||||||
|
|
||||||
|
scope :lettings, -> { where(log_type: "lettings") } |
||||||
|
scope :sales, -> { where(log_type: "sales") } |
||||||
|
end |
||||||
@ -0,0 +1,53 @@ |
|||||||
|
require "csv" |
||||||
|
|
||||||
|
module Imports |
||||||
|
class VariableDefinitionsService |
||||||
|
attr_reader :path, :count |
||||||
|
|
||||||
|
def initialize(path:) |
||||||
|
@path = path |
||||||
|
@count = 0 |
||||||
|
end |
||||||
|
|
||||||
|
def call |
||||||
|
files = Dir.glob(File.join(@path, "*.csv")) |
||||||
|
files.each do |file| |
||||||
|
process_file(file) |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
private |
||||||
|
|
||||||
|
def process_file(file) |
||||||
|
file_name = File.basename(file, ".csv") |
||||||
|
parsed_file_name = file_name.split("_") |
||||||
|
log_type = parsed_file_name[0] |
||||||
|
year = "20#{parsed_file_name[2]}".to_i |
||||||
|
|
||||||
|
records_added = 0 |
||||||
|
|
||||||
|
CSV.foreach(file) do |row| |
||||||
|
next if row.empty? |
||||||
|
|
||||||
|
variable = row[0].downcase |
||||||
|
definition = row[1..].join(",") |
||||||
|
next if variable.nil? || definition.nil? |
||||||
|
|
||||||
|
existing_record = CsvVariableDefinition.find_by(variable: variable.strip, definition: definition.strip, log_type:) |
||||||
|
|
||||||
|
if existing_record.nil? |
||||||
|
CsvVariableDefinition.create!( |
||||||
|
variable: variable.strip, |
||||||
|
definition: definition.strip, |
||||||
|
log_type:, |
||||||
|
year:, |
||||||
|
) |
||||||
|
records_added += 1 |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
Rails.logger.debug "Added #{records_added} variable/definition records for file: #{file_name}. Duplicates excluded." |
||||||
|
@count += records_added |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
Can't render this file because it has a wrong number of fields in line 20.
|
|
Can't render this file because it has a wrong number of fields in line 25.
|
|
Can't render this file because it has a wrong number of fields in line 20.
|
|
Can't render this file because it has a wrong number of fields in line 20.
|
@ -0,0 +1,15 @@ |
|||||||
|
class CreateCsvVariableDefinitions < ActiveRecord::Migration[7.0] |
||||||
|
def change |
||||||
|
create_table :csv_variable_definitions do |t| |
||||||
|
t.string :variable, null: false |
||||||
|
t.string :definition, null: false |
||||||
|
t.string :log_type, null: false |
||||||
|
t.integer :year, null: false |
||||||
|
t.datetime :last_accessed |
||||||
|
t.timestamps |
||||||
|
end |
||||||
|
|
||||||
|
add_check_constraint :csv_variable_definitions, "log_type IN ('lettings', 'sales')", name: "log_type_check" |
||||||
|
add_check_constraint :csv_variable_definitions, "year BETWEEN 2000 AND 2099", name: "year_check" |
||||||
|
end |
||||||
|
end |
||||||
@ -0,0 +1,9 @@ |
|||||||
|
namespace :data_import do |
||||||
|
desc "Add CsvVariableDefinition records for each file in the specified directory" |
||||||
|
task :add_variable_definitions, [:path] => :environment do |_task, args| |
||||||
|
path = Rails.root.join(args[:path]) |
||||||
|
service = Imports::VariableDefinitionsService.new(path:) |
||||||
|
service.call |
||||||
|
Rails.logger.info "CSV Variable Definitions: #{service.count} total records added" |
||||||
|
end |
||||||
|
end |
||||||
@ -0,0 +1,8 @@ |
|||||||
|
FactoryBot.define do |
||||||
|
factory :csv_variable_definition do |
||||||
|
variable { "variable" } |
||||||
|
definition { "definition" } |
||||||
|
log_type { "lettings" } |
||||||
|
year { 2024 } |
||||||
|
end |
||||||
|
end |
||||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
Can't render this file because it has a wrong number of fields in line 20.
|
|
Can't render this file because it has a wrong number of fields in line 25.
|
|
Can't render this file because it has a wrong number of fields in line 20.
|
|
Can't render this file because it has a wrong number of fields in line 20.
|
@ -0,0 +1,41 @@ |
|||||||
|
require "rails_helper" |
||||||
|
require "rake" |
||||||
|
|
||||||
|
RSpec.describe "log_variable_definitions" do |
||||||
|
describe ":add_variable_definitions", type: :task do |
||||||
|
subject(:task) { Rake::Task["data_import:add_variable_definitions"] } |
||||||
|
|
||||||
|
let(:path) { "spec/fixtures/variable_definitions" } |
||||||
|
|
||||||
|
before do |
||||||
|
Rake.application.rake_require("tasks/log_variable_definitions") |
||||||
|
Rake::Task.define_task(:environment) |
||||||
|
task.reenable |
||||||
|
end |
||||||
|
|
||||||
|
it "adds CsvVariableDefinition records from each file in the specified directory" do |
||||||
|
expect { task.invoke(path) }.to change(CsvVariableDefinition, :count).by(416) |
||||||
|
end |
||||||
|
|
||||||
|
it "handles an empty directory without errors" do |
||||||
|
empty_path = "spec/fixtures/empty_directory" |
||||||
|
FileUtils.mkdir_p(empty_path) |
||||||
|
expect { task.invoke(empty_path) }.not_to raise_error |
||||||
|
expect(CsvVariableDefinition.count).to eq(0) |
||||||
|
end |
||||||
|
|
||||||
|
it "does not create duplicate records if run multiple times" do |
||||||
|
CsvVariableDefinition.delete_all |
||||||
|
initial_count = CsvVariableDefinition.count |
||||||
|
|
||||||
|
task.invoke(path) |
||||||
|
first_run_count = CsvVariableDefinition.count |
||||||
|
|
||||||
|
task.invoke(path) |
||||||
|
second_run_count = CsvVariableDefinition.count |
||||||
|
|
||||||
|
expect(first_run_count).to eq(initial_count + 416) |
||||||
|
expect(second_run_count).to eq(first_run_count) |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
@ -0,0 +1,44 @@ |
|||||||
|
require "rails_helper" |
||||||
|
|
||||||
|
RSpec.describe Imports::VariableDefinitionsService, type: :service do |
||||||
|
let(:path) { "spec/fixtures/variable_definitions" } |
||||||
|
let(:service) { described_class.new(path:) } |
||||||
|
|
||||||
|
describe "#initialize" do |
||||||
|
it "initializes with the correct path and count" do |
||||||
|
expect(service.path).to eq(path) |
||||||
|
expect(service.count).to eq(0) |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
describe "#call" do |
||||||
|
before do |
||||||
|
allow(Dir).to receive(:glob).and_return(%w[lettings_download_23_24.csv lettings_download_24_25.csv sales_download_23_24.csv sales_download_24_25.csv]) |
||||||
|
allow(service).to receive(:process_file) |
||||||
|
end |
||||||
|
|
||||||
|
it "processes each file in the directory" do |
||||||
|
service.call |
||||||
|
%w[lettings_download_23_24.csv lettings_download_24_25.csv sales_download_23_24.csv sales_download_24_25.csv].each do |file| |
||||||
|
expect(service).to have_received(:process_file).with(file) |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
describe "#process_file" do |
||||||
|
let(:file) { "spec/fixtures/variable_definitions/lettings_download_23_24.csv" } |
||||||
|
let(:csv_content) { [["id", "Log ID"], ["status", "Status of log"], ["duplicate_set_id", "ID of a set of duplicate logs"]] } |
||||||
|
|
||||||
|
before do |
||||||
|
allow(CSV).to receive(:foreach).and_yield(csv_content[0]).and_yield(csv_content[1]).and_yield(csv_content[2]) |
||||||
|
end |
||||||
|
|
||||||
|
context "when no existing record" do |
||||||
|
it "creates new records" do |
||||||
|
expect { |
||||||
|
service.send(:process_file, file) |
||||||
|
}.to change(CsvVariableDefinition, :count).by(3) |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
Loading…
Reference in new issue