Browse Source
* Revert "CLDC-3241: Part 2 - Explanation of variables on CSVs (#2586)"
This reverts commit e045588775.
* Update Gemfile.lock
pull/2608/head
Manny Dinssa
5 months ago
committed by
GitHub
36 changed files with 117 additions and 2361 deletions
@ -1,10 +0,0 @@
|
||||
class CsvVariableDefinition < ApplicationRecord |
||||
validates :variable, presence: true |
||||
validates :definition, presence: true |
||||
validates :log_type, presence: true, inclusion: { in: %w[lettings sales] } |
||||
validates :year, presence: true, numericality: { only_integer: true, greater_than_or_equal_to: 2000, less_than_or_equal_to: 2099 } |
||||
attribute :last_accessed, :datetime |
||||
|
||||
scope :lettings, -> { where(log_type: "lettings") } |
||||
scope :sales, -> { where(log_type: "sales") } |
||||
end |
||||
@ -1,53 +0,0 @@
|
||||
require "csv" |
||||
|
||||
module Imports |
||||
class VariableDefinitionsService |
||||
attr_reader :path, :count |
||||
|
||||
def initialize(path:) |
||||
@path = path |
||||
@count = 0 |
||||
end |
||||
|
||||
def call |
||||
files = Dir.glob(File.join(@path, "*.csv")) |
||||
files.each do |file| |
||||
process_file(file) |
||||
end |
||||
end |
||||
|
||||
private |
||||
|
||||
def process_file(file) |
||||
file_name = File.basename(file, ".csv") |
||||
parsed_file_name = file_name.split("_") |
||||
log_type = parsed_file_name[0] |
||||
year = "20#{parsed_file_name[2]}".to_i |
||||
|
||||
records_added = 0 |
||||
|
||||
CSV.foreach(file) do |row| |
||||
next if row.empty? |
||||
|
||||
variable = row[0].downcase |
||||
definition = row[1..].join(",") |
||||
next if variable.nil? || definition.nil? |
||||
|
||||
existing_record = CsvVariableDefinition.find_by(variable: variable.strip, definition: definition.strip, log_type:) |
||||
|
||||
if existing_record.nil? |
||||
CsvVariableDefinition.create!( |
||||
variable: variable.strip, |
||||
definition: definition.strip, |
||||
log_type:, |
||||
year:, |
||||
) |
||||
records_added += 1 |
||||
end |
||||
end |
||||
|
||||
Rails.logger.debug "Added #{records_added} variable/definition records for file: #{file_name}. Duplicates excluded." |
||||
@count += records_added |
||||
end |
||||
end |
||||
end |
||||
|
Can't render this file because it has a wrong number of fields in line 20.
|
|
Can't render this file because it has a wrong number of fields in line 25.
|
|
Can't render this file because it has a wrong number of fields in line 20.
|
|
Can't render this file because it has a wrong number of fields in line 20.
|
@ -1,15 +0,0 @@
|
||||
class CreateCsvVariableDefinitions < ActiveRecord::Migration[7.0] |
||||
def change |
||||
create_table :csv_variable_definitions do |t| |
||||
t.string :variable, null: false |
||||
t.string :definition, null: false |
||||
t.string :log_type, null: false |
||||
t.integer :year, null: false |
||||
t.datetime :last_accessed |
||||
t.timestamps |
||||
end |
||||
|
||||
add_check_constraint :csv_variable_definitions, "log_type IN ('lettings', 'sales')", name: "log_type_check" |
||||
add_check_constraint :csv_variable_definitions, "year BETWEEN 2000 AND 2099", name: "year_check" |
||||
end |
||||
end |
||||
@ -1,9 +0,0 @@
|
||||
namespace :data_import do |
||||
desc "Add CsvVariableDefinition records for each file in the specified directory" |
||||
task :add_variable_definitions, [:path] => :environment do |_task, args| |
||||
path = Rails.root.join(args[:path]) |
||||
service = Imports::VariableDefinitionsService.new(path:) |
||||
service.call |
||||
Rails.logger.info "CSV Variable Definitions: #{service.count} total records added" |
||||
end |
||||
end |
||||
@ -1,8 +0,0 @@
|
||||
FactoryBot.define do |
||||
factory :csv_variable_definition do |
||||
variable { "variable" } |
||||
definition { "definition" } |
||||
log_type { "lettings" } |
||||
year { 2024 } |
||||
end |
||||
end |
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
Can't render this file because it has a wrong number of fields in line 20.
|
|
Can't render this file because it has a wrong number of fields in line 25.
|
|
Can't render this file because it has a wrong number of fields in line 20.
|
|
Can't render this file because it has a wrong number of fields in line 20.
|
@ -1,41 +0,0 @@
|
||||
require "rails_helper" |
||||
require "rake" |
||||
|
||||
RSpec.describe "log_variable_definitions" do |
||||
describe ":add_variable_definitions", type: :task do |
||||
subject(:task) { Rake::Task["data_import:add_variable_definitions"] } |
||||
|
||||
let(:path) { "spec/fixtures/variable_definitions" } |
||||
|
||||
before do |
||||
Rake.application.rake_require("tasks/log_variable_definitions") |
||||
Rake::Task.define_task(:environment) |
||||
task.reenable |
||||
end |
||||
|
||||
it "adds CsvVariableDefinition records from each file in the specified directory" do |
||||
expect { task.invoke(path) }.to change(CsvVariableDefinition, :count).by(416) |
||||
end |
||||
|
||||
it "handles an empty directory without errors" do |
||||
empty_path = "spec/fixtures/empty_directory" |
||||
FileUtils.mkdir_p(empty_path) |
||||
expect { task.invoke(empty_path) }.not_to raise_error |
||||
expect(CsvVariableDefinition.count).to eq(0) |
||||
end |
||||
|
||||
it "does not create duplicate records if run multiple times" do |
||||
CsvVariableDefinition.delete_all |
||||
initial_count = CsvVariableDefinition.count |
||||
|
||||
task.invoke(path) |
||||
first_run_count = CsvVariableDefinition.count |
||||
|
||||
task.invoke(path) |
||||
second_run_count = CsvVariableDefinition.count |
||||
|
||||
expect(first_run_count).to eq(initial_count + 416) |
||||
expect(second_run_count).to eq(first_run_count) |
||||
end |
||||
end |
||||
end |
||||
@ -1,44 +0,0 @@
|
||||
require "rails_helper" |
||||
|
||||
RSpec.describe Imports::VariableDefinitionsService, type: :service do |
||||
let(:path) { "spec/fixtures/variable_definitions" } |
||||
let(:service) { described_class.new(path:) } |
||||
|
||||
describe "#initialize" do |
||||
it "initializes with the correct path and count" do |
||||
expect(service.path).to eq(path) |
||||
expect(service.count).to eq(0) |
||||
end |
||||
end |
||||
|
||||
describe "#call" do |
||||
before do |
||||
allow(Dir).to receive(:glob).and_return(%w[lettings_download_23_24.csv lettings_download_24_25.csv sales_download_23_24.csv sales_download_24_25.csv]) |
||||
allow(service).to receive(:process_file) |
||||
end |
||||
|
||||
it "processes each file in the directory" do |
||||
service.call |
||||
%w[lettings_download_23_24.csv lettings_download_24_25.csv sales_download_23_24.csv sales_download_24_25.csv].each do |file| |
||||
expect(service).to have_received(:process_file).with(file) |
||||
end |
||||
end |
||||
end |
||||
|
||||
describe "#process_file" do |
||||
let(:file) { "spec/fixtures/variable_definitions/lettings_download_23_24.csv" } |
||||
let(:csv_content) { [["id", "Log ID"], ["status", "Status of log"], ["duplicate_set_id", "ID of a set of duplicate logs"]] } |
||||
|
||||
before do |
||||
allow(CSV).to receive(:foreach).and_yield(csv_content[0]).and_yield(csv_content[1]).and_yield(csv_content[2]) |
||||
end |
||||
|
||||
context "when no existing record" do |
||||
it "creates new records" do |
||||
expect { |
||||
service.send(:process_file, file) |
||||
}.to change(CsvVariableDefinition, :count).by(3) |
||||
end |
||||
end |
||||
end |
||||
end |
||||
Loading…
Reference in new issue