From 6b61a68df739871d1304a446fe386d5a1e62a1b5 Mon Sep 17 00:00:00 2001
From: kosiakkatrina <54268893+kosiakkatrina@users.noreply.github.com>
Date: Fri, 22 Nov 2024 08:41:32 +0000
Subject: [PATCH] CLDC-3700 Update csv downloads (#2785)
* Save a csv download record
* Allow downloading CSVs
* Make links expire
* Send correct download link
* Fix world
* Make csv downloads work locally
* Set expiration time on the record
* lint
* Update download urls
* Update expiration time
* Update expired link content
* Add a page to view the download
* lint
* lint again
---
app/controllers/csv_downloads_controller.rb | 27 ++++
app/jobs/email_csv_job.rb | 13 +-
app/jobs/scheme_email_csv_job.rb | 13 +-
app/models/csv_download.rb | 10 ++
app/policies/csv_download_policy.rb | 16 +++
app/services/csv/downloader.rb | 50 +++++++
app/views/csv_downloads/show.html.erb | 10 ++
.../errors/download_link_expired.html.erb | 8 ++
config/routes.rb | 7 +
.../20241118104046_add_csv_download_table.rb | 12 ++
db/schema.rb | 14 +-
spec/factories/csv_download.rb | 9 ++
spec/jobs/email_csv_job_spec.rb | 25 +++-
spec/jobs/scheme_email_csv_job_spec.rb | 36 ++++-
.../requests/csv_downloads_controller_spec.rb | 136 ++++++++++++++++++
15 files changed, 371 insertions(+), 15 deletions(-)
create mode 100644 app/controllers/csv_downloads_controller.rb
create mode 100644 app/models/csv_download.rb
create mode 100644 app/policies/csv_download_policy.rb
create mode 100644 app/services/csv/downloader.rb
create mode 100644 app/views/csv_downloads/show.html.erb
create mode 100644 app/views/errors/download_link_expired.html.erb
create mode 100644 db/migrate/20241118104046_add_csv_download_table.rb
create mode 100644 spec/factories/csv_download.rb
create mode 100644 spec/requests/csv_downloads_controller_spec.rb
diff --git a/app/controllers/csv_downloads_controller.rb b/app/controllers/csv_downloads_controller.rb
new file mode 100644
index 000000000..25f70026f
--- /dev/null
+++ b/app/controllers/csv_downloads_controller.rb
@@ -0,0 +1,27 @@
+class CsvDownloadsController < ApplicationController
+ before_action :authenticate_user!
+
+ def show
+ @csv_download = CsvDownload.find(params[:id])
+ authorize @csv_download
+
+ return render "errors/download_link_expired" if @csv_download.expired?
+ end
+
+ def download
+ csv_download = CsvDownload.find(params[:id])
+ authorize csv_download
+
+ return render "errors/download_link_expired" if csv_download.expired?
+
+ downloader = Csv::Downloader.new(csv_download:)
+
+ if Rails.env.development?
+ downloader.call
+ send_file downloader.path, filename: csv_download.filename, type: "text/csv"
+ else
+ presigned_url = downloader.presigned_url
+ redirect_to presigned_url, allow_other_host: true
+ end
+ end
+end
diff --git a/app/jobs/email_csv_job.rb b/app/jobs/email_csv_job.rb
index 58f2d50b8..dd0f2917c 100644
--- a/app/jobs/email_csv_job.rb
+++ b/app/jobs/email_csv_job.rb
@@ -1,9 +1,10 @@
class EmailCsvJob < ApplicationJob
+ include Rails.application.routes.url_helpers
queue_as :default
BYTE_ORDER_MARK = "\uFEFF".freeze # Required to ensure Excel always reads CSV as UTF-8
- EXPIRATION_TIME = 24.hours.to_i
+ EXPIRATION_TIME = 48.hours.to_i
def perform(user, search_term = nil, filters = {}, all_orgs = false, organisation = nil, codes_only_export = false, log_type = "lettings", year = nil) # rubocop:disable Style/OptionalBooleanParameter - sidekiq can't serialise named params
export_type = codes_only_export ? "codes" : "labels"
@@ -20,10 +21,16 @@ class EmailCsvJob < ApplicationJob
filename = "#{[log_type, 'logs', organisation&.name, Time.zone.now].compact.join('-')}.csv"
- storage_service = Storage::S3Service.new(Configuration::EnvConfigurationService.new, ENV["BULK_UPLOAD_BUCKET"])
+ storage_service = if FeatureToggle.upload_enabled?
+ Storage::S3Service.new(Configuration::EnvConfigurationService.new, ENV["BULK_UPLOAD_BUCKET"])
+ else
+ Storage::LocalDiskService.new
+ end
+
storage_service.write_file(filename, BYTE_ORDER_MARK + csv_string)
+ csv_download = CsvDownload.create!(user:, organisation: user.organisation, filename:, download_type: log_type, expiration_time: EXPIRATION_TIME)
- url = storage_service.get_presigned_url(filename, EXPIRATION_TIME)
+ url = csv_download_url(csv_download.id, host: ENV["APP_HOST"])
CsvDownloadMailer.new.send_csv_download_mail(user, url, EXPIRATION_TIME)
end
diff --git a/app/jobs/scheme_email_csv_job.rb b/app/jobs/scheme_email_csv_job.rb
index 44d016a90..803d3dce3 100644
--- a/app/jobs/scheme_email_csv_job.rb
+++ b/app/jobs/scheme_email_csv_job.rb
@@ -1,9 +1,10 @@
class SchemeEmailCsvJob < ApplicationJob
+ include Rails.application.routes.url_helpers
queue_as :default
BYTE_ORDER_MARK = "\uFEFF".freeze # Required to ensure Excel always reads CSV as UTF-8
- EXPIRATION_TIME = 24.hours.to_i
+ EXPIRATION_TIME = 48.hours.to_i
def perform(user, search_term = nil, filters = {}, all_orgs = false, organisation = nil, download_type = "combined") # rubocop:disable Style/OptionalBooleanParameter - sidekiq can't serialise named params
unfiltered_schemes = if organisation.present? && user.support?
@@ -23,10 +24,16 @@ class SchemeEmailCsvJob < ApplicationJob
filename = "#{['schemes-and-locations', organisation&.name, Time.zone.now].compact.join('-')}.csv"
end
- storage_service = Storage::S3Service.new(Configuration::EnvConfigurationService.new, ENV["BULK_UPLOAD_BUCKET"])
+ storage_service = if FeatureToggle.upload_enabled?
+ Storage::S3Service.new(Configuration::EnvConfigurationService.new, ENV["BULK_UPLOAD_BUCKET"])
+ else
+ Storage::LocalDiskService.new
+ end
+
storage_service.write_file(filename, BYTE_ORDER_MARK + csv_string)
+ csv_download = CsvDownload.create!(user:, organisation: user.organisation, filename:, download_type:, expiration_time: EXPIRATION_TIME)
- url = storage_service.get_presigned_url(filename, EXPIRATION_TIME)
+ url = csv_download_url(csv_download.id, host: ENV["APP_HOST"])
CsvDownloadMailer.new.send_csv_download_mail(user, url, EXPIRATION_TIME)
end
diff --git a/app/models/csv_download.rb b/app/models/csv_download.rb
new file mode 100644
index 000000000..4064c62f3
--- /dev/null
+++ b/app/models/csv_download.rb
@@ -0,0 +1,10 @@
+class CsvDownload < ApplicationRecord
+ enum download_type: { lettings: "lettings", sales: "sales", schemes: "schemes", locations: "locations", combined: "combined" }
+
+ belongs_to :user
+ belongs_to :organisation
+
+ def expired?
+ created_at < expiration_time.seconds.ago
+ end
+end
diff --git a/app/policies/csv_download_policy.rb b/app/policies/csv_download_policy.rb
new file mode 100644
index 000000000..04471ccd0
--- /dev/null
+++ b/app/policies/csv_download_policy.rb
@@ -0,0 +1,16 @@
+class CsvDownloadPolicy
+ attr_reader :current_user, :csv_download
+
+ def initialize(current_user, csv_download)
+ @current_user = current_user
+ @csv_download = csv_download
+ end
+
+ def show?
+ @current_user == @csv_download.user || @current_user.support? || @current_user.organisation == @csv_download.organisation
+ end
+
+ def download?
+ @current_user == @csv_download.user || @current_user.support? || @current_user.organisation == @csv_download.organisation
+ end
+end
diff --git a/app/services/csv/downloader.rb b/app/services/csv/downloader.rb
new file mode 100644
index 000000000..24545bc41
--- /dev/null
+++ b/app/services/csv/downloader.rb
@@ -0,0 +1,50 @@
+class Csv::Downloader
+ attr_reader :csv_download
+
+ delegate :path, to: :file
+
+ def initialize(csv_download:)
+ @csv_download = csv_download
+ end
+
+ def call
+ download
+ end
+
+ def delete_local_file!
+ file.unlink
+ end
+
+ def presigned_url
+ s3_storage_service.get_presigned_url(csv_download.filename, 60, response_content_disposition: "attachment; filename=#{csv_download.filename}")
+ end
+
+private
+
+ def download
+ io = storage_service.get_file_io(csv_download.filename)
+ file.write(io.read)
+ io.close
+ file.close
+ end
+
+ def file
+ @file ||= Tempfile.new
+ end
+
+ def storage_service
+ @storage_service ||= if FeatureToggle.upload_enabled?
+ s3_storage_service
+ else
+ local_disk_storage_service
+ end
+ end
+
+ def s3_storage_service
+ Storage::S3Service.new(Configuration::EnvConfigurationService.new, ENV["BULK_UPLOAD_BUCKET"])
+ end
+
+ def local_disk_storage_service
+ Storage::LocalDiskService.new
+ end
+end
diff --git a/app/views/csv_downloads/show.html.erb b/app/views/csv_downloads/show.html.erb
new file mode 100644
index 000000000..18f8a67fe
--- /dev/null
+++ b/app/views/csv_downloads/show.html.erb
@@ -0,0 +1,10 @@
+<% title = "Downlaod CSV file" %>
+<% content_for :title, title %>
+
+
+
+
You are about to download a CSV file
+
Filename: <%= @csv_download.filename %>
+ <%= govuk_button_link_to "Download CSV", download_csv_download_path(@csv_download) %>
+
+
diff --git a/app/views/errors/download_link_expired.html.erb b/app/views/errors/download_link_expired.html.erb
new file mode 100644
index 000000000..7477ee6b3
--- /dev/null
+++ b/app/views/errors/download_link_expired.html.erb
@@ -0,0 +1,8 @@
+<% content_for :title, "This link has expired" %>
+
+
+
+
This link has expired.
+
Download the logs again to get a new link.
+
+
diff --git a/config/routes.rb b/config/routes.rb
index 6ac7b3f34..1c7af8c59 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -382,6 +382,13 @@ Rails.application.routes.draw do
end
end
+ resources :csv_downloads, path: "csv-downloads" do
+ member do
+ get "/", to: "csv_downloads#show", as: "show"
+ get "download", to: "csv_downloads#download"
+ end
+ end
+
scope via: :all do
match "/404", to: "errors#not_found"
match "/429", to: "errors#too_many_requests", status: 429
diff --git a/db/migrate/20241118104046_add_csv_download_table.rb b/db/migrate/20241118104046_add_csv_download_table.rb
new file mode 100644
index 000000000..9b4f73f0b
--- /dev/null
+++ b/db/migrate/20241118104046_add_csv_download_table.rb
@@ -0,0 +1,12 @@
+class AddCsvDownloadTable < ActiveRecord::Migration[7.0]
+ def change
+ create_table :csv_downloads do |t|
+ t.column :download_type, :string
+ t.column :filename, :string
+ t.column :expiration_time, :integer
+ t.timestamps
+ t.references :user
+ t.references :organisation
+ end
+ end
+end
diff --git a/db/schema.rb b/db/schema.rb
index ef635628c..5b6ddacdb 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema[7.0].define(version: 2024_10_31_102744) do
+ActiveRecord::Schema[7.0].define(version: 2024_11_18_104046) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
@@ -64,6 +64,18 @@ ActiveRecord::Schema[7.0].define(version: 2024_10_31_102744) do
t.datetime "discarded_at"
end
+ create_table "csv_downloads", force: :cascade do |t|
+ t.string "download_type"
+ t.string "filename"
+ t.integer "expiration_time"
+ t.datetime "created_at", null: false
+ t.datetime "updated_at", null: false
+ t.bigint "user_id"
+ t.bigint "organisation_id"
+ t.index ["organisation_id"], name: "index_csv_downloads_on_organisation_id"
+ t.index ["user_id"], name: "index_csv_downloads_on_user_id"
+ end
+
create_table "csv_variable_definitions", force: :cascade do |t|
t.string "variable", null: false
t.string "definition", null: false
diff --git a/spec/factories/csv_download.rb b/spec/factories/csv_download.rb
new file mode 100644
index 000000000..415f69de6
--- /dev/null
+++ b/spec/factories/csv_download.rb
@@ -0,0 +1,9 @@
+FactoryBot.define do
+ factory :csv_download do
+ download_type { "lettings" }
+ user { create(:user) }
+ organisation { user.organisation }
+ filename { "lettings.csv" }
+ expiration_time { 24.hours.to_i }
+ end
+end
diff --git a/spec/jobs/email_csv_job_spec.rb b/spec/jobs/email_csv_job_spec.rb
index 8e8a027ea..59cf7bf7d 100644
--- a/spec/jobs/email_csv_job_spec.rb
+++ b/spec/jobs/email_csv_job_spec.rb
@@ -3,8 +3,6 @@ require "rails_helper"
describe EmailCsvJob do
include Helpers
- test_url = :test_url
-
let(:job) { described_class.new }
let(:user) { FactoryBot.create(:user) }
let(:storage_service) { instance_double(Storage::S3Service) }
@@ -22,7 +20,6 @@ describe EmailCsvJob do
before do
allow(Storage::S3Service).to receive(:new).and_return(storage_service)
allow(storage_service).to receive(:write_file)
- allow(storage_service).to receive(:get_presigned_url).and_return(test_url)
allow(Csv::SalesLogCsvService).to receive(:new).and_return(sales_log_csv_service)
allow(sales_log_csv_service).to receive(:prepare_csv).and_return("")
@@ -67,6 +64,16 @@ describe EmailCsvJob do
expect(lettings_log_csv_service).to receive(:prepare_csv).with(lettings_logs)
job.perform(user, nil, {}, nil, nil, codes_only_export)
end
+
+ it "creates a CsvDownload record" do
+ job.perform(user, nil, {}, nil, nil, codes_only_export, "lettings")
+ expect(CsvDownload.count).to eq(1)
+ expect(CsvDownload.first.user).to eq(user)
+ expect(CsvDownload.first.organisation).to eq(user.organisation)
+ expect(CsvDownload.first.filename).to match(/lettings-logs-.*\.csv/)
+ expect(CsvDownload.first.download_type).to eq("lettings")
+ expect(CsvDownload.first.expiration_time).to eq(172_800)
+ end
end
context "when exporting sales logs" do
@@ -102,10 +109,20 @@ describe EmailCsvJob do
expect(sales_log_csv_service).to receive(:prepare_csv).with(sales_logs)
job.perform(user, nil, {}, nil, nil, codes_only_export, "sales")
end
+
+ it "creates a CsvDownload record" do
+ job.perform(user, nil, {}, nil, nil, codes_only_export, "sales")
+ expect(CsvDownload.count).to eq(1)
+ expect(CsvDownload.first.user).to eq(user)
+ expect(CsvDownload.first.organisation).to eq(user.organisation)
+ expect(CsvDownload.first.filename).to match(/sales-logs-.*\.csv/)
+ expect(CsvDownload.first.download_type).to eq("sales")
+ expect(CsvDownload.first.expiration_time).to eq(172_800)
+ end
end
it "sends an E-mail with the presigned URL and duration" do
- expect(mailer).to receive(:send_csv_download_mail).with(user, test_url, instance_of(Integer))
+ expect(mailer).to receive(:send_csv_download_mail).with(user, /csv-downloads/, instance_of(Integer))
job.perform(user)
end
end
diff --git a/spec/jobs/scheme_email_csv_job_spec.rb b/spec/jobs/scheme_email_csv_job_spec.rb
index 5ddaa91a6..efb75b698 100644
--- a/spec/jobs/scheme_email_csv_job_spec.rb
+++ b/spec/jobs/scheme_email_csv_job_spec.rb
@@ -3,10 +3,8 @@ require "rails_helper"
describe SchemeEmailCsvJob do
include Helpers
- test_url = :test_url
-
let(:job) { described_class.new }
- let(:storage_service) { instance_double(Storage::S3Service, write_file: nil, get_presigned_url: test_url) }
+ let(:storage_service) { instance_double(Storage::S3Service, write_file: nil) }
let(:mailer) { instance_double(CsvDownloadMailer, send_csv_download_mail: nil) }
let(:user) { FactoryBot.create(:user) }
@@ -53,6 +51,16 @@ describe SchemeEmailCsvJob do
job.perform(user, nil, {}, nil, nil, download_type)
end
end
+
+ it "creates a CsvDownload record" do
+ job.perform(user, nil, {}, nil, nil, download_type)
+ expect(CsvDownload.count).to eq(1)
+ expect(CsvDownload.first.user).to eq(user)
+ expect(CsvDownload.first.organisation).to eq(user.organisation)
+ expect(CsvDownload.first.filename).to match(/schemes-.*\.csv/)
+ expect(CsvDownload.first.download_type).to eq("schemes")
+ expect(CsvDownload.first.expiration_time).to eq(172_800)
+ end
end
context "when download type locations" do
@@ -62,6 +70,16 @@ describe SchemeEmailCsvJob do
expect(storage_service).to receive(:write_file).with(/locations-.*\.csv/, anything)
job.perform(user, nil, {}, nil, nil, download_type)
end
+
+ it "creates a CsvDownload record" do
+ job.perform(user, nil, {}, nil, nil, download_type)
+ expect(CsvDownload.count).to eq(1)
+ expect(CsvDownload.first.user).to eq(user)
+ expect(CsvDownload.first.organisation).to eq(user.organisation)
+ expect(CsvDownload.first.filename).to match(/locations-.*\.csv/)
+ expect(CsvDownload.first.download_type).to eq("locations")
+ expect(CsvDownload.first.expiration_time).to eq(172_800)
+ end
end
context "when download type combined" do
@@ -71,6 +89,16 @@ describe SchemeEmailCsvJob do
expect(storage_service).to receive(:write_file).with(/schemes-and-locations.*\.csv/, anything)
job.perform(user, nil, {}, nil, nil, download_type)
end
+
+ it "creates a CsvDownload record" do
+ job.perform(user, nil, {}, nil, nil, download_type)
+ expect(CsvDownload.count).to eq(1)
+ expect(CsvDownload.first.user).to eq(user)
+ expect(CsvDownload.first.organisation).to eq(user.organisation)
+ expect(CsvDownload.first.filename).to match(/schemes-and-locations-.*\.csv/)
+ expect(CsvDownload.first.download_type).to eq("combined")
+ expect(CsvDownload.first.expiration_time).to eq(172_800)
+ end
end
it "includes the organisation name in the filename when one is provided" do
@@ -117,7 +145,7 @@ describe SchemeEmailCsvJob do
end
it "sends an E-mail with the presigned URL and duration" do
- expect(mailer).to receive(:send_csv_download_mail).with(user, test_url, instance_of(Integer))
+ expect(mailer).to receive(:send_csv_download_mail).with(user, /csv-downloads/, instance_of(Integer))
job.perform(user)
end
end
diff --git a/spec/requests/csv_downloads_controller_spec.rb b/spec/requests/csv_downloads_controller_spec.rb
new file mode 100644
index 000000000..982077a12
--- /dev/null
+++ b/spec/requests/csv_downloads_controller_spec.rb
@@ -0,0 +1,136 @@
+require "rails_helper"
+
+RSpec.describe CsvDownloadsController, type: :request do
+ describe "GET #show" do
+ let(:page) { Capybara::Node::Simple.new(response.body) }
+ let(:csv_user) { create(:user) }
+ let(:csv_download) { create(:csv_download, user: csv_user, organisation: csv_user.organisation) }
+ let(:get_file_io) do
+ io = StringIO.new
+ io.write("hello")
+ io.rewind
+ io
+ end
+ let(:mock_storage_service) { instance_double(Storage::S3Service, get_file_io:, get_presigned_url: "https://example.com") }
+
+ before do
+ allow(Storage::S3Service).to receive(:new).and_return(mock_storage_service)
+ end
+
+ context "when user is not signed in" do
+ it "redirects to sign in page" do
+ get "/csv-downloads/#{csv_download.id}"
+ expect(response).to redirect_to("/account/sign-in")
+ end
+ end
+
+ context "when user is signed in" do
+ before do
+ sign_in user
+ end
+
+ context "and the user is from a different organisation" do
+ let(:user) { create(:user) }
+
+ before do
+ get "/csv-downloads/#{csv_download.id}"
+ end
+
+ it "returns page not found" do
+ expect(response).to have_http_status(:unauthorized)
+ end
+ end
+
+ context "and is the user who generated the csv" do
+ let(:user) { csv_user }
+
+ before do
+ get "/csv-downloads/#{csv_download.id}"
+ end
+
+ it "allows downloading the csv" do
+ expect(response).to have_http_status(:ok)
+ expect(page).to have_link("Download CSV", href: "/csv-downloads/#{csv_download.id}/download")
+ end
+ end
+
+ context "and is the user is from the same organisation" do
+ let(:user) { create(:user, organisation: csv_user.organisation) }
+
+ before do
+ get "/csv-downloads/#{csv_download.id}"
+ end
+
+ it "allows downloading the csv" do
+ expect(response).to have_http_status(:ok)
+ expect(page).to have_link("Download CSV", href: "/csv-downloads/#{csv_download.id}/download")
+ end
+ end
+ end
+ end
+
+ describe "GET #download" do
+ let(:csv_user) { create(:user) }
+ let(:csv_download) { create(:csv_download, user: csv_user, organisation: csv_user.organisation) }
+ let(:get_file_io) do
+ io = StringIO.new
+ io.write("hello")
+ io.rewind
+ io
+ end
+ let(:mock_storage_service) { instance_double(Storage::S3Service, get_file_io:, get_presigned_url: "https://example.com") }
+
+ before do
+ allow(Storage::S3Service).to receive(:new).and_return(mock_storage_service)
+ end
+
+ context "when user is not signed in" do
+ it "redirects to sign in page" do
+ get "/csv-downloads/#{csv_download.id}/download"
+ expect(response).to redirect_to("/account/sign-in")
+ end
+ end
+
+ context "when user is signed in" do
+ before do
+ sign_in user
+ end
+
+ context "and the user is from a different organisation" do
+ let(:user) { create(:user) }
+
+ before do
+ get "/csv-downloads/#{csv_download.id}/download"
+ end
+
+ it "returns page not found" do
+ expect(response).to have_http_status(:unauthorized)
+ end
+ end
+
+ context "and is the user who generated the csv" do
+ let(:user) { csv_user }
+
+ before do
+ get "/csv-downloads/#{csv_download.id}/download"
+ end
+
+ it "allows downloading the csv" do
+ expect(response).to have_http_status(:found)
+ end
+ end
+
+ context "and is the user is from the same organisation" do
+ let(:user) { create(:user, organisation: csv_user.organisation) }
+
+ before do
+ get "/csv-downloads/#{csv_download.id}/download"
+ end
+
+ it "allows downloading the csv" do
+ expect(response).to have_http_status(:found)
+ end
+ end
+ end
+ end
+end