CLDC-3517 Download collection resources from S3 (#2671)

* Get collection resources from S3

* Remove resource files from codebase

* Stub some collection resource requests

* Use env var for bucket name

* Update some tests

* Extract methods into a service

app/controllers/start_controller.rb

@@ -1,4 +1,6 @@
 class StartController < ApplicationController
+  include CollectionResourcesHelper
+
   def index
     if current_user
       @homepage_presenter = HomepagePresenter.new(current_user)
@@ -7,114 +9,67 @@ class StartController < ApplicationController
   end
 
   def download_24_25_sales_form
-    send_file(
+    download_resource("2024_25_sales_paper_form.pdf", "2024-25 Sales paper form.pdf")
-      Rails.root.join("public/files/2024_25_sales_paper_form.pdf"),
-      filename: "2024-25 Sales paper form.pdf",
-      type: "application/pdf",
-    )
   end
 
   def download_23_24_sales_form
-    send_file(
+    download_resource("2023_24_sales_paper_form.pdf", "2023-24 Sales paper form.pdf")
-      Rails.root.join("public/files/2023_24_sales_paper_form.pdf"),
-      filename: "2023-24 Sales paper form.pdf",
-      type: "application/pdf",
-    )
   end
 
   def download_24_25_lettings_form
-    send_file(
+    download_resource("2024_25_lettings_paper_form.pdf", "2024-25 Lettings paper form.pdf")
-      Rails.root.join("public/files/2024_25_lettings_paper_form.pdf"),
-      filename: "2024-25 Lettings paper form.pdf",
-      type: "application/pdf",
-    )
   end
 
   def download_23_24_lettings_form
-    send_file(
+    download_resource("2023_24_lettings_paper_form.pdf", "2023-24 Lettings paper form.pdf")
-      Rails.root.join("public/files/2023_24_lettings_paper_form.pdf"),
-      filename: "2023-24 Lettings paper form.pdf",
-      type: "application/pdf",
-    )
   end
 
   def download_24_25_lettings_bulk_upload_template
-    send_file(
+    download_resource("bulk-upload-lettings-template-2024-25.xlsx", "2024-25-lettings-bulk-upload-template.xlsx")
-      Rails.root.join("public/files/bulk-upload-lettings-template-2024-25.xlsx"),
-      filename: "2024-25-lettings-bulk-upload-template.xlsx",
-      type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
-    )
   end
 
   def download_24_25_lettings_bulk_upload_specification
-    send_file(
+    download_resource("bulk-upload-lettings-specification-2024-25.xlsx", "2024-25-lettings-bulk-upload-specification.xlsx")
-      Rails.root.join("public/files/bulk-upload-lettings-specification-2024-25.xlsx"),
-      filename: "2024-25-lettings-bulk-upload-specification.xlsx",
-      type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
-    )
   end
 
   def download_24_25_sales_bulk_upload_template
-    send_file(
+    download_resource("bulk-upload-sales-template-2024-25.xlsx", "2024-25-sales-bulk-upload-template.xlsx")
-      Rails.root.join("public/files/bulk-upload-sales-template-2024-25.xlsx"),
-      filename: "2024-25-sales-bulk-upload-template.xlsx",
-      type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
-    )
   end
 
   def download_24_25_sales_bulk_upload_specification
-    send_file(
+    download_resource("bulk-upload-sales-specification-2024-25.xlsx", "2024-25-sales-bulk-upload-specification.xlsx")
-      Rails.root.join("public/files/bulk-upload-sales-specification-2024-25.xlsx"),
-      filename: "2024-25-sales-bulk-upload-specification.xlsx",
-      type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
-    )
   end
 
   def download_23_24_lettings_bulk_upload_template
-    send_file(
+    download_resource("bulk-upload-lettings-template-2023-24.xlsx", "2023-24-lettings-bulk-upload-template.xlsx")
-      Rails.root.join("public/files/bulk-upload-lettings-template-2023-24.xlsx"),
-      filename: "2023-24-lettings-bulk-upload-template.xlsx",
-      type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
-    )
   end
 
   def download_23_24_lettings_bulk_upload_legacy_template
-    send_file(
+    download_resource("bulk-upload-lettings-legacy-template-2023-24.xlsx", "2023-24-lettings-bulk-upload-legacy-template.xlsx")
-      Rails.root.join("public/files/bulk-upload-lettings-legacy-template-2023-24.xlsx"),
-      filename: "2023-24-lettings-bulk-upload-legacy-template.xlsx",
-      type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
-    )
   end
 
   def download_23_24_lettings_bulk_upload_specification
-    send_file(
+    download_resource("bulk-upload-lettings-specification-2023-24.xlsx", "2023-24-lettings-bulk-upload-specification.xlsx")
-      Rails.root.join("public/files/bulk-upload-lettings-specification-2023-24.xlsx"),
-      filename: "2023-24-lettings-bulk-upload-specification.xlsx",
-      type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
-    )
   end
 
   def download_23_24_sales_bulk_upload_template
-    send_file(
+    download_resource("bulk-upload-sales-template-2023-24.xlsx", "2023-24-sales-bulk-upload-template.xlsx")
-      Rails.root.join("public/files/bulk-upload-sales-template-2023-24.xlsx"),
-      filename: "2023-24-sales-bulk-upload-template.xlsx",
-      type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
-    )
   end
 
   def download_23_24_sales_bulk_upload_legacy_template
-    send_file(
+    download_resource("bulk-upload-sales-legacy-template-2023-24.xlsx", "2023-24-sales-bulk-upload-legacy-template.xlsx")
-      Rails.root.join("public/files/bulk-upload-sales-legacy-template-2023-24.xlsx"),
-      filename: "2023-24-sales-bulk-upload-legacy-template.xlsx",
-      type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
-    )
   end
 
   def download_23_24_sales_bulk_upload_specification
-    send_file(
+    download_resource("bulk-upload-sales-specification-2023-24.xlsx", "2023-24-sales-bulk-upload-specification.xlsx")
-      Rails.root.join("public/files/bulk-upload-sales-specification-2023-24.xlsx"),
+  end
-      filename: "2023-24-sales-bulk-upload-specification.xlsx",
+
-      type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+private
-    )
+
+  def download_resource(filename, download_filename)
+    file = CollectionResourcesService.new.get_file(filename)
+    render_not_found unless file
+
+    send_data(file, disposition: "attachment", filename: download_filename)
   end
 end

app/helpers/collection_resources_helper.rb

@@ -1,15 +1,22 @@
 module CollectionResourcesHelper
+  HUMAN_READABLE_CONTENT_TYPE = { "application/pdf": "PDF",
+                                  "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "Microsoft Excel",
+                                  "application/vnd.ms-excel": "Microsoft Excel (Old Format)",
+                                  "application/msword": "Microsoft Word",
+                                  "application/vnd.openxmlformats-officedocument.wordprocessingml.document": "Microsoft Word (DOCX)",
+                                  "image/jpeg": "JPEG Image",
+                                  "image/png": "PNG Image",
+                                  "text/plain": "Text Document",
+                                  "text/html": "HTML Document" }.freeze
+
   def file_type_size_and_pages(file, number_of_pages: nil)
-    extension_mapping = {
+    file_pages = number_of_pages ? pluralize(number_of_pages, "page") : nil
-      "xlsx" => "Microsoft Excel",
+    metadata = CollectionResourcesService.new.get_file_metadata(file)
-      "pdf" => "PDF",
-    }
-    extension = File.extname(file)[1..]
 
-    file_type = extension_mapping.fetch(extension, extension)
+    return [file_pages].compact.join(", ") unless metadata
 
-    file_size = number_to_human_size(File.size("public/files/#{file}"), precision: 0, significant: false)
+    file_size = number_to_human_size(metadata["Content-Length"].to_i)
-    file_pages = number_of_pages ? pluralize(number_of_pages, "page") : nil
+    file_type = HUMAN_READABLE_CONTENT_TYPE[metadata["Content-Type"].to_sym] || "Unknown File Type"
     [file_type, file_size, file_pages].compact.join(", ")
   end
 end

app/models/forms/bulk_upload_lettings/prepare_your_file.rb

@@ -35,25 +35,25 @@ module Forms
       def legacy_template_path
         case year
         when 2023
-          "/files/bulk-upload-lettings-legacy-template-2023-24.xlsx"
+          download_23_24_lettings_bulk_upload_legacy_template_path
         end
       end
 
       def template_path
         case year
         when 2023
-          "/files/bulk-upload-lettings-template-2023-24.xlsx"
+          download_23_24_lettings_bulk_upload_template_path
         when 2024
-          "/files/bulk-upload-lettings-template-2024-25.xlsx"
+          download_24_25_lettings_bulk_upload_template_path
         end
       end
 
       def specification_path
         case year
         when 2023
-          "/files/bulk-upload-lettings-specification-2023-24.xlsx"
+          download_23_24_lettings_bulk_upload_specification_path
         when 2024
-          "/files/bulk-upload-lettings-specification-2024-25.xlsx"
+          download_24_25_lettings_bulk_upload_specification_path
         end
       end
 

app/models/forms/bulk_upload_sales/prepare_your_file.rb

@@ -34,25 +34,25 @@ module Forms
       def legacy_template_path
         case year
         when 2023
-          "/files/bulk-upload-sales-legacy-template-2023-24.xlsx"
+          download_23_24_sales_bulk_upload_legacy_template_path
         end
       end
 
       def template_path
         case year
         when 2023
-          "/files/bulk-upload-sales-template-2023-24.xlsx"
+          download_23_24_sales_bulk_upload_template_path
         when 2024
-          "/files/bulk-upload-sales-template-2024-25.xlsx"
+          download_24_25_sales_bulk_upload_template_path
         end
       end
 
       def specification_path
         case year
         when 2023
-          "/files/bulk-upload-sales-specification-2023-24.xlsx"
+          download_23_24_sales_bulk_upload_specification_path
         when 2024
-          "/files/bulk-upload-sales-specification-2024-25.xlsx"
+          download_24_25_sales_bulk_upload_specification_path
         end
       end
 

app/services/collection_resources_service.rb

@@ -0,0 +1,33 @@
+class CollectionResourcesService
+  def initialize
+    @storage_service = Storage::S3Service.new(Configuration::EnvConfigurationService.new, ENV["COLLECTION_RESOURCES_BUCKET"])
+  end
+
+  def get_file(file)
+    storage_service = Storage::S3Service.new(Configuration::EnvConfigurationService.new, ENV["COLLECTION_RESOURCES_BUCKET"])
+    url = "https://#{storage_service.configuration.bucket_name}.s3.amazonaws.com/#{file}"
+    uri = URI.parse(url)
+
+    response = Net::HTTP.start(uri.host, uri.port, use_ssl: true) do |http|
+      request = Net::HTTP::Get.new(uri)
+      http.request(request)
+    end
+
+    return unless response.is_a?(Net::HTTPSuccess)
+
+    response.body
+  end
+
+  def get_file_metadata(file)
+    storage_service = Storage::S3Service.new(Configuration::EnvConfigurationService.new, ENV["COLLECTION_RESOURCES_BUCKET"])
+    url = "https://#{storage_service.configuration.bucket_name}.s3.amazonaws.com/#{file}"
+    uri = URI.parse(url)
+
+    response = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |http|
+      http.request_head(uri)
+    end
+    return unless response.is_a?(Net::HTTPSuccess) || response.is_a?(Net::HTTPRedirection)
+
+    response
+  end
+end

spec/features/accessibility_spec.rb

@@ -3,6 +3,7 @@ require "rails_helper"
 RSpec.describe "Accessibility", js: true do
   let(:user) { create(:user, :support) }
   let!(:other_user) { create(:user, name: "new user", organisation: user.organisation, email: "new_user@example.com", confirmation_token: "abc") }
+  let(:storage_service) { instance_double(Storage::S3Service) }
 
   def find_routes(type, resource, subresource)
     routes = Rails.application.routes.routes.select do |route|
@@ -20,6 +21,8 @@ RSpec.describe "Accessibility", js: true do
   end
 
   before do
+    allow(Storage::S3Service).to receive(:new).and_return(storage_service)
+    allow(storage_service).to receive(:configuration).and_return(OpenStruct.new(bucket_name: "core-test-collection-resources"))
     allow(user).to receive(:need_two_factor_authentication?).and_return(false)
     sign_in(user)
   end

spec/features/lettings_log_spec.rb

@@ -1,6 +1,13 @@
 require "rails_helper"
 
 RSpec.describe "Lettings Log Features" do
+  let(:storage_service) { instance_double(Storage::S3Service) }
+
+  before do
+    allow(Storage::S3Service).to receive(:new).and_return(storage_service)
+    allow(storage_service).to receive(:configuration).and_return(OpenStruct.new(bucket_name: "core-test-collection-resources"))
+  end
+
   context "when searching for specific logs" do
     context "when I am signed in and there are logs in the database" do
       let(:user) { create(:user, last_sign_in_at: Time.zone.now) }

spec/features/notifications_spec.rb

@@ -3,6 +3,12 @@ require_relative "form/helpers"
 
 RSpec.describe "Notifications Features" do
   include Helpers
+  let(:storage_service) { instance_double(Storage::S3Service) }
+
+  before do
+    allow(Storage::S3Service).to receive(:new).and_return(storage_service)
+    allow(storage_service).to receive(:configuration).and_return(OpenStruct.new(bucket_name: "core-test-collection-resources"))
+  end
 
   context "when there are notifications" do
     let!(:user) { FactoryBot.create(:user) }

spec/features/organisation_spec.rb

@@ -10,8 +10,11 @@ RSpec.describe "User Features" do
   let(:notify_client) { instance_double(Notifications::Client) }
   let(:confirmation_token) { "MCDH5y6Km-U7CFPgAMVS" }
   let(:devise_notify_mailer) { DeviseNotifyMailer.new }
+  let(:storage_service) { instance_double(Storage::S3Service) }
 
   before do
+    allow(Storage::S3Service).to receive(:new).and_return(storage_service)
+    allow(storage_service).to receive(:configuration).and_return(OpenStruct.new(bucket_name: "core-test-collection-resources"))
     allow(DeviseNotifyMailer).to receive(:new).and_return(devise_notify_mailer)
     allow(devise_notify_mailer).to receive(:notify_client).and_return(notify_client)
     allow(Devise).to receive(:friendly_token).and_return(confirmation_token)

spec/features/start_page_spec.rb

@@ -4,6 +4,12 @@ require_relative "form/helpers"
 RSpec.describe "Start Page Features" do
   include Helpers
   let(:user) { FactoryBot.create(:user) }
+  let(:storage_service) { instance_double(Storage::S3Service) }
+
+  before do
+    allow(Storage::S3Service).to receive(:new).and_return(storage_service)
+    allow(storage_service).to receive(:configuration).and_return(OpenStruct.new(bucket_name: "core-test-collection-resources"))
+  end
 
   context "when the user is signed in" do
     before do

spec/features/test_spec.rb

@@ -1,5 +1,12 @@
 require "rails_helper"
 RSpec.describe "Test Features" do
+  let(:storage_service) { instance_double(Storage::S3Service) }
+
+  before do
+    allow(Storage::S3Service).to receive(:new).and_return(storage_service)
+    allow(storage_service).to receive(:configuration).and_return(OpenStruct.new(bucket_name: "core-test-collection-resources"))
+  end
+
   it "Displays the name of the app" do
     visit(root_path)
     expect(page).to have_content("Submit social housing lettings and sales data (CORE)")

spec/features/user_spec.rb

@@ -6,12 +6,15 @@ RSpec.describe "User Features" do
   let(:notify_client) { instance_double(Notifications::Client) }
   let(:reset_password_token) { "MCDH5y6Km-U7CFPgAMVS" }
   let(:devise_notify_mailer) { DeviseNotifyMailer.new }
+  let(:storage_service) { instance_double(Storage::S3Service) }
 
   before do
     allow(DeviseNotifyMailer).to receive(:new).and_return(devise_notify_mailer)
     allow(devise_notify_mailer).to receive(:notify_client).and_return(notify_client)
     allow(notify_client).to receive(:send_email).and_return(true)
     allow(Devise.token_generator).to receive(:generate).and_return(reset_password_token)
+    allow(Storage::S3Service).to receive(:new).and_return(storage_service)
+    allow(storage_service).to receive(:configuration).and_return(OpenStruct.new(bucket_name: "core-test-collection-resources"))
   end
 
   context "when the user navigates to lettings logs" do

spec/helpers/collection_resources_helper_spec.rb

@@ -3,15 +3,31 @@ require "rails_helper"
 RSpec.describe CollectionResourcesHelper do
   let(:current_user) { create(:user, :data_coordinator) }
   let(:user) { create(:user, :data_coordinator) }
+  let(:storage_service) { instance_double(Storage::S3Service) }
+
+  before do
+    allow(Storage::S3Service).to receive(:new).and_return(storage_service)
+    allow(storage_service).to receive(:configuration).and_return(OpenStruct.new(bucket_name: "core-test-collection-resources"))
+  end
 
   describe "when displaying file metadata" do
     context "with pages" do
+      before do
+        stub_request(:head, "https://core-test-collection-resources.s3.amazonaws.com/2023_24_lettings_paper_form.pdf")
+          .to_return(status: 200, body: "", headers: { "Content-Length" => 292_864, "Content-Type" => "application/pdf" })
+      end
+
       it "returns correct metadata" do
         expect(file_type_size_and_pages("2023_24_lettings_paper_form.pdf", number_of_pages: 8)).to eq("PDF, 286 KB, 8 pages")
       end
     end
 
     context "without pages" do
+      before do
+        stub_request(:head, "https://core-test-collection-resources.s3.amazonaws.com/bulk-upload-lettings-template-2023-24.xlsx")
+          .to_return(status: 200, body: "", headers: { "Content-Length" => 19_456, "Content-Type" => "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" })
+      end
+
       it "returns correct metadata" do
         expect(file_type_size_and_pages("bulk-upload-lettings-template-2023-24.xlsx")).to eq("Microsoft Excel, 19 KB")
       end

spec/request_helper.rb

@@ -103,6 +103,9 @@ module RequestHelper
       address = request.uri.query_values["query"].split(",")
       { status: 200, body: { results: [{ DPA: { MATCH: 0.9, BUILDING_NAME: "result #{address[0]}", POST_TOWN: "result town or city", POSTCODE: address[1], UPRN: "1" } }] }.to_json, headers: {} }
     end
+
+    WebMock.stub_request(:head, /https:\/\/core-test-collection-resources\.s3\.amazonaws\.com/)
+         .to_return(status: 200, body: "", headers: { "Content-Type" => "application/pdf", "Content-Length" => 1000 })
   end
 
   def self.real_http_requests

spec/requests/auth/passwords_controller_spec.rb

@@ -5,11 +5,14 @@ RSpec.describe Auth::PasswordsController, type: :request do
   let(:page) { Capybara::Node::Simple.new(response.body) }
   let(:notify_client) { instance_double(Notifications::Client) }
   let(:devise_notify_mailer) { DeviseNotifyMailer.new }
+  let(:storage_service) { instance_double(Storage::S3Service) }
 
   before do
     allow(DeviseNotifyMailer).to receive(:new).and_return(devise_notify_mailer)
     allow(devise_notify_mailer).to receive(:notify_client).and_return(notify_client)
     allow(notify_client).to receive(:send_email).and_return(true)
+    allow(Storage::S3Service).to receive(:new).and_return(storage_service)
+    allow(storage_service).to receive(:configuration).and_return(OpenStruct.new(bucket_name: "core-test-collection-resources"))
   end
 
   context "when a regular user" do

spec/requests/maintenance_controller_spec.rb

@@ -3,8 +3,11 @@ require "rails_helper"
 RSpec.describe MaintenanceController, type: :request do
   let(:page) { Capybara::Node::Simple.new(response.body) }
   let(:user) { FactoryBot.create(:user) }
+  let(:storage_service) { instance_double(Storage::S3Service) }
 
   before do
+    allow(Storage::S3Service).to receive(:new).and_return(storage_service)
+    allow(storage_service).to receive(:configuration).and_return(OpenStruct.new(bucket_name: "core-test-collection-resources"))
     sign_in user
   end
 

spec/requests/rails_admin_controller_spec.rb

@@ -4,9 +4,12 @@ RSpec.describe "RailsAdmin", type: :request do
   let(:user) { create(:user) }
   let(:support_user) { create(:user, :support) }
   let(:page) { Capybara::Node::Simple.new(response.body) }
+  let(:storage_service) { instance_double(Storage::S3Service) }
 
   before do
     allow(support_user).to receive(:need_two_factor_authentication?).and_return(false)
+    allow(Storage::S3Service).to receive(:new).and_return(storage_service)
+    allow(storage_service).to receive(:configuration).and_return(OpenStruct.new(bucket_name: "core-test-collection-resources"))
   end
 
   describe "GET /admin" do

spec/requests/start_controller_spec.rb

@@ -5,11 +5,14 @@ RSpec.describe StartController, type: :request do
   let(:page) { Capybara::Node::Simple.new(response.body) }
   let(:notify_client) { instance_double(Notifications::Client) }
   let(:devise_notify_mailer) { DeviseNotifyMailer.new }
+  let(:storage_service) { instance_double(Storage::S3Service) }
 
   before do
     allow(DeviseNotifyMailer).to receive(:new).and_return(devise_notify_mailer)
     allow(devise_notify_mailer).to receive(:notify_client).and_return(notify_client)
     allow(notify_client).to receive(:send_email).and_return(true)
+    allow(Storage::S3Service).to receive(:new).and_return(storage_service)
+    allow(storage_service).to receive(:configuration).and_return(OpenStruct.new(bucket_name: "core-test-collection-resources"))
   end
 
   describe "GET" do

spec/requests/users_controller_spec.rb

@@ -10,11 +10,14 @@ RSpec.describe UsersController, type: :request do
   let(:params) { { id: user.id, user: { name: new_name } } }
   let(:notify_client) { instance_double(Notifications::Client) }
   let(:devise_notify_mailer) { DeviseNotifyMailer.new }
+  let(:storage_service) { instance_double(Storage::S3Service) }
 
   before do
     allow(DeviseNotifyMailer).to receive(:new).and_return(devise_notify_mailer)
     allow(devise_notify_mailer).to receive(:notify_client).and_return(notify_client)
     allow(notify_client).to receive(:send_email).and_return(true)
+    allow(Storage::S3Service).to receive(:new).and_return(storage_service)
+    allow(storage_service).to receive(:configuration).and_return(OpenStruct.new(bucket_name: "core-test-collection-resources"))
   end
 
   context "when user is not signed in" do