Refactor collection resources to use storage service (#2684)

* Refactor collection resources to use storage service

* Add a feature flag

* Update tests

* Update stubs

app/controllers/start_controller.rb

@@ -68,7 +68,7 @@ private
 
   def download_resource(filename, download_filename)
     file = CollectionResourcesService.new.get_file(filename)
-    render_not_found unless file
+    return render_not_found unless file
 
     send_data(file, disposition: "attachment", filename: download_filename)
   end

app/helpers/collection_resources_helper.rb

@@ -15,8 +15,8 @@ module CollectionResourcesHelper
 
     return [file_pages].compact.join(", ") unless metadata
 
-    file_size = number_to_human_size(metadata["Content-Length"].to_i)
+    file_size = number_to_human_size(metadata["content_length"].to_i)
-    file_type = HUMAN_READABLE_CONTENT_TYPE[metadata["Content-Type"].to_sym] || "Unknown File Type"
+    file_type = HUMAN_READABLE_CONTENT_TYPE[metadata["content_type"].to_sym] || "Unknown File Type"
     [file_type, file_size, file_pages].compact.join(", ")
   end
 end

app/services/collection_resources_service.rb

@@ -1,33 +1,21 @@
 class CollectionResourcesService
   def initialize
-    @storage_service = Storage::S3Service.new(Configuration::EnvConfigurationService.new, ENV["COLLECTION_RESOURCES_BUCKET"])
+    @storage_service = if FeatureToggle.local_storage?
+                         Storage::LocalDiskService.new
+                       else
+                         Storage::S3Service.new(Configuration::EnvConfigurationService.new, ENV["COLLECTION_RESOURCES_BUCKET"])
+                       end
   end
 
   def get_file(file)
-    storage_service = Storage::S3Service.new(Configuration::EnvConfigurationService.new, ENV["COLLECTION_RESOURCES_BUCKET"])
+    @storage_service.get_file_io(file)
-    url = "https://#{storage_service.configuration.bucket_name}.s3.amazonaws.com/#{file}"
+  rescue StandardError
-    uri = URI.parse(url)
+    nil
-
-    response = Net::HTTP.start(uri.host, uri.port, use_ssl: true) do |http|
-      request = Net::HTTP::Get.new(uri)
-      http.request(request)
-    end
-
-    return unless response.is_a?(Net::HTTPSuccess)
-
-    response.body
   end
 
   def get_file_metadata(file)
-    storage_service = Storage::S3Service.new(Configuration::EnvConfigurationService.new, ENV["COLLECTION_RESOURCES_BUCKET"])
+    @storage_service.get_file_metadata(file)
-    url = "https://#{storage_service.configuration.bucket_name}.s3.amazonaws.com/#{file}"
+  rescue StandardError
-    uri = URI.parse(url)
+    nil
-
-    response = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |http|
-      http.request_head(uri)
-    end
-    return unless response.is_a?(Net::HTTPSuccess) || response.is_a?(Net::HTTPRedirection)
-
-    response
   end
 end

app/services/feature_toggle.rb

@@ -34,4 +34,8 @@ class FeatureToggle
   def self.delete_user_enabled?
     true
   end
+
+  def self.local_storage?
+    Rails.env.development?
+  end
 end

app/services/storage/local_disk_service.rb

@@ -22,5 +22,14 @@ module Storage
         f.write data
       end
     end
+
+    def get_file_metadata(filename)
+      path = Rails.root.join("tmp/storage", filename)
+
+      {
+        "content_length" => File.size(path),
+        "content_type" => MiniMime.lookup_by_filename(path.to_s)&.content_type || "application/octet-stream",
+      }
+    end
   end
 end

app/services/storage/s3_service.rb

@@ -39,6 +39,10 @@ module Storage
       )
     end
 
+    def get_file_metadata(file_name)
+      @client.head_object(bucket: @configuration.bucket_name, key: file_name)
+    end
+
   private
 
     def create_configuration

spec/features/accessibility_spec.rb

@@ -3,7 +3,7 @@ require "rails_helper"
 RSpec.describe "Accessibility", js: true do
   let(:user) { create(:user, :support) }
   let!(:other_user) { create(:user, name: "new user", organisation: user.organisation, email: "new_user@example.com", confirmation_token: "abc") }
-  let(:storage_service) { instance_double(Storage::S3Service) }
+  let(:storage_service) { instance_double(Storage::S3Service, get_file_metadata: nil) }
 
   def find_routes(type, resource, subresource)
     routes = Rails.application.routes.routes.select do |route|

spec/features/lettings_log_spec.rb

@@ -1,7 +1,7 @@
 require "rails_helper"
 
 RSpec.describe "Lettings Log Features" do
-  let(:storage_service) { instance_double(Storage::S3Service) }
+  let(:storage_service) { instance_double(Storage::S3Service, get_file_metadata: nil) }
 
   before do
     allow(Storage::S3Service).to receive(:new).and_return(storage_service)

spec/features/notifications_spec.rb

@@ -3,7 +3,7 @@ require_relative "form/helpers"
 
 RSpec.describe "Notifications Features" do
   include Helpers
-  let(:storage_service) { instance_double(Storage::S3Service) }
+  let(:storage_service) { instance_double(Storage::S3Service, get_file_metadata: nil) }
 
   before do
     allow(Storage::S3Service).to receive(:new).and_return(storage_service)

spec/features/organisation_spec.rb

@@ -10,7 +10,7 @@ RSpec.describe "User Features" do
   let(:notify_client) { instance_double(Notifications::Client) }
   let(:confirmation_token) { "MCDH5y6Km-U7CFPgAMVS" }
   let(:devise_notify_mailer) { DeviseNotifyMailer.new }
-  let(:storage_service) { instance_double(Storage::S3Service) }
+  let(:storage_service) { instance_double(Storage::S3Service, get_file_metadata: nil) }
 
   before do
     allow(Storage::S3Service).to receive(:new).and_return(storage_service)

spec/features/start_page_spec.rb

@@ -4,7 +4,7 @@ require_relative "form/helpers"
 RSpec.describe "Start Page Features" do
   include Helpers
   let(:user) { FactoryBot.create(:user) }
-  let(:storage_service) { instance_double(Storage::S3Service) }
+  let(:storage_service) { instance_double(Storage::S3Service, get_file_metadata: nil) }
 
   before do
     allow(Storage::S3Service).to receive(:new).and_return(storage_service)

spec/features/test_spec.rb

@@ -1,6 +1,6 @@
 require "rails_helper"
 RSpec.describe "Test Features" do
-  let(:storage_service) { instance_double(Storage::S3Service) }
+  let(:storage_service) { instance_double(Storage::S3Service, get_file_metadata: nil) }
 
   before do
     allow(Storage::S3Service).to receive(:new).and_return(storage_service)

spec/features/user_spec.rb

@@ -6,7 +6,7 @@ RSpec.describe "User Features" do
   let(:notify_client) { instance_double(Notifications::Client) }
   let(:reset_password_token) { "MCDH5y6Km-U7CFPgAMVS" }
   let(:devise_notify_mailer) { DeviseNotifyMailer.new }
-  let(:storage_service) { instance_double(Storage::S3Service) }
+  let(:storage_service) { instance_double(Storage::S3Service, get_file_metadata: nil) }
 
   before do
     allow(DeviseNotifyMailer).to receive(:new).and_return(devise_notify_mailer)

spec/helpers/collection_resources_helper_spec.rb

@@ -3,7 +3,7 @@ require "rails_helper"
 RSpec.describe CollectionResourcesHelper do
   let(:current_user) { create(:user, :data_coordinator) }
   let(:user) { create(:user, :data_coordinator) }
-  let(:storage_service) { instance_double(Storage::S3Service) }
+  let(:storage_service) { instance_double(Storage::S3Service, get_file_metadata: nil) }
 
   before do
     allow(Storage::S3Service).to receive(:new).and_return(storage_service)
@@ -13,8 +13,7 @@ RSpec.describe CollectionResourcesHelper do
   describe "when displaying file metadata" do
     context "with pages" do
       before do
-        stub_request(:head, "https://core-test-collection-resources.s3.amazonaws.com/2023_24_lettings_paper_form.pdf")
+        allow(storage_service).to receive(:get_file_metadata).with("2023_24_lettings_paper_form.pdf").and_return("content_length" => 292_864, "content_type" => "application/pdf")
-          .to_return(status: 200, body: "", headers: { "Content-Length" => 292_864, "Content-Type" => "application/pdf" })
       end
 
       it "returns correct metadata" do
@@ -24,8 +23,7 @@ RSpec.describe CollectionResourcesHelper do
 
     context "without pages" do
       before do
-        stub_request(:head, "https://core-test-collection-resources.s3.amazonaws.com/bulk-upload-lettings-template-2023-24.xlsx")
+        allow(storage_service).to receive(:get_file_metadata).with("bulk-upload-lettings-template-2023-24.xlsx").and_return("content_length" => 19_456, "content_type" => "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
-          .to_return(status: 200, body: "", headers: { "Content-Length" => 19_456, "Content-Type" => "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" })
       end
 
       it "returns correct metadata" do

spec/request_helper.rb

@@ -103,9 +103,6 @@ module RequestHelper
       address = request.uri.query_values["query"].split(",")
       { status: 200, body: { results: [{ DPA: { MATCH: 0.9, BUILDING_NAME: "result #{address[0]}", POST_TOWN: "result town or city", POSTCODE: address[1], UPRN: "1" } }] }.to_json, headers: {} }
     end
-
-    WebMock.stub_request(:head, /https:\/\/core-test-collection-resources\.s3\.amazonaws\.com/)
-         .to_return(status: 200, body: "", headers: { "Content-Type" => "application/pdf", "Content-Length" => 1000 })
   end
 
   def self.real_http_requests

spec/requests/auth/passwords_controller_spec.rb

@@ -5,7 +5,7 @@ RSpec.describe Auth::PasswordsController, type: :request do
   let(:page) { Capybara::Node::Simple.new(response.body) }
   let(:notify_client) { instance_double(Notifications::Client) }
   let(:devise_notify_mailer) { DeviseNotifyMailer.new }
-  let(:storage_service) { instance_double(Storage::S3Service) }
+  let(:storage_service) { instance_double(Storage::S3Service, get_file_metadata: nil) }
 
   before do
     allow(DeviseNotifyMailer).to receive(:new).and_return(devise_notify_mailer)

spec/requests/maintenance_controller_spec.rb

@@ -3,7 +3,7 @@ require "rails_helper"
 RSpec.describe MaintenanceController, type: :request do
   let(:page) { Capybara::Node::Simple.new(response.body) }
   let(:user) { FactoryBot.create(:user) }
-  let(:storage_service) { instance_double(Storage::S3Service) }
+  let(:storage_service) { instance_double(Storage::S3Service, get_file_metadata: nil) }
 
   before do
     allow(Storage::S3Service).to receive(:new).and_return(storage_service)

spec/requests/rails_admin_controller_spec.rb

@@ -4,7 +4,7 @@ RSpec.describe "RailsAdmin", type: :request do
   let(:user) { create(:user) }
   let(:support_user) { create(:user, :support) }
   let(:page) { Capybara::Node::Simple.new(response.body) }
-  let(:storage_service) { instance_double(Storage::S3Service) }
+  let(:storage_service) { instance_double(Storage::S3Service, get_file_metadata: nil) }
 
   before do
     allow(support_user).to receive(:need_two_factor_authentication?).and_return(false)

spec/requests/start_controller_spec.rb

@@ -5,7 +5,7 @@ RSpec.describe StartController, type: :request do
   let(:page) { Capybara::Node::Simple.new(response.body) }
   let(:notify_client) { instance_double(Notifications::Client) }
   let(:devise_notify_mailer) { DeviseNotifyMailer.new }
-  let(:storage_service) { instance_double(Storage::S3Service) }
+  let(:storage_service) { instance_double(Storage::S3Service, get_file_metadata: nil) }
 
   before do
     allow(DeviseNotifyMailer).to receive(:new).and_return(devise_notify_mailer)

spec/requests/users_controller_spec.rb

@@ -10,7 +10,7 @@ RSpec.describe UsersController, type: :request do
   let(:params) { { id: user.id, user: { name: new_name } } }
   let(:notify_client) { instance_double(Notifications::Client) }
   let(:devise_notify_mailer) { DeviseNotifyMailer.new }
-  let(:storage_service) { instance_double(Storage::S3Service) }
+  let(:storage_service) { instance_double(Storage::S3Service, get_file_metadata: nil) }
 
   before do
     allow(DeviseNotifyMailer).to receive(:new).and_return(devise_notify_mailer)