CLDC-1520: Support external S3 configuration (#835)

* Move storage classes into a dedicated module
* Move configuration classes into a dedicated module
* Refactor configuration service
* Implement configuration via env variables

app/services/archive_storage_service.rb

@@ -1,24 +0,0 @@
-class ArchiveStorageService < StorageService
-  MAX_SIZE = 50 * (1024**2) # 50MiB
-
-  def initialize(archive_io)
-    super()
-    @archive = Zip::File.open_buffer(archive_io)
-  end
-
-  def list_files(folder)
-    @archive.glob(File.join(folder, "*.*"))
-            .map(&:name)
-  end
-
-  def folder_present?(folder)
-    !list_files(folder).empty?
-  end
-
-  def get_file_io(file_name)
-    entry = @archive.get_entry(file_name)
-    raise "File too large to be extracted" if entry.size > MAX_SIZE
-
-    entry.get_input_stream
-  end
-end

app/services/configuration/configuration_service.rb

@@ -0,0 +1,54 @@
+module Configuration
+  class ConfigurationService
+    attr_reader :s3_buckets, :redis_uris
+
+    def initialize(logger = Rails.logger)
+      @logger = logger
+      @config = read_config
+      @s3_buckets = read_s3_buckets
+      @redis_uris = read_redis_uris
+    end
+
+    def s3_config_present?
+      config_present? && @config.key?(:"aws-s3-bucket")
+    end
+
+    def redis_config_present?
+      config_present? && @config.key?(:redis)
+    end
+
+  private
+
+    def config_present?
+      raise NotImplementedError
+    end
+
+    def read_config
+      raise NotImplementedError
+    end
+
+    def read_s3_buckets
+      return {} unless s3_config_present?
+
+      s3_buckets = {}
+      @config[:"aws-s3-bucket"].each do |bucket_config|
+        if bucket_config.key?(:instance_name)
+          s3_buckets[bucket_config[:instance_name].to_sym] = bucket_config
+        end
+      end
+      s3_buckets
+    end
+
+    def read_redis_uris
+      return {} unless redis_config_present?
+
+      redis_uris = {}
+      @config[:redis].each do |redis_config|
+        if redis_config.key?(:instance_name)
+          redis_uris[redis_config[:instance_name].to_sym] = redis_config.dig(:credentials, :uri)
+        end
+      end
+      redis_uris
+    end
+  end
+end

app/services/configuration/env_configuration_service.rb

@@ -0,0 +1,37 @@
+module Configuration
+  class EnvConfigurationService < ConfigurationService
+  private
+
+    def config_present?
+      !ENV["S3_CONFIG"].nil? || !ENV["REDIS_CONFIG"].nil?
+    end
+
+    def read_config
+      unless config_present?
+        @logger.warn("Could not find S3_CONFIG or REDIS_CONFIG in the environment variables!")
+        return {}
+      end
+
+      config = {}
+      assign_config(config, :"aws-s3-bucket", "S3_CONFIG")
+      assign_config(config, :redis, "REDIS_CONFIG")
+      config
+    end
+
+    def assign_config(config, symbol, env_variable)
+      config_hash = parse_json_config(env_variable)
+      config[symbol] = config_hash unless config_hash.empty?
+    end
+
+    def parse_json_config(env_variable_name)
+      if ENV[env_variable_name].present?
+        begin
+          return JSON.parse(ENV[env_variable_name], { symbolize_names: true })
+        rescue StandardError
+          @logger.warn("Could not parse #{env_variable_name}!")
+        end
+      end
+      {}
+    end
+  end
+end

app/services/configuration/paas_configuration_service.rb

@@ -0,0 +1,23 @@
+module Configuration
+  class PaasConfigurationService < ConfigurationService
+  private
+
+    def config_present?
+      !ENV["VCAP_SERVICES"].nil?
+    end
+
+    def read_config
+      unless config_present?
+        @logger.warn("Could not find VCAP_SERVICES in the environment variables!")
+        return {}
+      end
+
+      begin
+        JSON.parse(ENV["VCAP_SERVICES"], { symbolize_names: true })
+      rescue StandardError
+        @logger.warn("Could not parse VCAP_SERVICES!")
+        {}
+      end
+    end
+  end
+end

app/services/paas_configuration_service.rb

@@ -1,62 +0,0 @@
-class PaasConfigurationService
-  attr_reader :s3_buckets, :redis_uris
-
-  def initialize(logger = Rails.logger)
-    @logger = logger
-    @paas_config = read_pass_config
-    @s3_buckets = read_s3_buckets
-    @redis_uris = read_redis_uris
-  end
-
-  def config_present?
-    !ENV["VCAP_SERVICES"].nil?
-  end
-
-  def s3_config_present?
-    config_present? && @paas_config.key?(:"aws-s3-bucket")
-  end
-
-  def redis_config_present?
-    config_present? && @paas_config.key?(:redis)
-  end
-
-private
-
-  def read_pass_config
-    unless config_present?
-      @logger.warn("Could not find VCAP_SERVICES in the environment!")
-      return {}
-    end
-
-    begin
-      JSON.parse(ENV["VCAP_SERVICES"], { symbolize_names: true })
-    rescue StandardError
-      @logger.warn("Could not parse VCAP_SERVICES!")
-      {}
-    end
-  end
-
-  def read_s3_buckets
-    return {} unless s3_config_present?
-
-    s3_buckets = {}
-    @paas_config[:"aws-s3-bucket"].each do |bucket_config|
-      if bucket_config.key?(:instance_name)
-        s3_buckets[bucket_config[:instance_name].to_sym] = bucket_config
-      end
-    end
-    s3_buckets
-  end
-
-  def read_redis_uris
-    return {} unless redis_config_present?
-
-    redis_uris = {}
-    @paas_config[:redis].each do |redis_config|
-      if redis_config.key?(:instance_name)
-        redis_uris[redis_config[:instance_name].to_sym] = redis_config.dig(:credentials, :uri)
-      end
-    end
-    redis_uris
-  end
-end

app/services/s3_storage_service.rb

@@ -1,78 +0,0 @@
-class S3StorageService < StorageService
-  attr_reader :configuration
-
-  def initialize(paas_config_service, paas_instance_name)
-    super()
-    @paas_config_service = paas_config_service
-    @paas_instance_name = (paas_instance_name || "").to_sym
-    @configuration = create_configuration
-    @client = create_client
-  end
-
-  def list_files(folder)
-    @client.list_objects_v2(bucket: @configuration.bucket_name, prefix: folder)
-           .flat_map { |response| response.contents.map(&:key) }
-  end
-
-  def folder_present?(folder)
-    response = @client.list_objects_v2(bucket: @configuration.bucket_name, prefix: folder, max_keys: 1)
-    response.key_count == 1
-  end
-
-  def get_file_io(file_name)
-    @client.get_object(bucket: @configuration.bucket_name, key: file_name)
-           .body
-  end
-
-  def write_file(file_name, data)
-    @client.put_object(
-      body: data,
-      bucket: @configuration.bucket_name,
-      key: file_name,
-    )
-  end
-
-private
-
-  def create_configuration
-    unless @paas_config_service.config_present?
-      raise "No PaaS configuration present"
-    end
-    unless @paas_config_service.s3_buckets.key?(@paas_instance_name)
-      raise "#{@paas_instance_name} instance name could not be found"
-    end
-
-    bucket_config = @paas_config_service.s3_buckets[@paas_instance_name]
-    StorageConfiguration.new(bucket_config[:credentials])
-  end
-
-  def create_client
-    credentials =
-      Aws::Credentials.new(
-        @configuration.access_key_id,
-        @configuration.secret_access_key,
-      )
-    Aws::S3::Client.new(
-      region: @configuration.region,
-      credentials:,
-    )
-  end
-end
-
-class StorageConfiguration
-  attr_reader :access_key_id, :secret_access_key, :bucket_name, :region
-
-  def initialize(credentials)
-    @access_key_id = credentials[:aws_access_key_id]
-    @secret_access_key = credentials[:aws_secret_access_key]
-    @bucket_name = credentials[:bucket_name]
-    @region = credentials[:aws_region]
-  end
-
-  def ==(other)
-    @access_key_id == other.access_key_id &&
-      @secret_access_key == other.secret_access_key &&
-      @bucket_name == other.bucket_name &&
-      @region == other.region
-  end
-end

app/services/storage/archive_service.rb

@@ -0,0 +1,26 @@
+module Storage
+  class ArchiveService < StorageService
+    MAX_SIZE = 50 * (1024**2) # 50MiB
+
+    def initialize(archive_io)
+      super()
+      @archive = Zip::File.open_buffer(archive_io)
+    end
+
+    def list_files(folder)
+      @archive.glob(File.join(folder, "*.*"))
+              .map(&:name)
+    end
+
+    def folder_present?(folder)
+      !list_files(folder).empty?
+    end
+
+    def get_file_io(file_name)
+      entry = @archive.get_entry(file_name)
+      raise "File too large to be extracted" if entry.size > MAX_SIZE
+
+      entry.get_input_stream
+    end
+  end
+end

app/services/storage/s3_service.rb

@@ -0,0 +1,80 @@
+module Storage
+  class S3Service < StorageService
+    attr_reader :configuration
+
+    def initialize(config_service, paas_instance_name)
+      super()
+      @config_service = config_service
+      @instance_name = (paas_instance_name || "").to_sym
+      @configuration = create_configuration
+      @client = create_client
+    end
+
+    def list_files(folder)
+      @client.list_objects_v2(bucket: @configuration.bucket_name, prefix: folder)
+             .flat_map { |response| response.contents.map(&:key) }
+    end
+
+    def folder_present?(folder)
+      response = @client.list_objects_v2(bucket: @configuration.bucket_name, prefix: folder, max_keys: 1)
+      response.key_count == 1
+    end
+
+    def get_file_io(file_name)
+      @client.get_object(bucket: @configuration.bucket_name, key: file_name)
+             .body
+    end
+
+    def write_file(file_name, data)
+      @client.put_object(
+        body: data,
+        bucket: @configuration.bucket_name,
+        key: file_name,
+      )
+    end
+
+  private
+
+    def create_configuration
+      unless @config_service.s3_config_present?
+        raise "No S3 bucket is present in the PaaS configuration"
+      end
+      unless @config_service.s3_buckets.key?(@instance_name)
+        raise "#{@instance_name} instance name could not be found"
+      end
+
+      bucket_config = @config_service.s3_buckets[@instance_name]
+      StorageConfiguration.new(bucket_config[:credentials])
+    end
+
+    def create_client
+      credentials =
+        Aws::Credentials.new(
+          @configuration.access_key_id,
+          @configuration.secret_access_key,
+        )
+      Aws::S3::Client.new(
+        region: @configuration.region,
+        credentials:,
+      )
+    end
+  end
+
+  class StorageConfiguration
+    attr_reader :access_key_id, :secret_access_key, :bucket_name, :region
+
+    def initialize(credentials)
+      @access_key_id = credentials[:aws_access_key_id]
+      @secret_access_key = credentials[:aws_secret_access_key]
+      @bucket_name = credentials[:bucket_name]
+      @region = credentials[:aws_region]
+    end
+
+    def ==(other)
+      @access_key_id == other.access_key_id &&
+        @secret_access_key == other.secret_access_key &&
+        @bucket_name == other.bucket_name &&
+        @region == other.region
+    end
+  end
+end

app/services/storage/storage_service.rb

@@ -0,0 +1,19 @@
+module Storage
+  class StorageService
+    def list_files(_folder)
+      raise NotImplementedError
+    end
+
+    def folder_present?(_folder)
+      raise NotImplementedError
+    end
+
+    def get_file_io(_file_name)
+      raise NotImplementedError
+    end
+
+    def write_file(_file_name, _data)
+      raise NotImplementedError
+    end
+  end
+end

app/services/storage_service.rb

@@ -1,17 +0,0 @@
-class StorageService
-  def list_files(_folder)
-    raise NotImplementedError
-  end
-
-  def folder_present?(_folder)
-    raise NotImplementedError
-  end
-
-  def get_file_io(_file_name)
-    raise NotImplementedError
-  end
-
-  def write_file(_file_name, _data)
-    raise NotImplementedError
-  end
-end

config/initializers/rack_attack.rb

@@ -1,10 +1,11 @@
-require "paas_configuration_service"
+require "configuration/configuration_service"
+require "configuration/paas_configuration_service"
 
 if Rails.env.development? || Rails.env.test?
   Rack::Attack.cache.store = ActiveSupport::Cache::MemoryStore.new
   Rack::Attack.enabled = false
 else
-  redis_url = PaasConfigurationService.new.redis_uris[:"dluhc-core-#{Rails.env}-redis-rate-limit"]
+  redis_url = Configuration::PaasConfigurationService.new.redis_uris[:"dluhc-core-#{Rails.env}-redis-rate-limit"]
   Rack::Attack.cache.store = ActiveSupport::Cache::RedisCacheStore.new(url: redis_url)
 end
 

lib/tasks/data_export.rake

@@ -4,7 +4,7 @@ namespace :core do
     format = args[:format]
     full_update = args[:full_update].present? && args[:full_update] == "true"
 
-    storage_service = S3StorageService.new(PaasConfigurationService.new, ENV["EXPORT_PAAS_INSTANCE"])
+    storage_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["EXPORT_PAAS_INSTANCE"])
     export_service = Exports::CaseLogExportService.new(storage_service)
 
     if format.present? && format == "CSV"

lib/tasks/data_import.rake

@@ -5,7 +5,7 @@ namespace :core do
     path = args[:path]
     raise "Usage: rake core:data_import['data_type', 'path/to/xml_files']" if path.blank? || type.blank?
 
-    storage_service = S3StorageService.new(PaasConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
+    storage_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
 
     case type
     when "organisation"

lib/tasks/data_import_field.rake

@@ -5,7 +5,7 @@ namespace :core do
     path = args[:path]
     raise "Usage: rake core:data_import_field['field','path/to/xml_files']" if path.blank? || field.blank?
 
-    storage_service = S3StorageService.new(PaasConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
+    storage_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
 
     # We only allow a reduced list of known fields to be updatable
     case field

lib/tasks/full_import.rake

@@ -6,9 +6,9 @@ namespace :core do
     archive_path = args[:archive_path]
     raise "Usage: rake core:full_import['path/to/archive']" if archive_path.blank?
 
-    s3_service = S3StorageService.new(PaasConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
+    s3_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
     archive_io = s3_service.get_file_io(archive_path)
-    archive_service = ArchiveStorageService.new(archive_io)
+    archive_service = Storage::ArchiveService.new(archive_io)
 
     import_list = [
       Import.new(Imports::OrganisationImportService, :create_organisations, "institution"),

spec/lib/tasks/data_export_spec.rb

@@ -5,8 +5,8 @@ describe "rake core:data_export", type: task do
   subject(:task) { Rake::Task["core:data_export"] }
 
   let(:paas_instance) { "paas_export_instance" }
-  let(:storage_service) { instance_double(S3StorageService) }
+  let(:storage_service) { instance_double(Storage::S3Service) }
-  let(:paas_config_service) { instance_double(PaasConfigurationService) }
+  let(:paas_config_service) { instance_double(Configuration::PaasConfigurationService) }
   let(:export_service) { instance_double(Exports::CaseLogExportService) }
 
   before do
@@ -14,8 +14,8 @@ describe "rake core:data_export", type: task do
     Rake::Task.define_task(:environment)
     task.reenable
 
-    allow(S3StorageService).to receive(:new).and_return(storage_service)
+    allow(Storage::S3Service).to receive(:new).and_return(storage_service)
-    allow(PaasConfigurationService).to receive(:new).and_return(paas_config_service)
+    allow(Configuration::PaasConfigurationService).to receive(:new).and_return(paas_config_service)
     allow(Exports::CaseLogExportService).to receive(:new).and_return(export_service)
     allow(ENV).to receive(:[])
     allow(ENV).to receive(:[]).with("EXPORT_PAAS_INSTANCE").and_return(paas_instance)
@@ -23,7 +23,7 @@ describe "rake core:data_export", type: task do
 
   context "when exporting case logs with no parameters" do
     it "starts the XML export process" do
-      expect(S3StorageService).to receive(:new).with(paas_config_service, paas_instance)
+      expect(Storage::S3Service).to receive(:new).with(paas_config_service, paas_instance)
       expect(Exports::CaseLogExportService).to receive(:new).with(storage_service)
       expect(export_service).to receive(:export_xml_case_logs)
 
@@ -33,7 +33,7 @@ describe "rake core:data_export", type: task do
 
   context "when exporting case logs with CSV format" do
     it "starts the CSV export process" do
-      expect(S3StorageService).to receive(:new).with(paas_config_service, paas_instance)
+      expect(Storage::S3Service).to receive(:new).with(paas_config_service, paas_instance)
       expect(Exports::CaseLogExportService).to receive(:new).with(storage_service)
       expect(export_service).to receive(:export_csv_case_logs)
 

spec/lib/tasks/data_import_spec.rb

@@ -5,16 +5,16 @@ describe "rake core:data_import", type: :task do
   subject(:task) { Rake::Task["core:data_import"] }
 
   let(:instance_name) { "paas_import_instance" }
-  let(:storage_service) { instance_double(S3StorageService) }
+  let(:storage_service) { instance_double(Storage::S3Service) }
-  let(:paas_config_service) { instance_double(PaasConfigurationService) }
+  let(:paas_config_service) { instance_double(Configuration::PaasConfigurationService) }
 
   before do
     Rake.application.rake_require("tasks/data_import")
     Rake::Task.define_task(:environment)
     task.reenable
 
-    allow(S3StorageService).to receive(:new).and_return(storage_service)
+    allow(Storage::S3Service).to receive(:new).and_return(storage_service)
-    allow(PaasConfigurationService).to receive(:new).and_return(paas_config_service)
+    allow(Configuration::PaasConfigurationService).to receive(:new).and_return(paas_config_service)
     allow(ENV).to receive(:[])
     allow(ENV).to receive(:[]).with("IMPORT_PAAS_INSTANCE").and_return(instance_name)
   end
@@ -29,7 +29,7 @@ describe "rake core:data_import", type: :task do
     end
 
     it "creates an organisation from the given XML file" do
-      expect(S3StorageService).to receive(:new).with(paas_config_service, instance_name)
+      expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name)
       expect(Imports::OrganisationImportService).to receive(:new).with(storage_service)
       expect(import_service).to receive(:create_organisations).with(fixture_path)
 
@@ -47,7 +47,7 @@ describe "rake core:data_import", type: :task do
     end
 
     it "creates a user from the given XML file" do
-      expect(S3StorageService).to receive(:new).with(paas_config_service, instance_name)
+      expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name)
       expect(Imports::UserImportService).to receive(:new).with(storage_service)
       expect(import_service).to receive(:create_users).with(fixture_path)
 
@@ -65,7 +65,7 @@ describe "rake core:data_import", type: :task do
     end
 
     it "creates an organisation from the given XML file" do
-      expect(S3StorageService).to receive(:new).with(paas_config_service, instance_name)
+      expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name)
       expect(Imports::DataProtectionConfirmationImportService).to receive(:new).with(storage_service)
       expect(import_service).to receive(:create_data_protection_confirmations).with(fixture_path)
 
@@ -83,7 +83,7 @@ describe "rake core:data_import", type: :task do
     end
 
     it "creates an organisation la from the given XML file" do
-      expect(S3StorageService).to receive(:new).with(paas_config_service, instance_name)
+      expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name)
       expect(Imports::OrganisationRentPeriodImportService).to receive(:new).with(storage_service)
       expect(import_service).to receive(:create_organisation_rent_periods).with(fixture_path)
 
@@ -101,7 +101,7 @@ describe "rake core:data_import", type: :task do
     end
 
     it "creates case logs from the given XML file" do
-      expect(S3StorageService).to receive(:new).with(paas_config_service, instance_name)
+      expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name)
       expect(Imports::CaseLogsImportService).to receive(:new).with(storage_service)
       expect(import_service).to receive(:create_logs).with(fixture_path)
 
@@ -119,7 +119,7 @@ describe "rake core:data_import", type: :task do
     end
 
     it "creates a scheme from the given XML file" do
-      expect(S3StorageService).to receive(:new).with(paas_config_service, instance_name)
+      expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name)
       expect(Imports::SchemeImportService).to receive(:new).with(storage_service)
       expect(import_service).to receive(:create_schemes).with(fixture_path)
 
@@ -137,7 +137,7 @@ describe "rake core:data_import", type: :task do
     end
 
     it "creates a scheme location from the given XML file" do
-      expect(S3StorageService).to receive(:new).with(paas_config_service, instance_name)
+      expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name)
       expect(Imports::SchemeLocationImportService).to receive(:new).with(storage_service)
       expect(import_service).to receive(:create_scheme_locations).with(fixture_path)
 

spec/lib/tasks/date_import_field_spec.rb

@@ -5,16 +5,16 @@ describe "rake core:data_import_field", type: :task do
   subject(:task) { Rake::Task["core:data_import_field"] }
 
   let(:instance_name) { "paas_import_instance" }
-  let(:storage_service) { instance_double(S3StorageService) }
+  let(:storage_service) { instance_double(Storage::S3Service) }
-  let(:paas_config_service) { instance_double(PaasConfigurationService) }
+  let(:paas_config_service) { instance_double(Configuration::PaasConfigurationService) }
 
   before do
     Rake.application.rake_require("tasks/data_import_field")
     Rake::Task.define_task(:environment)
     task.reenable
 
-    allow(S3StorageService).to receive(:new).and_return(storage_service)
+    allow(Storage::S3Service).to receive(:new).and_return(storage_service)
-    allow(PaasConfigurationService).to receive(:new).and_return(paas_config_service)
+    allow(Configuration::PaasConfigurationService).to receive(:new).and_return(paas_config_service)
     allow(ENV).to receive(:[])
     allow(ENV).to receive(:[]).with("IMPORT_PAAS_INSTANCE").and_return(instance_name)
   end
@@ -32,7 +32,7 @@ describe "rake core:data_import_field", type: :task do
       let(:field) { "tenant_code" }
 
       it "properly configures the storage service" do
-        expect(S3StorageService).to receive(:new).with(paas_config_service, instance_name)
+        expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name)
         task.invoke(field, fixture_path)
       end
 
@@ -46,7 +46,7 @@ describe "rake core:data_import_field", type: :task do
       let(:field) { "lettings_allocation" }
 
       it "properly configures the storage service" do
-        expect(S3StorageService).to receive(:new).with(paas_config_service, instance_name)
+        expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name)
         task.invoke(field, fixture_path)
       end
 
@@ -60,7 +60,7 @@ describe "rake core:data_import_field", type: :task do
       let(:field) { "major_repairs" }
 
       it "properly configures the storage service" do
-        expect(S3StorageService).to receive(:new).with(paas_config_service, instance_name)
+        expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name)
         task.invoke(field, fixture_path)
       end
 

spec/lib/tasks/full_import_spec.rb

@@ -5,19 +5,19 @@ require "zip"
 describe "rake core:full_import", type: :task do
   subject(:task) { Rake::Task["core:full_import"] }
 
-  let(:s3_service) { instance_double(S3StorageService) }
+  let(:s3_service) { instance_double(Storage::S3Service) }
-  let(:archive_service) { instance_double(ArchiveStorageService) }
+  let(:archive_service) { instance_double(Storage::ArchiveService) }
-  let(:paas_config_service) { instance_double(PaasConfigurationService) }
+  let(:paas_config_service) { instance_double(Configuration::PaasConfigurationService) }
 
   before do
     Rake.application.rake_require("tasks/full_import")
     Rake::Task.define_task(:environment)
     task.reenable
 
-    allow(PaasConfigurationService).to receive(:new).and_return(paas_config_service)
+    allow(Configuration::PaasConfigurationService).to receive(:new).and_return(paas_config_service)
-    allow(S3StorageService).to receive(:new).and_return(s3_service)
+    allow(Storage::S3Service).to receive(:new).and_return(s3_service)
     allow(s3_service).to receive(:get_file_io)
-    allow(ArchiveStorageService).to receive(:new).and_return(archive_service)
+    allow(Storage::ArchiveService).to receive(:new).and_return(archive_service)
   end
 
   context "when starting a full import" do

spec/services/configuration/env_configuration_service_spec.rb

@@ -0,0 +1,128 @@
+require "rails_helper"
+
+RSpec.describe Configuration::EnvConfigurationService do
+  subject(:config_service) { described_class.new(logger) }
+
+  let(:logger) { instance_double(ActiveSupport::LogSubscriber) }
+
+  context "when environment configurations are unavailable" do
+    before { allow(logger).to receive(:warn) }
+
+    it "returns the S3 configuration as not present" do
+      expect(config_service.s3_config_present?).to be(false)
+    end
+
+    it "returns the redis configuration as not present" do
+      expect(config_service.redis_config_present?).to be(false)
+    end
+
+    it "does not retrieve any S3 bucket configuration" do
+      expect(config_service.s3_buckets).to be_a(Hash)
+      expect(config_service.s3_buckets).to be_empty
+    end
+
+    it "does not retrieve any redis configuration" do
+      expect(config_service.redis_uris).to be_a(Hash)
+      expect(config_service.redis_uris).to be_empty
+    end
+  end
+
+  context "when environment configurations are present but invalid" do
+    let(:env_variable) { "random text" }
+
+    before do
+      allow(ENV).to receive(:[]).with("S3_CONFIG").and_return(env_variable)
+      allow(ENV).to receive(:[]).with("REDIS_CONFIG").and_return(env_variable)
+      allow(logger).to receive(:warn)
+    end
+
+    it "logs an error when checking if the S3 config is present" do
+      expect(logger).to receive(:warn).with("Could not parse S3_CONFIG!")
+      config_service.s3_config_present?
+    end
+
+    it "logs an error when checking if the Redis config is present" do
+      expect(logger).to receive(:warn).with("Could not parse REDIS_CONFIG!")
+      config_service.redis_config_present?
+    end
+  end
+
+  context "when environment configurations are present with S3 configured" do
+    let(:s3_config) do
+      <<~JSON
+        [
+          {
+            "instance_name": "bucket_1",
+            "credentials": {
+              "aws_access_key_id": "123",
+              "aws_secret_access_key": "456",
+              "aws_region": "eu-west-1",
+              "bucket_name": "my-bucket"
+            }
+          },
+          {
+            "instance_name": "bucket_2",
+            "credentials": {
+              "aws_access_key_id": "789",
+              "aws_secret_access_key": "012",
+              "aws_region": "eu-west-2",
+              "bucket_name": "my-bucket2"
+            }
+          }
+        ]
+      JSON
+    end
+
+    before do
+      allow(ENV).to receive(:[]).with("REDIS_CONFIG")
+      allow(ENV).to receive(:[]).with("S3_CONFIG").and_return(s3_config)
+    end
+
+    it "returns the S3 configuration as present" do
+      expect(config_service.s3_config_present?).to be(true)
+    end
+
+    it "returns the redis configuration as not present" do
+      expect(config_service.redis_config_present?).to be(false)
+    end
+
+    it "does retrieve the S3 bucket configurations" do
+      s3_buckets = config_service.s3_buckets
+
+      expect(s3_buckets).not_to be_empty
+      expect(s3_buckets.count).to be(2)
+      expect(s3_buckets).to have_key(:bucket_1)
+      expect(s3_buckets).to have_key(:bucket_2)
+    end
+  end
+
+  context "when environment configurations are present with redis configured" do
+    let(:redis_config) do
+      <<-JSON
+        [{"instance_name": "redis_1", "credentials": {"uri": "redis_uri" }}]
+      JSON
+    end
+
+    before do
+      allow(ENV).to receive(:[]).with("S3_CONFIG")
+      allow(ENV).to receive(:[]).with("REDIS_CONFIG").and_return(redis_config)
+    end
+
+    it "returns the redis configuration as present" do
+      expect(config_service.redis_config_present?).to be(true)
+    end
+
+    it "returns the S3 configuration as not present" do
+      expect(config_service.s3_config_present?).to be(false)
+    end
+
+    it "does retrieve the redis configurations" do
+      redis_uris = config_service.redis_uris
+
+      expect(redis_uris).not_to be_empty
+      expect(redis_uris.count).to be(1)
+      expect(redis_uris).to have_key(:redis_1)
+      expect(redis_uris[:redis_1]).to eq("redis_uri")
+    end
+  end
+end

spec/services/configuration/paas_configuration_service_spec.rb

@@ -1,6 +1,6 @@
 require "rails_helper"
 
-RSpec.describe PaasConfigurationService do
+RSpec.describe Configuration::PaasConfigurationService do
   subject(:config_service) { described_class.new(logger) }
 
   let(:logger) { instance_double(ActiveSupport::LogSubscriber) }
@@ -8,14 +8,38 @@ RSpec.describe PaasConfigurationService do
   context "when the paas configuration is unavailable" do
     before { allow(logger).to receive(:warn) }
 
-    it "returns the configuration as not present" do
+    it "returns the S3 configuration as not present" do
-      expect(config_service.config_present?).to be(false)
+      expect(config_service.s3_config_present?).to be(false)
+    end
+
+    it "returns the redis configuration as not present" do
+      expect(config_service.redis_config_present?).to be(false)
+    end
+
+    it "does not retrieve any S3 bucket configuration" do
+      expect(config_service.s3_buckets).to be_a(Hash)
+      expect(config_service.s3_buckets).to be_empty
+    end
+
+    it "does not retrieve any redis configuration" do
+      expect(config_service.redis_uris).to be_a(Hash)
+      expect(config_service.redis_uris).to be_empty
+    end
+  end
+
+  context "when the paas configuration is present but empty" do
+    before do
+      allow(ENV).to receive(:[]).with("VCAP_SERVICES").and_return("{}")
     end
 
     it "returns the S3 configuration as not present" do
       expect(config_service.s3_config_present?).to be(false)
     end
 
+    it "returns the redis configuration as not present" do
+      expect(config_service.redis_config_present?).to be(false)
+    end
+
     it "does not retrieve any S3 bucket configuration" do
       expect(config_service.s3_buckets).to be_a(Hash)
       expect(config_service.s3_buckets).to be_empty
@@ -27,7 +51,7 @@ RSpec.describe PaasConfigurationService do
     end
   end
 
-  context "when configuration is present but invalid" do
+  context "when the paas configuration is present but invalid" do
     let(:vcap_services) { "random text" }
 
     before do
@@ -35,16 +59,40 @@ RSpec.describe PaasConfigurationService do
       allow(logger).to receive(:warn)
     end
 
-    it "logs an error" do
+    it "logs an error when checking if the S3 config is present" do
       expect(logger).to receive(:warn).with("Could not parse VCAP_SERVICES!")
       config_service.s3_config_present?
     end
+
+    it "logs an error when checking if the Redis config is present" do
+      expect(logger).to receive(:warn).with("Could not parse VCAP_SERVICES!")
+      config_service.redis_config_present?
+    end
   end
 
-  context "when the paas configuration is present with S3 buckets" do
+  context "when the paas configuration is present with S3 configured" do
     let(:vcap_services) do
-      <<-JSON
+      <<~JSON
-        {"aws-s3-bucket": [{"instance_name": "bucket_1"},{"instance_name": "bucket_2"}]}
+        {"aws-s3-bucket":
+          [{
+            "instance_name": "bucket_1",
+            "credentials": {
+              "aws_access_key_id": "123",
+              "aws_secret_access_key": "456",
+              "aws_region": "eu-west-1",
+              "bucket_name": "my-bucket"
+            }
+          },
+          {
+            "instance_name": "bucket_2",
+            "credentials": {
+              "aws_access_key_id": "789",
+              "aws_secret_access_key": "012",
+              "aws_region": "eu-west-2",
+              "bucket_name": "my-bucket2"
+            }
+          }]
+        }
       JSON
     end
 
@@ -52,14 +100,14 @@ RSpec.describe PaasConfigurationService do
       allow(ENV).to receive(:[]).with("VCAP_SERVICES").and_return(vcap_services)
     end
 
-    it "returns the configuration as present" do
-      expect(config_service.config_present?).to be(true)
-    end
-
     it "returns the S3 configuration as present" do
       expect(config_service.s3_config_present?).to be(true)
     end
 
+    it "returns the redis configuration as not present" do
+      expect(config_service.redis_config_present?).to be(false)
+    end
+
     it "does retrieve the S3 bucket configurations" do
       s3_buckets = config_service.s3_buckets
 
@@ -70,26 +118,7 @@ RSpec.describe PaasConfigurationService do
     end
   end
 
-  context "when the paas configuration is present without S3 buckets" do
+  context "when the paas configuration is present with redis configured" do
-    before do
-      allow(ENV).to receive(:[]).with("VCAP_SERVICES").and_return("{}")
-    end
-
-    it "returns the configuration as present" do
-      expect(config_service.config_present?).to be(true)
-    end
-
-    it "returns the S3 configuration as not present" do
-      expect(config_service.s3_config_present?).to be(false)
-    end
-
-    it "does not retrieve any S3 bucket configuration" do
-      expect(config_service.s3_buckets).to be_a(Hash)
-      expect(config_service.s3_buckets).to be_empty
-    end
-  end
-
-  context "when the paas configuration is present with redis configurations" do
     let(:vcap_services) do
       <<-JSON
         {"redis": [{"instance_name": "redis_1", "credentials": {"uri": "redis_uri" }}]}
@@ -100,14 +129,14 @@ RSpec.describe PaasConfigurationService do
       allow(ENV).to receive(:[]).with("VCAP_SERVICES").and_return(vcap_services)
     end
 
-    it "returns the configuration as present" do
-      expect(config_service.config_present?).to be(true)
-    end
-
     it "returns the redis configuration as present" do
       expect(config_service.redis_config_present?).to be(true)
     end
 
+    it "returns the S3 configuration as not present" do
+      expect(config_service.s3_config_present?).to be(false)
+    end
+
     it "does retrieve the redis configurations" do
       redis_uris = config_service.redis_uris
 
@@ -117,23 +146,4 @@ RSpec.describe PaasConfigurationService do
       expect(redis_uris[:redis_1]).to eq("redis_uri")
     end
   end
-
-  context "when the paas configuration is present without redis configuration" do
-    before do
-      allow(ENV).to receive(:[]).with("VCAP_SERVICES").and_return("{}")
-    end
-
-    it "returns the configuration as present" do
-      expect(config_service.config_present?).to be(true)
-    end
-
-    it "returns the redis configuration as not present" do
-      expect(config_service.redis_config_present?).to be(false)
-    end
-
-    it "does not retrieve any redis uris from configuration" do
-      expect(config_service.redis_uris).to be_a(Hash)
-      expect(config_service.redis_uris).to be_empty
-    end
-  end
 end

spec/services/exports/case_log_export_service_spec.rb

@@ -3,7 +3,7 @@ require "rails_helper"
 RSpec.describe Exports::CaseLogExportService do
   subject(:export_service) { described_class.new(storage_service) }
 
-  let(:storage_service) { instance_double(S3StorageService) }
+  let(:storage_service) { instance_double(Storage::S3Service) }
 
   let(:xml_export_file) { File.open("spec/fixtures/exports/general_needs_log.xml", "r:UTF-8") }
   let(:local_manifest_file) { File.open("spec/fixtures/exports/manifest.xml", "r:UTF-8") }

spec/services/imports/case_logs_field_import_service_spec.rb

@@ -3,7 +3,7 @@ require "rails_helper"
 RSpec.describe Imports::CaseLogsFieldImportService do
   subject(:import_service) { described_class.new(storage_service, logger) }
 
-  let(:storage_service) { instance_double(S3StorageService) }
+  let(:storage_service) { instance_double(Storage::S3Service) }
   let(:logger) { instance_double(ActiveSupport::Logger) }
 
   let(:real_2021_2022_form) { Form.new("config/forms/2021_2022.json", "2021_2022") }

spec/services/imports/case_logs_import_service_spec.rb

@@ -3,7 +3,7 @@ require "rails_helper"
 RSpec.describe Imports::CaseLogsImportService do
   subject(:case_log_service) { described_class.new(storage_service, logger) }
 
-  let(:storage_service) { instance_double(S3StorageService) }
+  let(:storage_service) { instance_double(Storage::S3Service) }
   let(:logger) { instance_double(ActiveSupport::Logger) }
 
   let(:real_2021_2022_form) { Form.new("config/forms/2021_2022.json", "2021_2022") }

spec/services/imports/data_protection_confirmation_import_service_spec.rb

@@ -5,7 +5,7 @@ RSpec.describe Imports::DataProtectionConfirmationImportService do
   let(:old_org_id) { "7c5bd5fb549c09a2c55d7cb90d7ba84927e64618" }
   let(:old_id) { old_org_id }
   let(:import_file) { File.open("#{fixture_directory}/#{old_id}.xml") }
-  let(:storage_service) { instance_double(S3StorageService) }
+  let(:storage_service) { instance_double(Storage::S3Service) }
   let(:logger) { instance_double(ActiveSupport::Logger) }
 
   context "when importing data protection confirmations" do

spec/services/imports/organisation_import_service_spec.rb

@@ -1,7 +1,7 @@
 require "rails_helper"
 
 RSpec.describe Imports::OrganisationImportService do
-  let(:storage_service) { instance_double(S3StorageService) }
+  let(:storage_service) { instance_double(Storage::S3Service) }
   let(:logger) { instance_double(Rails::Rack::Logger) }
   let(:folder_name) { "organisations" }
   let(:filenames) { %w[my_folder/my_file1.xml my_folder/my_file2.xml] }

spec/services/imports/organisation_rent_period_import_service_spec.rb

@@ -5,7 +5,7 @@ RSpec.describe Imports::OrganisationRentPeriodImportService do
   let(:old_org_id) { "44026acc7ed5c29516b26f2a5deb639e5e37966d" }
   let(:old_id) { "ebd22326d33e389e9f1bfd546979d2c05f9e68d6" }
   let(:import_file) { File.open("#{fixture_directory}/#{old_id}.xml") }
-  let(:storage_service) { instance_double(S3StorageService) }
+  let(:storage_service) { instance_double(Storage::S3Service) }
   let(:logger) { instance_double(ActiveSupport::Logger) }
 
   context "when importing organisation rent periods" do

spec/services/imports/scheme_import_service_spec.rb

@@ -3,7 +3,7 @@ require "rails_helper"
 RSpec.describe Imports::SchemeImportService do
   subject(:scheme_service) { described_class.new(storage_service, logger) }
 
-  let(:storage_service) { instance_double(S3StorageService) }
+  let(:storage_service) { instance_double(Storage::S3Service) }
   let(:logger) { instance_double(ActiveSupport::Logger) }
 
   let(:fixture_directory) { "spec/fixtures/imports/mgmtgroups" }

spec/services/imports/scheme_location_import_service_spec.rb

@@ -3,7 +3,7 @@ require "rails_helper"
 RSpec.describe Imports::SchemeLocationImportService do
   subject(:location_service) { described_class.new(storage_service, logger) }
 
-  let(:storage_service) { instance_double(S3StorageService) }
+  let(:storage_service) { instance_double(Storage::S3Service) }
   let(:logger) { instance_double(ActiveSupport::Logger) }
 
   let(:fixture_directory) { "spec/fixtures/imports/schemes" }

spec/services/imports/user_import_service_spec.rb

@@ -5,7 +5,7 @@ RSpec.describe Imports::UserImportService do
   let(:old_user_id) { "fc7625a02b24ae16162aa63ae7cb33feeec0c373" }
   let(:old_org_id) { "7c5bd5fb549c09a2c55d7cb90d7ba84927e64618" }
   let(:user_file) { File.open("#{fixture_directory}/#{old_user_id}.xml") }
-  let(:storage_service) { instance_double(S3StorageService) }
+  let(:storage_service) { instance_double(Storage::S3Service) }
   let(:logger) { instance_double(ActiveSupport::Logger) }
   let(:notify_client) { instance_double(Notifications::Client) }
   let(:devise_notify_mailer) { DeviseNotifyMailer.new }

spec/services/storage/archive_service_spec.rb

@@ -1,6 +1,6 @@
 require "rails_helper"
 
-RSpec.describe ArchiveStorageService do
+RSpec.describe Storage::ArchiveService do
   subject(:archive_service) { described_class.new(archive_content) }
 
   let(:compressed_folder) { "my_directory" }

spec/services/storage/s3_service_spec.rb

@@ -1,6 +1,6 @@
 require "rails_helper"
 
-RSpec.describe S3StorageService do
+RSpec.describe Storage::S3Service do
   let(:instance_name) { "instance_1" }
   let(:bucket_name) { "bucket_1" }
   let(:vcap_services) do
@@ -20,18 +20,24 @@ RSpec.describe S3StorageService do
   end
 
   context "when we create a storage service with no PaaS Configuration present" do
-    subject(:storage_service) { described_class.new(PaasConfigurationService.new, "random_instance") }
+    subject(:storage_service) { described_class.new(Configuration::PaasConfigurationService.new, "random_instance") }
 
     it "raises an exception" do
-      expect { storage_service }.to raise_error(RuntimeError, /No PaaS configuration present/)
+      expect { storage_service }.to raise_error(RuntimeError, "No S3 bucket is present in the PaaS configuration")
     end
   end
 
-  context "when we create a storage service with an unknown instance name" do
+  context "when we create a storage service and the S3 instance name is not found in the PaaS configuration" do
-    subject(:storage_service) { described_class.new(PaasConfigurationService.new, "random_instance") }
+    subject(:storage_service) { described_class.new(Configuration::PaasConfigurationService.new, "random_instance") }
+
+    let(:vcap_services) do
+      <<-JSON
+        {"aws-s3-bucket": []}
+      JSON
+    end
 
     before do
-      allow(ENV).to receive(:[]).with("VCAP_SERVICES").and_return("{}")
+      allow(ENV).to receive(:[]).with("VCAP_SERVICES").and_return(vcap_services)
     end
 
     it "raises an exception" do
@@ -40,7 +46,7 @@ RSpec.describe S3StorageService do
   end
 
   context "when we create a storage service with a valid instance name" do
-    subject(:storage_service) { described_class.new(PaasConfigurationService.new, instance_name) }
+    subject(:storage_service) { described_class.new(Configuration::PaasConfigurationService.new, instance_name) }
 
     before do
       allow(ENV).to receive(:[])
@@ -48,11 +54,11 @@ RSpec.describe S3StorageService do
     end
 
     it "creates a Storage Configuration" do
-      expect(storage_service.configuration).to be_an(StorageConfiguration)
+      expect(storage_service.configuration).to be_an(Storage::StorageConfiguration)
     end
 
     it "sets the expected parameters in the configuration" do
-      expected_configuration = StorageConfiguration.new(
+      expected_configuration = Storage::StorageConfiguration.new(
         {
           aws_access_key_id: "key_id",
           aws_region: "eu-west-2",
@@ -65,7 +71,7 @@ RSpec.describe S3StorageService do
   end
 
   context "when we create a storage service and write a stubbed object" do
-    subject(:storage_service) { described_class.new(PaasConfigurationService.new, instance_name) }
+    subject(:storage_service) { described_class.new(Configuration::PaasConfigurationService.new, instance_name) }
 
     let(:filename) { "my_file" }
     let(:content) { "content" }
@@ -100,7 +106,7 @@ RSpec.describe S3StorageService do
   end
 
   context "when we create a storage service" do
-    subject(:storage_service) { described_class.new(PaasConfigurationService.new, instance_name) }
+    subject(:storage_service) { described_class.new(Configuration::PaasConfigurationService.new, instance_name) }
 
     let(:s3_client_stub) { Aws::S3::Client.new(stub_responses: true) }