Browse Source
* Move storage classes into a dedicated module * Move configuration classes into a dedicated module * Refactor configuration service * Implement configuration via env variablespull/846/head
Stéphane Meny
2 years ago
committed by
GitHub
32 changed files with 497 additions and 294 deletions
@ -1,24 +0,0 @@
|
||||
class ArchiveStorageService < StorageService |
||||
MAX_SIZE = 50 * (1024**2) # 50MiB |
||||
|
||||
def initialize(archive_io) |
||||
super() |
||||
@archive = Zip::File.open_buffer(archive_io) |
||||
end |
||||
|
||||
def list_files(folder) |
||||
@archive.glob(File.join(folder, "*.*")) |
||||
.map(&:name) |
||||
end |
||||
|
||||
def folder_present?(folder) |
||||
!list_files(folder).empty? |
||||
end |
||||
|
||||
def get_file_io(file_name) |
||||
entry = @archive.get_entry(file_name) |
||||
raise "File too large to be extracted" if entry.size > MAX_SIZE |
||||
|
||||
entry.get_input_stream |
||||
end |
||||
end |
||||
@ -0,0 +1,54 @@
|
||||
module Configuration |
||||
class ConfigurationService |
||||
attr_reader :s3_buckets, :redis_uris |
||||
|
||||
def initialize(logger = Rails.logger) |
||||
@logger = logger |
||||
@config = read_config |
||||
@s3_buckets = read_s3_buckets |
||||
@redis_uris = read_redis_uris |
||||
end |
||||
|
||||
def s3_config_present? |
||||
config_present? && @config.key?(:"aws-s3-bucket") |
||||
end |
||||
|
||||
def redis_config_present? |
||||
config_present? && @config.key?(:redis) |
||||
end |
||||
|
||||
private |
||||
|
||||
def config_present? |
||||
raise NotImplementedError |
||||
end |
||||
|
||||
def read_config |
||||
raise NotImplementedError |
||||
end |
||||
|
||||
def read_s3_buckets |
||||
return {} unless s3_config_present? |
||||
|
||||
s3_buckets = {} |
||||
@config[:"aws-s3-bucket"].each do |bucket_config| |
||||
if bucket_config.key?(:instance_name) |
||||
s3_buckets[bucket_config[:instance_name].to_sym] = bucket_config |
||||
end |
||||
end |
||||
s3_buckets |
||||
end |
||||
|
||||
def read_redis_uris |
||||
return {} unless redis_config_present? |
||||
|
||||
redis_uris = {} |
||||
@config[:redis].each do |redis_config| |
||||
if redis_config.key?(:instance_name) |
||||
redis_uris[redis_config[:instance_name].to_sym] = redis_config.dig(:credentials, :uri) |
||||
end |
||||
end |
||||
redis_uris |
||||
end |
||||
end |
||||
end |
||||
@ -0,0 +1,37 @@
|
||||
module Configuration |
||||
class EnvConfigurationService < ConfigurationService |
||||
private |
||||
|
||||
def config_present? |
||||
!ENV["S3_CONFIG"].nil? || !ENV["REDIS_CONFIG"].nil? |
||||
end |
||||
|
||||
def read_config |
||||
unless config_present? |
||||
@logger.warn("Could not find S3_CONFIG or REDIS_CONFIG in the environment variables!") |
||||
return {} |
||||
end |
||||
|
||||
config = {} |
||||
assign_config(config, :"aws-s3-bucket", "S3_CONFIG") |
||||
assign_config(config, :redis, "REDIS_CONFIG") |
||||
config |
||||
end |
||||
|
||||
def assign_config(config, symbol, env_variable) |
||||
config_hash = parse_json_config(env_variable) |
||||
config[symbol] = config_hash unless config_hash.empty? |
||||
end |
||||
|
||||
def parse_json_config(env_variable_name) |
||||
if ENV[env_variable_name].present? |
||||
begin |
||||
return JSON.parse(ENV[env_variable_name], { symbolize_names: true }) |
||||
rescue StandardError |
||||
@logger.warn("Could not parse #{env_variable_name}!") |
||||
end |
||||
end |
||||
{} |
||||
end |
||||
end |
||||
end |
||||
@ -0,0 +1,23 @@
|
||||
module Configuration |
||||
class PaasConfigurationService < ConfigurationService |
||||
private |
||||
|
||||
def config_present? |
||||
!ENV["VCAP_SERVICES"].nil? |
||||
end |
||||
|
||||
def read_config |
||||
unless config_present? |
||||
@logger.warn("Could not find VCAP_SERVICES in the environment variables!") |
||||
return {} |
||||
end |
||||
|
||||
begin |
||||
JSON.parse(ENV["VCAP_SERVICES"], { symbolize_names: true }) |
||||
rescue StandardError |
||||
@logger.warn("Could not parse VCAP_SERVICES!") |
||||
{} |
||||
end |
||||
end |
||||
end |
||||
end |
||||
@ -1,62 +0,0 @@
|
||||
class PaasConfigurationService |
||||
attr_reader :s3_buckets, :redis_uris |
||||
|
||||
def initialize(logger = Rails.logger) |
||||
@logger = logger |
||||
@paas_config = read_pass_config |
||||
@s3_buckets = read_s3_buckets |
||||
@redis_uris = read_redis_uris |
||||
end |
||||
|
||||
def config_present? |
||||
!ENV["VCAP_SERVICES"].nil? |
||||
end |
||||
|
||||
def s3_config_present? |
||||
config_present? && @paas_config.key?(:"aws-s3-bucket") |
||||
end |
||||
|
||||
def redis_config_present? |
||||
config_present? && @paas_config.key?(:redis) |
||||
end |
||||
|
||||
private |
||||
|
||||
def read_pass_config |
||||
unless config_present? |
||||
@logger.warn("Could not find VCAP_SERVICES in the environment!") |
||||
return {} |
||||
end |
||||
|
||||
begin |
||||
JSON.parse(ENV["VCAP_SERVICES"], { symbolize_names: true }) |
||||
rescue StandardError |
||||
@logger.warn("Could not parse VCAP_SERVICES!") |
||||
{} |
||||
end |
||||
end |
||||
|
||||
def read_s3_buckets |
||||
return {} unless s3_config_present? |
||||
|
||||
s3_buckets = {} |
||||
@paas_config[:"aws-s3-bucket"].each do |bucket_config| |
||||
if bucket_config.key?(:instance_name) |
||||
s3_buckets[bucket_config[:instance_name].to_sym] = bucket_config |
||||
end |
||||
end |
||||
s3_buckets |
||||
end |
||||
|
||||
def read_redis_uris |
||||
return {} unless redis_config_present? |
||||
|
||||
redis_uris = {} |
||||
@paas_config[:redis].each do |redis_config| |
||||
if redis_config.key?(:instance_name) |
||||
redis_uris[redis_config[:instance_name].to_sym] = redis_config.dig(:credentials, :uri) |
||||
end |
||||
end |
||||
redis_uris |
||||
end |
||||
end |
||||
@ -1,78 +0,0 @@
|
||||
class S3StorageService < StorageService |
||||
attr_reader :configuration |
||||
|
||||
def initialize(paas_config_service, paas_instance_name) |
||||
super() |
||||
@paas_config_service = paas_config_service |
||||
@paas_instance_name = (paas_instance_name || "").to_sym |
||||
@configuration = create_configuration |
||||
@client = create_client |
||||
end |
||||
|
||||
def list_files(folder) |
||||
@client.list_objects_v2(bucket: @configuration.bucket_name, prefix: folder) |
||||
.flat_map { |response| response.contents.map(&:key) } |
||||
end |
||||
|
||||
def folder_present?(folder) |
||||
response = @client.list_objects_v2(bucket: @configuration.bucket_name, prefix: folder, max_keys: 1) |
||||
response.key_count == 1 |
||||
end |
||||
|
||||
def get_file_io(file_name) |
||||
@client.get_object(bucket: @configuration.bucket_name, key: file_name) |
||||
.body |
||||
end |
||||
|
||||
def write_file(file_name, data) |
||||
@client.put_object( |
||||
body: data, |
||||
bucket: @configuration.bucket_name, |
||||
key: file_name, |
||||
) |
||||
end |
||||
|
||||
private |
||||
|
||||
def create_configuration |
||||
unless @paas_config_service.config_present? |
||||
raise "No PaaS configuration present" |
||||
end |
||||
unless @paas_config_service.s3_buckets.key?(@paas_instance_name) |
||||
raise "#{@paas_instance_name} instance name could not be found" |
||||
end |
||||
|
||||
bucket_config = @paas_config_service.s3_buckets[@paas_instance_name] |
||||
StorageConfiguration.new(bucket_config[:credentials]) |
||||
end |
||||
|
||||
def create_client |
||||
credentials = |
||||
Aws::Credentials.new( |
||||
@configuration.access_key_id, |
||||
@configuration.secret_access_key, |
||||
) |
||||
Aws::S3::Client.new( |
||||
region: @configuration.region, |
||||
credentials:, |
||||
) |
||||
end |
||||
end |
||||
|
||||
class StorageConfiguration |
||||
attr_reader :access_key_id, :secret_access_key, :bucket_name, :region |
||||
|
||||
def initialize(credentials) |
||||
@access_key_id = credentials[:aws_access_key_id] |
||||
@secret_access_key = credentials[:aws_secret_access_key] |
||||
@bucket_name = credentials[:bucket_name] |
||||
@region = credentials[:aws_region] |
||||
end |
||||
|
||||
def ==(other) |
||||
@access_key_id == other.access_key_id && |
||||
@secret_access_key == other.secret_access_key && |
||||
@bucket_name == other.bucket_name && |
||||
@region == other.region |
||||
end |
||||
end |
||||
@ -0,0 +1,26 @@
|
||||
module Storage |
||||
class ArchiveService < StorageService |
||||
MAX_SIZE = 50 * (1024**2) # 50MiB |
||||
|
||||
def initialize(archive_io) |
||||
super() |
||||
@archive = Zip::File.open_buffer(archive_io) |
||||
end |
||||
|
||||
def list_files(folder) |
||||
@archive.glob(File.join(folder, "*.*")) |
||||
.map(&:name) |
||||
end |
||||
|
||||
def folder_present?(folder) |
||||
!list_files(folder).empty? |
||||
end |
||||
|
||||
def get_file_io(file_name) |
||||
entry = @archive.get_entry(file_name) |
||||
raise "File too large to be extracted" if entry.size > MAX_SIZE |
||||
|
||||
entry.get_input_stream |
||||
end |
||||
end |
||||
end |
||||
@ -0,0 +1,80 @@
|
||||
module Storage |
||||
class S3Service < StorageService |
||||
attr_reader :configuration |
||||
|
||||
def initialize(config_service, paas_instance_name) |
||||
super() |
||||
@config_service = config_service |
||||
@instance_name = (paas_instance_name || "").to_sym |
||||
@configuration = create_configuration |
||||
@client = create_client |
||||
end |
||||
|
||||
def list_files(folder) |
||||
@client.list_objects_v2(bucket: @configuration.bucket_name, prefix: folder) |
||||
.flat_map { |response| response.contents.map(&:key) } |
||||
end |
||||
|
||||
def folder_present?(folder) |
||||
response = @client.list_objects_v2(bucket: @configuration.bucket_name, prefix: folder, max_keys: 1) |
||||
response.key_count == 1 |
||||
end |
||||
|
||||
def get_file_io(file_name) |
||||
@client.get_object(bucket: @configuration.bucket_name, key: file_name) |
||||
.body |
||||
end |
||||
|
||||
def write_file(file_name, data) |
||||
@client.put_object( |
||||
body: data, |
||||
bucket: @configuration.bucket_name, |
||||
key: file_name, |
||||
) |
||||
end |
||||
|
||||
private |
||||
|
||||
def create_configuration |
||||
unless @config_service.s3_config_present? |
||||
raise "No S3 bucket is present in the PaaS configuration" |
||||
end |
||||
unless @config_service.s3_buckets.key?(@instance_name) |
||||
raise "#{@instance_name} instance name could not be found" |
||||
end |
||||
|
||||
bucket_config = @config_service.s3_buckets[@instance_name] |
||||
StorageConfiguration.new(bucket_config[:credentials]) |
||||
end |
||||
|
||||
def create_client |
||||
credentials = |
||||
Aws::Credentials.new( |
||||
@configuration.access_key_id, |
||||
@configuration.secret_access_key, |
||||
) |
||||
Aws::S3::Client.new( |
||||
region: @configuration.region, |
||||
credentials:, |
||||
) |
||||
end |
||||
end |
||||
|
||||
class StorageConfiguration |
||||
attr_reader :access_key_id, :secret_access_key, :bucket_name, :region |
||||
|
||||
def initialize(credentials) |
||||
@access_key_id = credentials[:aws_access_key_id] |
||||
@secret_access_key = credentials[:aws_secret_access_key] |
||||
@bucket_name = credentials[:bucket_name] |
||||
@region = credentials[:aws_region] |
||||
end |
||||
|
||||
def ==(other) |
||||
@access_key_id == other.access_key_id && |
||||
@secret_access_key == other.secret_access_key && |
||||
@bucket_name == other.bucket_name && |
||||
@region == other.region |
||||
end |
||||
end |
||||
end |
||||
@ -0,0 +1,19 @@
|
||||
module Storage |
||||
class StorageService |
||||
def list_files(_folder) |
||||
raise NotImplementedError |
||||
end |
||||
|
||||
def folder_present?(_folder) |
||||
raise NotImplementedError |
||||
end |
||||
|
||||
def get_file_io(_file_name) |
||||
raise NotImplementedError |
||||
end |
||||
|
||||
def write_file(_file_name, _data) |
||||
raise NotImplementedError |
||||
end |
||||
end |
||||
end |
||||
@ -1,17 +0,0 @@
|
||||
class StorageService |
||||
def list_files(_folder) |
||||
raise NotImplementedError |
||||
end |
||||
|
||||
def folder_present?(_folder) |
||||
raise NotImplementedError |
||||
end |
||||
|
||||
def get_file_io(_file_name) |
||||
raise NotImplementedError |
||||
end |
||||
|
||||
def write_file(_file_name, _data) |
||||
raise NotImplementedError |
||||
end |
||||
end |
||||
@ -0,0 +1,128 @@
|
||||
require "rails_helper" |
||||
|
||||
RSpec.describe Configuration::EnvConfigurationService do |
||||
subject(:config_service) { described_class.new(logger) } |
||||
|
||||
let(:logger) { instance_double(ActiveSupport::LogSubscriber) } |
||||
|
||||
context "when environment configurations are unavailable" do |
||||
before { allow(logger).to receive(:warn) } |
||||
|
||||
it "returns the S3 configuration as not present" do |
||||
expect(config_service.s3_config_present?).to be(false) |
||||
end |
||||
|
||||
it "returns the redis configuration as not present" do |
||||
expect(config_service.redis_config_present?).to be(false) |
||||
end |
||||
|
||||
it "does not retrieve any S3 bucket configuration" do |
||||
expect(config_service.s3_buckets).to be_a(Hash) |
||||
expect(config_service.s3_buckets).to be_empty |
||||
end |
||||
|
||||
it "does not retrieve any redis configuration" do |
||||
expect(config_service.redis_uris).to be_a(Hash) |
||||
expect(config_service.redis_uris).to be_empty |
||||
end |
||||
end |
||||
|
||||
context "when environment configurations are present but invalid" do |
||||
let(:env_variable) { "random text" } |
||||
|
||||
before do |
||||
allow(ENV).to receive(:[]).with("S3_CONFIG").and_return(env_variable) |
||||
allow(ENV).to receive(:[]).with("REDIS_CONFIG").and_return(env_variable) |
||||
allow(logger).to receive(:warn) |
||||
end |
||||
|
||||
it "logs an error when checking if the S3 config is present" do |
||||
expect(logger).to receive(:warn).with("Could not parse S3_CONFIG!") |
||||
config_service.s3_config_present? |
||||
end |
||||
|
||||
it "logs an error when checking if the Redis config is present" do |
||||
expect(logger).to receive(:warn).with("Could not parse REDIS_CONFIG!") |
||||
config_service.redis_config_present? |
||||
end |
||||
end |
||||
|
||||
context "when environment configurations are present with S3 configured" do |
||||
let(:s3_config) do |
||||
<<~JSON |
||||
[ |
||||
{ |
||||
"instance_name": "bucket_1", |
||||
"credentials": { |
||||
"aws_access_key_id": "123", |
||||
"aws_secret_access_key": "456", |
||||
"aws_region": "eu-west-1", |
||||
"bucket_name": "my-bucket" |
||||
} |
||||
}, |
||||
{ |
||||
"instance_name": "bucket_2", |
||||
"credentials": { |
||||
"aws_access_key_id": "789", |
||||
"aws_secret_access_key": "012", |
||||
"aws_region": "eu-west-2", |
||||
"bucket_name": "my-bucket2" |
||||
} |
||||
} |
||||
] |
||||
JSON |
||||
end |
||||
|
||||
before do |
||||
allow(ENV).to receive(:[]).with("REDIS_CONFIG") |
||||
allow(ENV).to receive(:[]).with("S3_CONFIG").and_return(s3_config) |
||||
end |
||||
|
||||
it "returns the S3 configuration as present" do |
||||
expect(config_service.s3_config_present?).to be(true) |
||||
end |
||||
|
||||
it "returns the redis configuration as not present" do |
||||
expect(config_service.redis_config_present?).to be(false) |
||||
end |
||||
|
||||
it "does retrieve the S3 bucket configurations" do |
||||
s3_buckets = config_service.s3_buckets |
||||
|
||||
expect(s3_buckets).not_to be_empty |
||||
expect(s3_buckets.count).to be(2) |
||||
expect(s3_buckets).to have_key(:bucket_1) |
||||
expect(s3_buckets).to have_key(:bucket_2) |
||||
end |
||||
end |
||||
|
||||
context "when environment configurations are present with redis configured" do |
||||
let(:redis_config) do |
||||
<<-JSON |
||||
[{"instance_name": "redis_1", "credentials": {"uri": "redis_uri" }}] |
||||
JSON |
||||
end |
||||
|
||||
before do |
||||
allow(ENV).to receive(:[]).with("S3_CONFIG") |
||||
allow(ENV).to receive(:[]).with("REDIS_CONFIG").and_return(redis_config) |
||||
end |
||||
|
||||
it "returns the redis configuration as present" do |
||||
expect(config_service.redis_config_present?).to be(true) |
||||
end |
||||
|
||||
it "returns the S3 configuration as not present" do |
||||
expect(config_service.s3_config_present?).to be(false) |
||||
end |
||||
|
||||
it "does retrieve the redis configurations" do |
||||
redis_uris = config_service.redis_uris |
||||
|
||||
expect(redis_uris).not_to be_empty |
||||
expect(redis_uris.count).to be(1) |
||||
expect(redis_uris).to have_key(:redis_1) |
||||
expect(redis_uris[:redis_1]).to eq("redis_uri") |
||||
end |
||||
end |
||||
end |
||||
@ -1,6 +1,6 @@
|
||||
require "rails_helper" |
||||
|
||||
RSpec.describe ArchiveStorageService do |
||||
RSpec.describe Storage::ArchiveService do |
||||
subject(:archive_service) { described_class.new(archive_content) } |
||||
|
||||
let(:compressed_folder) { "my_directory" } |
||||
Loading…
Reference in new issue