From 7927cf99dafe24819f785ea3c89a64691ce91793 Mon Sep 17 00:00:00 2001 From: James Rose Date: Wed, 14 Sep 2022 10:58:29 +0100 Subject: [PATCH] Use service credentials to upload and presign CSV exports to S3 (#868) --- .github/workflows/production_pipeline.yml | 2 ++ .github/workflows/staging_pipeline.yml | 2 ++ app/jobs/email_csv_job.rb | 2 +- docs/infrastructure.md | 6 ++++++ 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/production_pipeline.yml b/.github/workflows/production_pipeline.yml index 4400d9239..23bb69c00 100644 --- a/.github/workflows/production_pipeline.yml +++ b/.github/workflows/production_pipeline.yml @@ -238,6 +238,7 @@ jobs: RAILS_MASTER_KEY: ${{ secrets.RAILS_MASTER_KEY }} IMPORT_PAAS_INSTANCE: ${{ secrets.IMPORT_PAAS_INSTANCE }} EXPORT_PAAS_INSTANCE: ${{ secrets.EXPORT_PAAS_INSTANCE }} + S3_CONFIG: ${{ secrets.S3_CONFIG }} CSV_DOWNLOAD_PAAS_INSTANCE: ${{ secrets.CSV_DOWNLOAD_PAAS_INSTANCE }} SENTRY_DSN: ${{ secrets.SENTRY_DSN }} run: | @@ -249,6 +250,7 @@ jobs: cf set-env $APP_NAME RAILS_MASTER_KEY $RAILS_MASTER_KEY cf set-env $APP_NAME IMPORT_PAAS_INSTANCE $IMPORT_PAAS_INSTANCE cf set-env $APP_NAME EXPORT_PAAS_INSTANCE $EXPORT_PAAS_INSTANCE + cf set-env $APP_NAME S3_CONFIG $S3_CONFIG cf set-env $APP_NAME CSV_DOWNLOAD_PAAS_INSTANCE $CSV_DOWNLOAD_PAAS_INSTANCE cf set-env $APP_NAME SENTRY_DSN $SENTRY_DSN cf push $APP_NAME --strategy rolling diff --git a/.github/workflows/staging_pipeline.yml b/.github/workflows/staging_pipeline.yml index 72d561652..d59024cd3 100644 --- a/.github/workflows/staging_pipeline.yml +++ b/.github/workflows/staging_pipeline.yml @@ -214,6 +214,7 @@ jobs: RAILS_MASTER_KEY: ${{ secrets.RAILS_MASTER_KEY }} IMPORT_PAAS_INSTANCE: ${{ secrets.IMPORT_PAAS_INSTANCE }} EXPORT_PAAS_INSTANCE: ${{ secrets.EXPORT_PAAS_INSTANCE }} + S3_CONFIG: ${{ secrets.S3_CONFIG }} CSV_DOWNLOAD_PAAS_INSTANCE: ${{ secrets.CSV_DOWNLOAD_PAAS_INSTANCE }} SENTRY_DSN: ${{ secrets.SENTRY_DSN }} run: | @@ -227,6 +228,7 @@ jobs: cf set-env $APP_NAME RAILS_MASTER_KEY $RAILS_MASTER_KEY cf set-env $APP_NAME IMPORT_PAAS_INSTANCE $IMPORT_PAAS_INSTANCE cf set-env $APP_NAME EXPORT_PAAS_INSTANCE $EXPORT_PAAS_INSTANCE + cf set-env $APP_NAME S3_CONFIG $S3_CONFIG cf set-env $APP_NAME CSV_DOWNLOAD_PAAS_INSTANCE $CSV_DOWNLOAD_PAAS_INSTANCE cf set-env $APP_NAME SENTRY_DSN $SENTRY_DSN cf push $APP_NAME --strategy rolling diff --git a/app/jobs/email_csv_job.rb b/app/jobs/email_csv_job.rb index 4cfe2af45..c7009645f 100644 --- a/app/jobs/email_csv_job.rb +++ b/app/jobs/email_csv_job.rb @@ -11,7 +11,7 @@ class EmailCsvJob < ApplicationJob filename = organisation.present? ? "logs-#{organisation.name}-#{Time.zone.now}.csv" : "logs-#{Time.zone.now}.csv" - storage_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["CSV_DOWNLOAD_PAAS_INSTANCE"]) + storage_service = Storage::S3Service.new(Configuration::EnvConfigurationService.new, ENV["CSV_DOWNLOAD_PAAS_INSTANCE"]) storage_service.write_file(filename, BYTE_ORDER_MARK + filtered_logs.to_csv(user)) url = storage_service.get_presigned_url(filename, EXPIRATION_TIME) diff --git a/docs/infrastructure.md b/docs/infrastructure.md index 5246c701c..31737cada 100644 --- a/docs/infrastructure.md +++ b/docs/infrastructure.md @@ -128,6 +128,7 @@ When a pull request is opened to `main` only the Test stage runs. ```bash cf create-service postgres tiny-unencrypted-13 dluhc-core-staging-postgres + cf create-service aws-s3-bucket default dluhc-core-staging-csv-bucket cf create-service aws-s3-bucket default dluhc-core-staging-import-bucket cf create-service aws-s3-bucket default dluhc-core-staging-export-bucket ``` @@ -141,6 +142,7 @@ When a pull request is opened to `main` only the Test stage runs. 5. Bind S3 services to app: ```bash + cf bind-service dluhc-core-staging dluhc-core-staging-csv-bucket cf bind-service dluhc-core-staging dluhc-core-staging-import-bucket -c '{"permissions": "read-only"}' cf bind-service dluhc-core-staging dluhc-core-staging-export-bucket -c '{"permissions": "read-write"}' ``` @@ -148,6 +150,7 @@ When a pull request is opened to `main` only the Test stage runs. 6. Create a service keys for accessing the S3 bucket from outside Gov PaaS: ```bash + cf create-service-key dluhc-core-staging-csv-bucket csv-bucket -c '{"allow_external_access": true}' cf create-service-key dluhc-core-staging-import-bucket data-import -c '{"allow_external_access": true}' cf create-service-key dluhc-core-staging-export-bucket data-export -c '{"allow_external_access": true, "permissions": "read-only"}' ``` @@ -170,6 +173,7 @@ When a pull request is opened to `main` only the Test stage runs. ```bash cf create-service postgres small-ha-13 dluhc-core-production-postgres + cf create-service aws-s3-bucket default dluhc-core-production-csv-bucket cf create-service aws-s3-bucket default dluhc-core-production-import-bucket cf create-service aws-s3-bucket default dluhc-core-production-export-bucket ``` @@ -183,6 +187,7 @@ When a pull request is opened to `main` only the Test stage runs. 5. Bind S3 services to app: ```bash + cf bind-service dluhc-core-production dluhc-core-production-csv-bucket cf bind-service dluhc-core-production dluhc-core-production-import-bucket -c '{"permissions": "read-only"}' cf bind-service dluhc-core-production dluhc-core-production-export-bucket -c '{"permissions": "read-write"}' ``` @@ -190,6 +195,7 @@ When a pull request is opened to `main` only the Test stage runs. 6. Create a service keys for accessing the S3 bucket from outside Gov PaaS: ```bash + cf create-service-key dluhc-core-production-csv-bucket dluhc-core-production-csv-bucket-service-key -c '{"allow_external_access": true}' cf create-service-key dluhc-core-production-import-bucket data-import -c '{"allow_external_access": true}' cf create-service-key dluhc-core-production-export-bucket data-export -c '{"allow_external_access": true, "permissions": "read-only"}' ```