Use service credentials to upload and presign CSV exports to S3 (#868)

.github/workflows/production_pipeline.yml

@@ -238,6 +238,7 @@ jobs:
           RAILS_MASTER_KEY: ${{ secrets.RAILS_MASTER_KEY }}
           IMPORT_PAAS_INSTANCE: ${{ secrets.IMPORT_PAAS_INSTANCE }}
           EXPORT_PAAS_INSTANCE: ${{ secrets.EXPORT_PAAS_INSTANCE }}
+          S3_CONFIG: ${{ secrets.S3_CONFIG }}
           CSV_DOWNLOAD_PAAS_INSTANCE: ${{ secrets.CSV_DOWNLOAD_PAAS_INSTANCE }}
           SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
         run: |
@@ -249,6 +250,7 @@ jobs:
           cf set-env $APP_NAME RAILS_MASTER_KEY $RAILS_MASTER_KEY
           cf set-env $APP_NAME IMPORT_PAAS_INSTANCE $IMPORT_PAAS_INSTANCE
           cf set-env $APP_NAME EXPORT_PAAS_INSTANCE $EXPORT_PAAS_INSTANCE
+          cf set-env $APP_NAME S3_CONFIG $S3_CONFIG
           cf set-env $APP_NAME CSV_DOWNLOAD_PAAS_INSTANCE $CSV_DOWNLOAD_PAAS_INSTANCE
           cf set-env $APP_NAME SENTRY_DSN $SENTRY_DSN
           cf push $APP_NAME --strategy rolling

.github/workflows/staging_pipeline.yml

@@ -214,6 +214,7 @@ jobs:
           RAILS_MASTER_KEY: ${{ secrets.RAILS_MASTER_KEY }}
           IMPORT_PAAS_INSTANCE: ${{ secrets.IMPORT_PAAS_INSTANCE }}
           EXPORT_PAAS_INSTANCE: ${{ secrets.EXPORT_PAAS_INSTANCE }}
+          S3_CONFIG: ${{ secrets.S3_CONFIG }}
           CSV_DOWNLOAD_PAAS_INSTANCE: ${{ secrets.CSV_DOWNLOAD_PAAS_INSTANCE }}
           SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
         run: |
@@ -227,6 +228,7 @@ jobs:
           cf set-env $APP_NAME RAILS_MASTER_KEY $RAILS_MASTER_KEY
           cf set-env $APP_NAME IMPORT_PAAS_INSTANCE $IMPORT_PAAS_INSTANCE
           cf set-env $APP_NAME EXPORT_PAAS_INSTANCE $EXPORT_PAAS_INSTANCE
+          cf set-env $APP_NAME S3_CONFIG $S3_CONFIG
           cf set-env $APP_NAME CSV_DOWNLOAD_PAAS_INSTANCE $CSV_DOWNLOAD_PAAS_INSTANCE
           cf set-env $APP_NAME SENTRY_DSN $SENTRY_DSN
           cf push $APP_NAME --strategy rolling

app/jobs/email_csv_job.rb

@@ -11,7 +11,7 @@ class EmailCsvJob < ApplicationJob
 
     filename = organisation.present? ? "logs-#{organisation.name}-#{Time.zone.now}.csv" : "logs-#{Time.zone.now}.csv"
 
-    storage_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["CSV_DOWNLOAD_PAAS_INSTANCE"])
+    storage_service = Storage::S3Service.new(Configuration::EnvConfigurationService.new, ENV["CSV_DOWNLOAD_PAAS_INSTANCE"])
     storage_service.write_file(filename, BYTE_ORDER_MARK + filtered_logs.to_csv(user))
 
     url = storage_service.get_presigned_url(filename, EXPIRATION_TIME)

docs/infrastructure.md

@@ -128,6 +128,7 @@ When a pull request is opened to `main` only the Test stage runs.
 
     ```bash
     cf create-service postgres tiny-unencrypted-13 dluhc-core-staging-postgres
+    cf create-service aws-s3-bucket default dluhc-core-staging-csv-bucket
     cf create-service aws-s3-bucket default dluhc-core-staging-import-bucket
     cf create-service aws-s3-bucket default dluhc-core-staging-export-bucket
     ```
@@ -141,6 +142,7 @@ When a pull request is opened to `main` only the Test stage runs.
 5. Bind S3 services to app:
 
     ```bash
+    cf bind-service dluhc-core-staging dluhc-core-staging-csv-bucket
     cf bind-service dluhc-core-staging dluhc-core-staging-import-bucket -c '{"permissions": "read-only"}'
     cf bind-service dluhc-core-staging dluhc-core-staging-export-bucket -c '{"permissions": "read-write"}'
     ```
@@ -148,6 +150,7 @@ When a pull request is opened to `main` only the Test stage runs.
 6. Create a service keys for accessing the S3 bucket from outside Gov PaaS:
 
     ```bash
+    cf create-service-key dluhc-core-staging-csv-bucket csv-bucket -c '{"allow_external_access": true}'
     cf create-service-key dluhc-core-staging-import-bucket data-import -c '{"allow_external_access": true}'
     cf create-service-key dluhc-core-staging-export-bucket data-export -c '{"allow_external_access": true, "permissions": "read-only"}'
     ```
@@ -170,6 +173,7 @@ When a pull request is opened to `main` only the Test stage runs.
 
     ```bash
     cf create-service postgres small-ha-13 dluhc-core-production-postgres
+    cf create-service aws-s3-bucket default dluhc-core-production-csv-bucket
     cf create-service aws-s3-bucket default dluhc-core-production-import-bucket
     cf create-service aws-s3-bucket default dluhc-core-production-export-bucket
     ```
@@ -183,6 +187,7 @@ When a pull request is opened to `main` only the Test stage runs.
 5. Bind S3 services to app:
 
     ```bash
+    cf bind-service dluhc-core-production dluhc-core-production-csv-bucket
     cf bind-service dluhc-core-production dluhc-core-production-import-bucket -c '{"permissions": "read-only"}'
     cf bind-service dluhc-core-production dluhc-core-production-export-bucket -c '{"permissions": "read-write"}'
     ```
@@ -190,6 +195,7 @@ When a pull request is opened to `main` only the Test stage runs.
 6. Create a service keys for accessing the S3 bucket from outside Gov PaaS:
 
     ```bash
+    cf create-service-key dluhc-core-production-csv-bucket dluhc-core-production-csv-bucket-service-key -c '{"allow_external_access": true}'
     cf create-service-key dluhc-core-production-import-bucket data-import -c '{"allow_external_access": true}'
     cf create-service-key dluhc-core-production-export-bucket data-export -c '{"allow_external_access": true, "permissions": "read-only"}'
     ```