Add review app pipeline (#993)

* Add review app pipeline

* disable 2FA on review apps

* update infrastructure docs about review apps

.github/workflows/review_pipeline.yml

@@ -0,0 +1,158 @@
+name: Review app pipeline
+
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+  workflow_dispatch:
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  postgres:
+    name: Provision postgres
+    runs-on: ubuntu-latest
+    environment: staging
+
+    steps:
+      - name: Install Cloud Foundry CLI
+        run: |
+          wget -q -O - https://packages.cloudfoundry.org/debian/cli.cloudfoundry.org.key | sudo apt-key add -
+          echo "deb https://packages.cloudfoundry.org/debian stable main" | sudo tee /etc/apt/sources.list.d/cloudfoundry-cli.list
+          sudo apt-get update
+          sudo apt-get install cf8-cli
+
+      - name: Provision postgres
+        env:
+          CF_USERNAME: ${{ secrets.CF_USERNAME }}
+          CF_PASSWORD: ${{ secrets.CF_PASSWORD }}
+          CF_API_ENDPOINT: ${{ secrets.CF_API_ENDPOINT }}
+          CF_SPACE: dev
+          CF_ORG: ${{ secrets.CF_ORG }}
+        run: |
+          cf api $CF_API_ENDPOINT
+          cf auth
+          cf target -o $CF_ORG -s $CF_SPACE
+          cf create-service postgres tiny-unencrypted-13 dluhc-core-review-${{ github.event.pull_request.number }}-postgres --wait
+
+  redis:
+    name: Provision redis
+    runs-on: ubuntu-latest
+    environment: staging
+
+    steps:
+      - name: Install Cloud Foundry CLI
+        run: |
+          wget -q -O - https://packages.cloudfoundry.org/debian/cli.cloudfoundry.org.key | sudo apt-key add -
+          echo "deb https://packages.cloudfoundry.org/debian stable main" | sudo tee /etc/apt/sources.list.d/cloudfoundry-cli.list
+          sudo apt-get update
+          sudo apt-get install cf8-cli
+
+      - name: Provision redis
+        env:
+          CF_USERNAME: ${{ secrets.CF_USERNAME }}
+          CF_PASSWORD: ${{ secrets.CF_PASSWORD }}
+          CF_API_ENDPOINT: ${{ secrets.CF_API_ENDPOINT }}
+          CF_SPACE: dev
+          CF_ORG: ${{ secrets.CF_ORG }}
+        run: |
+          cf api $CF_API_ENDPOINT
+          cf auth
+          cf target -o $CF_ORG -s $CF_SPACE
+          cf create-service redis micro-6.x dluhc-core-review-${{ github.event.pull_request.number }}-redis --wait
+
+  deploy:
+    name: Deploy review app
+    runs-on: ubuntu-latest
+    environment: staging
+    needs: [postgres, redis]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Install Cloud Foundry CLI
+        run: |
+          wget -q -O - https://packages.cloudfoundry.org/debian/cli.cloudfoundry.org.key | sudo apt-key add -
+          echo "deb https://packages.cloudfoundry.org/debian stable main" | sudo tee /etc/apt/sources.list.d/cloudfoundry-cli.list
+          sudo apt-get update
+          sudo apt-get install cf8-cli
+
+      - name: Setup review app without starting
+        env:
+          CF_USERNAME: ${{ secrets.CF_USERNAME }}
+          CF_PASSWORD: ${{ secrets.CF_PASSWORD }}
+          CF_API_ENDPOINT: ${{ secrets.CF_API_ENDPOINT }}
+          CF_SPACE: dev
+          CF_ORG: ${{ secrets.CF_ORG }}
+          APP_NAME: dluhc-core-review-${{ github.event.pull_request.number }}
+        run: |
+          cf api $CF_API_ENDPOINT
+          cf auth
+          cf target -o $CF_ORG -s $CF_SPACE
+          cf push $APP_NAME \
+            --manifest ./config/cloud_foundry/review_manifest.yml \
+            --no-start
+
+      - name: Set environment variables
+        env:
+          APP_NAME: dluhc-core-review-${{ github.event.pull_request.number }}
+          API_USER: ${{ secrets.API_USER }}
+          API_KEY: ${{ secrets.API_KEY }}
+          GOVUK_NOTIFY_API_KEY: ${{ secrets.GOVUK_NOTIFY_API_KEY }}
+          RAILS_MASTER_KEY: ${{ secrets.RAILS_MASTER_KEY }}
+          IMPORT_PAAS_INSTANCE: ${{ secrets.IMPORT_PAAS_INSTANCE }}
+          EXPORT_PAAS_INSTANCE: ${{ secrets.EXPORT_PAAS_INSTANCE }}
+          S3_CONFIG: ${{ secrets.S3_CONFIG }}
+          CSV_DOWNLOAD_PAAS_INSTANCE: ${{ secrets.CSV_DOWNLOAD_PAAS_INSTANCE }}
+          SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
+        run: |
+          cf set-env $APP_NAME API_USER $API_USER
+          cf set-env $APP_NAME API_KEY $API_KEY
+          cf set-env $APP_NAME GOVUK_NOTIFY_API_KEY $GOVUK_NOTIFY_API_KEY
+          cf set-env $APP_NAME RAILS_MASTER_KEY $RAILS_MASTER_KEY
+          cf set-env $APP_NAME IMPORT_PAAS_INSTANCE $IMPORT_PAAS_INSTANCE
+          cf set-env $APP_NAME EXPORT_PAAS_INSTANCE $EXPORT_PAAS_INSTANCE
+          cf set-env $APP_NAME S3_CONFIG $S3_CONFIG
+          cf set-env $APP_NAME CSV_DOWNLOAD_PAAS_INSTANCE $CSV_DOWNLOAD_PAAS_INSTANCE
+          cf set-env $APP_NAME SENTRY_DSN $SENTRY_DSN
+
+      - name: Bind postgres service
+        env:
+          APP_NAME: dluhc-core-review-${{ github.event.pull_request.number }}
+          SERVICE_NAME: dluhc-core-review-${{ github.event.pull_request.number }}-postgres
+        run: |
+          cf bind-service $APP_NAME $SERVICE_NAME
+
+      - name: Bind redis service
+        env:
+          APP_NAME: dluhc-core-review-${{ github.event.pull_request.number }}
+          SERVICE_NAME: dluhc-core-review-${{ github.event.pull_request.number }}-redis
+        run: |
+          cf bind-service $APP_NAME $SERVICE_NAME
+
+      - name: Bind logit drain service
+        env:
+          APP_NAME: dluhc-core-review-${{ github.event.pull_request.number }}
+          SERVICE_NAME: logit-ssl-drain
+        run: |
+          cf bind-service $APP_NAME $SERVICE_NAME
+
+      - name: Start review app
+        env:
+          APP_NAME: dluhc-core-review-${{ github.event.pull_request.number }}
+        run: |
+          cf restage $APP_NAME --strategy rolling
+
+      - name: Comment on PR with URL
+        uses: unsplash/comment-on-pr@v1.3.0
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          msg: "Created review app at https://dluhc-core-review-${{ github.event.pull_request.number }}.london.cloudapps.digital"
+          check_for_duplicate_msg: true
+          duplicate_msg_pattern: Created review app at*

.github/workflows/review_teardown_pipeline.yml

@@ -0,0 +1,92 @@
+name: Review app teardown pipeline
+
+on:
+  pull_request:
+    types:
+      - closed
+  workflow_dispatch:
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  app:
+    name: Teardown app
+    runs-on: ubuntu-latest
+    environment: staging
+
+    steps:
+      - name: Install Cloud Foundry CLI
+        run: |
+          wget -q -O - https://packages.cloudfoundry.org/debian/cli.cloudfoundry.org.key | sudo apt-key add -
+          echo "deb https://packages.cloudfoundry.org/debian stable main" | sudo tee /etc/apt/sources.list.d/cloudfoundry-cli.list
+          sudo apt-get update
+          sudo apt-get install cf8-cli
+
+      - name: Teardown app
+        env:
+          CF_USERNAME: ${{ secrets.CF_USERNAME }}
+          CF_PASSWORD: ${{ secrets.CF_PASSWORD }}
+          CF_API_ENDPOINT: ${{ secrets.CF_API_ENDPOINT }}
+          CF_SPACE: dev
+          CF_ORG: ${{ secrets.CF_ORG }}
+        run: |
+          cf api $CF_API_ENDPOINT
+          cf auth
+          cf target -o $CF_ORG -s $CF_SPACE
+          cf delete dluhc-core-review-${{ github.event.pull_request.number }} -f -r
+
+  postgres:
+    name: Teardown postgres
+    runs-on: ubuntu-latest
+    environment: staging
+    needs: [app]
+
+    steps:
+      - name: Install Cloud Foundry CLI
+        run: |
+          wget -q -O - https://packages.cloudfoundry.org/debian/cli.cloudfoundry.org.key | sudo apt-key add -
+          echo "deb https://packages.cloudfoundry.org/debian stable main" | sudo tee /etc/apt/sources.list.d/cloudfoundry-cli.list
+          sudo apt-get update
+          sudo apt-get install cf8-cli
+
+      - name: Teardown postgres
+        env:
+          CF_USERNAME: ${{ secrets.CF_USERNAME }}
+          CF_PASSWORD: ${{ secrets.CF_PASSWORD }}
+          CF_API_ENDPOINT: ${{ secrets.CF_API_ENDPOINT }}
+          CF_SPACE: dev
+          CF_ORG: ${{ secrets.CF_ORG }}
+        run: |
+          cf api $CF_API_ENDPOINT
+          cf auth
+          cf target -o $CF_ORG -s $CF_SPACE
+          cf delete-service dluhc-core-review-${{ github.event.pull_request.number }}-postgres --wait -f
+
+  redis:
+    name: Teardown redis
+    runs-on: ubuntu-latest
+    environment: staging
+    needs: [app]
+
+    steps:
+      - name: Install Cloud Foundry CLI
+        run: |
+          wget -q -O - https://packages.cloudfoundry.org/debian/cli.cloudfoundry.org.key | sudo apt-key add -
+          echo "deb https://packages.cloudfoundry.org/debian stable main" | sudo tee /etc/apt/sources.list.d/cloudfoundry-cli.list
+          sudo apt-get update
+          sudo apt-get install cf8-cli
+
+      - name: Teardown redis
+        env:
+          CF_USERNAME: ${{ secrets.CF_USERNAME }}
+          CF_PASSWORD: ${{ secrets.CF_PASSWORD }}
+          CF_API_ENDPOINT: ${{ secrets.CF_API_ENDPOINT }}
+          CF_SPACE: dev
+          CF_ORG: ${{ secrets.CF_ORG }}
+        run: |
+          cf api $CF_API_ENDPOINT
+          cf auth
+          cf target -o $CF_ORG -s $CF_SPACE
+          cf delete-service dluhc-core-review-${{ github.event.pull_request.number }}-redis --wait -f

.rubocop.yml

@@ -27,3 +27,4 @@ Rails/UnknownEnv:
     - staging
     - development
     - test
+    - review

app/models/user.rb

@@ -125,6 +125,7 @@ class User < ApplicationRecord
 
   def need_two_factor_authentication?(_request)
     return false if Rails.env.development?
+    return false if Rails.env.review?
 
     support?
   end

config/cloud_foundry/review_manifest.yml

@@ -0,0 +1,21 @@
+---
+defaults: &defaults
+  buildpacks:
+    - https://github.com/cloudfoundry/ruby-buildpack.git#v1.8.59
+  processes:
+    - type: web
+      command: bundle exec rake cf:on_first_instance db:migrate db:seed && bin/rails server
+      instances: 1
+      memory: 1G
+    - type: worker
+      command: bundle exec sidekiq
+      health-check-type: process
+      instances: 1
+  health-check-type: http
+  health-check-http-endpoint: /health
+
+applications:
+  - name: dluhc-core-review
+    <<: *defaults
+    env:
+      RAILS_ENV: review

config/database.yml

@@ -31,6 +31,9 @@ development:
   database: <%= ENV['DB_DATABASE'] || 'data_collector_development' %>
   host: <%= ENV['DB_HOST'] || 'localhost' %>
 
+review:
+  <<: *default
+
 staging:
   <<: *default
 

config/environments/review_app.rb

@@ -0,0 +1,130 @@
+require "active_support/core_ext/integer/time"
+
+Rails.application.configure do
+  # Settings specified here will take precedence over those in config/application.rb.
+
+  # Code is not reloaded between requests.
+  config.cache_classes = true
+
+  # Eager load code on boot. This eager loads most of Rails and
+  # your application in memory, allowing both threaded web servers
+  # and those relying on copy on write to perform better.
+  # Rake tasks automatically ignore this option for performance.
+  config.eager_load = true
+
+  # Full error reports are disabled and caching is turned on.
+  config.consider_all_requests_local       = false
+  config.action_controller.perform_caching = true
+
+  # Ensures that a master key has been made available in either ENV["RAILS_MASTER_KEY"]
+  # or in config/master.key. This key is used to decrypt credentials (and other encrypted files).
+  # config.require_master_key = true
+
+  # Disable serving static files from the `/public` folder by default since
+  # Apache or NGINX already handles this.
+  config.public_file_server.enabled = ENV["RAILS_SERVE_STATIC_FILES"].present?
+
+  # Enable serving of images, stylesheets, and JavaScripts from an asset server.
+  # config.asset_host = 'http://assets.example.com'
+
+  # Specifies the header that your server uses for sending files.
+  # config.action_dispatch.x_sendfile_header = 'X-Sendfile' # for Apache
+  # config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for NGINX
+
+  # Store uploaded files on the local file system (see config/storage.yml for options).
+  config.active_storage.service = :local
+
+  # Mount Action Cable outside main process or domain.
+  # config.action_cable.mount_path = nil
+  # config.action_cable.url = 'wss://example.com/cable'
+  # config.action_cable.allowed_request_origins = [ 'http://example.com', /http:\/\/example.*/ ]
+
+  # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies.
+  # config.force_ssl = true
+
+  # Include generic and useful information about system operation, but avoid logging too much
+  # information to avoid inadvertent exposure of personally identifiable information (PII).
+  config.log_level = :info
+
+  # Prepend all log lines with the following tags.
+  config.log_tags = [:request_id]
+
+  # Use a different cache store in production.
+  # config.cache_store = :mem_cache_store
+
+  # Use a real queuing backend for Active Job (and separate queues per environment).
+  # config.active_job.queue_adapter     = :resque
+  # config.active_job.queue_name_prefix = "data_collector_production"
+
+  config.action_mailer.perform_caching = false
+
+  config.action_mailer.default_url_options = { host: ENV["APP_HOST"] }
+  config.action_mailer.delivery_method = :smtp
+  config.action_mailer.smtp_settings = {
+    address: "smtp.gmail.com",
+    port: 465,
+    domain: "gmail.com",
+    user_name: ENV["CORE_EMAIL_USERNAME"],
+    password: ENV["CORE_EMAIL_PASSWORD"],
+    authentication: "plain",
+    enable_starttls_auto: true,
+    ssl: true,
+  }
+
+  # Ignore bad email addresses and do not raise email delivery errors.
+  # Set this to true and configure the email server for immediate delivery to raise delivery errors.
+  # config.action_mailer.raise_delivery_errors = false
+
+  # Enable locale fallbacks for I18n (makes lookups for any locale fall back to
+  # the I18n.default_locale when a translation cannot be found).
+  config.i18n.fallbacks = true
+
+  # Send deprecation notices to registered listeners.
+  config.active_support.deprecation = :notify
+
+  # Log disallowed deprecations.
+  config.active_support.disallowed_deprecation = :log
+
+  # Tell Active Support which deprecation messages to disallow.
+  config.active_support.disallowed_deprecation_warnings = []
+
+  # Use default logging formatter so that PID and timestamp are not suppressed.
+  config.log_formatter = ::Logger::Formatter.new
+
+  # Use a different logger for distributed setups.
+  # require "syslog/logger"
+  # config.logger = ActiveSupport::TaggedLogging.new(Syslog::Logger.new 'app-name')
+
+  if ENV["RAILS_LOG_TO_STDOUT"].present?
+    logger           = ActiveSupport::Logger.new($stdout)
+    logger.formatter = config.log_formatter
+    config.logger    = ActiveSupport::TaggedLogging.new(logger)
+  end
+
+  # Do not dump schema after migrations.
+  config.active_record.dump_schema_after_migration = false
+
+  # Inserts middleware to perform automatic connection switching.
+  # The `database_selector` hash is used to pass options to the DatabaseSelector
+  # middleware. The `delay` is used to determine how long to wait after a write
+  # to send a subsequent read to the primary.
+  #
+  # The `database_resolver` class is used by the middleware to determine which
+  # database is appropriate to use based on the time delay.
+  #
+  # The `database_resolver_context` class is used by the middleware to set
+  # timestamps for the last write to the primary. The resolver uses the context
+  # class timestamps to determine how long to wait before reading from the
+  # replica.
+  #
+  # By default Rails will store a last write timestamp in the session. The
+  # DatabaseSelector middleware is designed as such you can define your own
+  # strategy for connection switching and pass that into the middleware through
+  # these configuration options.
+  # config.active_record.database_selector = { delay: 2.seconds }
+  # config.active_record.database_resolver = ActiveRecord::Middleware::DatabaseSelector::Resolver
+  # config.active_record.database_resolver_context = ActiveRecord::Middleware::DatabaseSelector::Resolver::Session
+
+  # see https://discuss.rubyonrails.org/t/cve-2022-32224-possible-rce-escalation-bug-with-serialized-columns-in-active-record/81017
+  config.active_record.yaml_column_permitted_classes = [Time]
+end

config/initializers/rack_attack.rb

@@ -4,6 +4,9 @@ require "configuration/paas_configuration_service"
 if Rails.env.development? || Rails.env.test?
   Rack::Attack.cache.store = ActiveSupport::Cache::MemoryStore.new
   Rack::Attack.enabled = false
+elsif Rails.env.review?
+  redis_url = Configuration::PaasConfigurationService.new.redis_uris.to_a[0][1]
+  Rack::Attack.cache.store = ActiveSupport::Cache::RedisCacheStore.new(url: redis_url)
 else
   redis_url = Configuration::PaasConfigurationService.new.redis_uris[:"dluhc-core-#{Rails.env}-redis"]
   Rack::Attack.cache.store = ActiveSupport::Cache::RedisCacheStore.new(url: redis_url)

config/initializers/sentry.rb

@@ -1,5 +1,5 @@
 Sentry.init do |config|
   config.breadcrumbs_logger = %i[active_support_logger http_logger]
-  config.enabled_environments = %w[production staging]
+  config.enabled_environments = %w[production staging review]
   config.traces_sample_rate = 0.2
 end

config/initializers/sidekiq.rb

@@ -11,3 +11,15 @@ if Rails.env.staging? || Rails.env.production?
     config.redis = { url: redis_url }
   end
 end
+
+if Rails.env.review?
+  redis_url = Configuration::PaasConfigurationService.new.redis_uris.to_a[0][1]
+
+  Sidekiq.configure_server do |config|
+    config.redis = { url: redis_url }
+  end
+
+  Sidekiq.configure_client do |config|
+    config.redis = { url: redis_url }
+  end
+end

db/seeds.rb

@@ -88,7 +88,7 @@ unless Rails.env.test?
     relationship_type: OrganisationRelationship::MANAGING,
   )
 
-  if Rails.env.development? && User.count.zero?
+  if (Rails.env.development? || Rails.env.review?) && User.count.zero?
     User.create!(
       name: "Provider",
       email: "provider@example.com",
@@ -119,7 +119,7 @@ unless Rails.env.test?
     pp "Seeded 3 dummy users"
   end
 
-  if Rails.env.development?
+  if Rails.env.development? || Rails.env.review?
     dummy_org = Organisation.find_or_create_by!(
       name: "FooBar LTD",
       address_line1: "Higher Kingston",
@@ -134,7 +134,7 @@ unless Rails.env.test?
     pp "Seeded dummy FooBar LTD organisation"
   end
 
-  if Rails.env.development? && Scheme.count.zero?
+  if (Rails.env.development? || Rails.env.review?) && Scheme.count.zero?
     scheme1 = Scheme.create!(
       service_name: "Beulahside Care",
       sensitive: 0,

docs/infrastructure.md

@@ -108,6 +108,20 @@ When a commit is made to `main` the following GitHub action jobs are triggered:
 
 When a pull request is opened to `main` only the Test stage runs.
 
+## Review apps
+
+When a pull request is opened a review app will be spun up. The reviews apps connect to their own PostgreSQL and Redis instances with its own worker.
+
+The review app github pipeline is independent of any test pipeline and therefore it will attempt to deploy regardless of the state the code is in.
+
+The usual seeding process takes place when the review app boots so there will be some minimal data that can be used to login with. 2FA has been disabled in the review apps for easier access.
+
+The app boots in a new environment called `review`. As such this is the environment you should filter by for sentry errors or to change any config.
+
+After a sucessful deployment a comment will be added to the pull request with the URL to the review app for your convenience. When a pull request is updated e.g. more code is added it will re-deploy the new code.
+
+Once a pull request has been closed the review app infrastructure will be tore down to save on any costs. Should you wish to re-open a closed pull request the review app will be spun up again.
+
 ## Setting up Infrastructure for a new environment
 
 ### Staging

spec/models/user_spec.rb

@@ -159,6 +159,19 @@ RSpec.describe User, type: :model do
         expect(user.need_two_factor_authentication?(nil)).to be false
       end
     end
+
+    context "when the user is in review environment" do
+      let(:user) { FactoryBot.create(:user, :support) }
+
+      before do
+        allow(Rails.env).to receive(:development?).and_return(false)
+        allow(Rails.env).to receive(:review?).and_return(true)
+      end
+
+      it "does not require 2FA" do
+        expect(user.need_two_factor_authentication?(nil)).to be false
+      end
+    end
   end
 
   describe "paper trail" do