Schedule data export tasks in sidekiq-cron

- Split combined XML and CSV data jobs into seperate background jobs
- Add sidekiq-cron to the project
- Schedule new XML and CSV jobs to run every day at 5am

Gemfile

@@ -50,7 +50,7 @@ gem "paper_trail"
 gem "paper_trail-globalid"
 # Request rate limiting
 gem "rack-attack"
-gem "redis"
+gem "redis", "~> 4.8"
 # Receive exceptions and configure alerts
 gem "sentry-rails"
 gem "sentry-ruby"
@@ -60,6 +60,7 @@ gem "possessive"
 gem "auto_strip_attributes"
 # Use sidekiq for background processing
 gem "sidekiq"
+gem "sidekiq-cron"
 
 group :development, :test do
   # Check gems for known vulnerabilities

Gemfile.lock

@@ -158,6 +158,8 @@ GEM
       rubocop
       smart_properties
     erubi (1.11.0)
+    et-orbi (1.2.7)
+      tzinfo
     excon (0.92.5)
     factory_bot (6.2.1)
       activesupport (>= 5.0.0)
@@ -167,6 +169,9 @@ GEM
     faker (2.23.0)
       i18n (>= 1.8.11, < 2)
     ffi (1.15.5)
+    fugit (1.7.1)
+      et-orbi (~> 1, >= 1.2.7)
+      raabro (~> 1.4)
     globalid (1.0.0)
       activesupport (>= 5.0)
     govuk-components (3.2.1)
@@ -267,6 +272,7 @@ GEM
     public_suffix (5.0.0)
     puma (5.6.5)
       nio4r (~> 2.0)
+    raabro (1.4.0)
     racc (1.6.0)
     rack (2.2.4)
     rack-attack (6.6.1)
@@ -308,10 +314,7 @@ GEM
     rb-inotify (0.10.1)
       ffi (~> 1.0)
     redcarpet (3.5.1)
-    redis (5.0.5)
+    redis (4.8.0)
-      redis-client (>= 0.9.0)
-    redis-client (0.9.0)
-      connection_pool
     regexp_parser (2.5.0)
     request_store (1.5.1)
       rack (>= 1.4)
@@ -385,6 +388,9 @@ GEM
       connection_pool (>= 2.2.2)
       rack (~> 2.0)
       redis (>= 4.5.0)
+    sidekiq-cron (1.8.0)
+      fugit (~> 1)
+      sidekiq (>= 4.2.1)
     simplecov (0.21.2)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
@@ -465,7 +471,7 @@ DEPENDENCIES
   rack-attack
   rack-mini-profiler (~> 2.0)
   rails (~> 7.0.2)
-  redis
+  redis (~> 4.8)
   roo
   rspec-rails
   rubocop-govuk (= 4.3.0)
@@ -475,6 +481,7 @@ DEPENDENCIES
   sentry-rails
   sentry-ruby
   sidekiq
+  sidekiq-cron
   simplecov
   stimulus-rails
   timecop (~> 0.9.4)

Procfile.dev

@@ -1,2 +1,3 @@
 web: bin/rails server -p 3000
+redis: redis-server
 js: yarn build --watch

app/jobs/data_export_csv_job.rb

@@ -0,0 +1,10 @@
+class DataExportCsvJob < ApplicationJob
+  queue_as :default
+
+  def perform
+    storage_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["EXPORT_PAAS_INSTANCE"])
+    export_service = Exports::LettingsLogExportService.new(storage_service)
+
+    export_service.export_csv_lettings_logs
+  end
+end

app/jobs/data_export_xml_job.rb

@@ -0,0 +1,10 @@
+class DataExportXmlJob < ApplicationJob
+  queue_as :default
+
+  def perform(full_update: false)
+    storage_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["EXPORT_PAAS_INSTANCE"])
+    export_service = Exports::LettingsLogExportService.new(storage_service)
+
+    export_service.export_xml_lettings_logs(full_update:)
+  end
+end

config/initializers/sidekiq.rb

@@ -1,4 +1,5 @@
 require "sidekiq/web"
+require "sidekiq/cron/web"
 
 if Rails.env.staging? || Rails.env.production?
   redis_url = Configuration::PaasConfigurationService.new.redis_uris[:"dluhc-core-#{Rails.env}-redis"]
@@ -11,3 +12,12 @@ if Rails.env.staging? || Rails.env.production?
     config.redis = { url: redis_url }
   end
 end
+
+# Until https://github.com/sidekiq-cron/sidekiq-cron/issues/357 is fixed.
+Redis.silence_deprecations = true
+
+Sidekiq.configure_server do |config|
+  config.on(:startup) do
+    Sidekiq::Cron::Job.load_from_hash YAML.load_file("config/sidekiq_cron_schedule.yml")
+  end
+end

config/sidekiq_cron_schedule.yml

@@ -0,0 +1,8 @@
+data_export_csv:
+  cron: "every day at 5am"
+  class: "DataExportCsvJob"
+  queue: default
+data_export_xml:
+  cron: "every day at 5am"
+  class: "DataExportXmlJob"
+  queue: default

db/schema.rb

@@ -14,6 +14,14 @@ ActiveRecord::Schema[7.0].define(version: 2022_10_19_082625) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
+  create_table "collections", force: :cascade do |t|
+    t.datetime "start_date"
+    t.datetime "end_date"
+    t.string "type"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+  end
+
   create_table "data_protection_confirmations", force: :cascade do |t|
     t.bigint "organisation_id"
     t.bigint "data_protection_officer_id"
@@ -237,12 +245,12 @@ ActiveRecord::Schema[7.0].define(version: 2022_10_19_082625) do
     t.integer "void_date_value_check"
     t.integer "housingneeds_type"
     t.integer "housingneeds_other"
-    t.index ["created_by_id"], name: "index_lettings_logs_on_created_by_id"
+    t.index ["created_by_id"], name: "index_case_logs_on_created_by_id"
-    t.index ["location_id"], name: "index_lettings_logs_on_location_id"
+    t.index ["location_id"], name: "index_case_logs_on_location_id"
-    t.index ["managing_organisation_id"], name: "index_lettings_logs_on_managing_organisation_id"
+    t.index ["managing_organisation_id"], name: "index_case_logs_on_managing_organisation_id"
-    t.index ["old_id"], name: "index_lettings_logs_on_old_id", unique: true
+    t.index ["old_id"], name: "index_case_logs_on_old_id", unique: true
-    t.index ["owning_organisation_id"], name: "index_lettings_logs_on_owning_organisation_id"
+    t.index ["owning_organisation_id"], name: "index_case_logs_on_owning_organisation_id"
-    t.index ["scheme_id"], name: "index_lettings_logs_on_scheme_id"
+    t.index ["scheme_id"], name: "index_case_logs_on_scheme_id"
   end
 
   create_table "locations", force: :cascade do |t|
@@ -319,6 +327,26 @@ ActiveRecord::Schema[7.0].define(version: 2022_10_19_082625) do
     t.index ["old_visible_id"], name: "index_organisations_on_old_visible_id", unique: true
   end
 
+  create_table "pages", force: :cascade do |t|
+    t.string "name"
+    t.string "header"
+    t.string "description"
+    t.integer "subsection_id"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+  end
+
+  create_table "questions", force: :cascade do |t|
+    t.string "name"
+    t.string "check_answer_label"
+    t.string "header"
+    t.string "type"
+    t.integer "page_id"
+    t.integer "width"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+  end
+
   create_table "sales_logs", force: :cascade do |t|
     t.integer "status", default: 0
     t.datetime "saledate"
@@ -334,15 +362,15 @@ ActiveRecord::Schema[7.0].define(version: 2022_10_19_082625) do
     t.integer "jointmore"
     t.integer "jointpur"
     t.integer "beds"
-    t.integer "companybuy"
     t.integer "age1"
     t.integer "age1_known"
     t.string "sex1"
+    t.integer "buy1livein"
+    t.integer "companybuy"
     t.integer "national"
     t.string "othernational"
     t.integer "ethnic"
     t.integer "ethnic_group"
-    t.integer "buy1livein"
     t.integer "buylivein"
     t.integer "builtype"
     t.integer "proptype"
@@ -355,18 +383,18 @@ ActiveRecord::Schema[7.0].define(version: 2022_10_19_082625) do
     t.integer "ecstat2"
     t.integer "privacynotice"
     t.integer "ecstat1"
-    t.integer "wheel"
     t.integer "hholdcount"
+    t.integer "wheel"
     t.integer "age3"
     t.integer "age3_known"
-    t.integer "income1"
-    t.integer "income1nk"
     t.integer "age4"
     t.integer "age4_known"
     t.integer "age5"
     t.integer "age5_known"
     t.integer "age6"
     t.integer "age6_known"
+    t.integer "income1"
+    t.integer "income1nk"
     t.index ["created_by_id"], name: "index_sales_logs_on_created_by_id"
     t.index ["managing_organisation_id"], name: "index_sales_logs_on_managing_organisation_id"
     t.index ["owning_organisation_id"], name: "index_sales_logs_on_owning_organisation_id"
@@ -396,6 +424,23 @@ ActiveRecord::Schema[7.0].define(version: 2022_10_19_082625) do
     t.index ["owning_organisation_id"], name: "index_schemes_on_owning_organisation_id"
   end
 
+  create_table "sections", force: :cascade do |t|
+    t.string "header"
+    t.string "description"
+    t.string "name"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.integer "collection_id"
+  end
+
+  create_table "subsections", force: :cascade do |t|
+    t.string "name"
+    t.string "label"
+    t.integer "section_id"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+  end
+
   create_table "users", force: :cascade do |t|
     t.string "email", default: "", null: false
     t.string "encrypted_password", default: "", null: false

lib/tasks/data_export.rake

@@ -4,13 +4,6 @@ namespace :core do
     format = args[:format]
     full_update = args[:full_update].present? && args[:full_update] == "true"
 
-    storage_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["EXPORT_PAAS_INSTANCE"])
+    DataExportJob.perform_later(format, full_update)
-    export_service = Exports::LettingsLogExportService.new(storage_service)
-
-    if format.present? && format == "CSV"
-      export_service.export_csv_lettings_logs
-    else
-      export_service.export_xml_lettings_logs(full_update:)
-    end
   end
 end