baarkerlounger
/
submit-social-housing-lettings-and-sales-data
mirror of https://github.com/communitiesuk/submit-social-housing-lettings-and-sales-data.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Schedule data export tasks in sidekiq-cron 

- Split combined XML and CSV data jobs into seperate background jobs
- Add sidekiq-cron to the project
- Schedule new XML and CSV jobs to run every day at 5am
data-export-jobs
James Rose 2 years ago
parent
c710d642cc
commit
9e2a5d48f6
9 changed files with 110 additions and 25 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 3
      Gemfile
  2. 17
      Gemfile.lock
  3. 1
      Procfile.dev
  4. 10
      app/jobs/data_export_csv_job.rb
  5. 10
      app/jobs/data_export_xml_job.rb
  6. 10
      config/initializers/sidekiq.rb
  7. 8
      config/sidekiq_cron_schedule.yml
  8. 67
      db/schema.rb
  9. 9
      lib/tasks/data_export.rake

3
Gemfile
Unescape View File

@ -50,7 +50,7 @@ gem "paper_trail"
gem "paper_trail-globalid"
# Request rate limiting
gem "rack-attack"
gem "redis"
gem "redis", "~> 4.8"
# Receive exceptions and configure alerts
gem "sentry-rails"
gem "sentry-ruby"
@ -60,6 +60,7 @@ gem "possessive"
gem "auto_strip_attributes"
# Use sidekiq for background processing
gem "sidekiq"
gem "sidekiq-cron"
group :development, :test do
# Check gems for known vulnerabilities

17
Gemfile.lock
Unescape View File

@ -158,6 +158,8 @@ GEM
rubocop
smart_properties
erubi (1.11.0)
et-orbi (1.2.7)
tzinfo
excon (0.92.5)
factory_bot (6.2.1)
activesupport (>= 5.0.0)
@ -167,6 +169,9 @@ GEM
faker (2.23.0)
i18n (>= 1.8.11, < 2)
ffi (1.15.5)
fugit (1.7.1)
et-orbi (~> 1, >= 1.2.7)
raabro (~> 1.4)
globalid (1.0.0)
activesupport (>= 5.0)
govuk-components (3.2.1)
@ -267,6 +272,7 @@ GEM
public_suffix (5.0.0)
puma (5.6.5)
nio4r (~> 2.0)
raabro (1.4.0)
racc (1.6.0)
rack (2.2.4)
rack-attack (6.6.1)
@ -308,10 +314,7 @@ GEM
rb-inotify (0.10.1)
ffi (~> 1.0)
redcarpet (3.5.1)
redis (5.0.5)
redis-client (>= 0.9.0)
redis-client (0.9.0)
connection_pool
redis (4.8.0)
regexp_parser (2.5.0)
request_store (1.5.1)
rack (>= 1.4)
@ -385,6 +388,9 @@ GEM
connection_pool (>= 2.2.2)
rack (~> 2.0)
redis (>= 4.5.0)
sidekiq-cron (1.8.0)
fugit (~> 1)
sidekiq (>= 4.2.1)
simplecov (0.21.2)
docile (~> 1.1)
simplecov-html (~> 0.11)
@ -465,7 +471,7 @@ DEPENDENCIES
rack-attack
rack-mini-profiler (~> 2.0)
rails (~> 7.0.2)
redis
redis (~> 4.8)
roo
rspec-rails
rubocop-govuk (= 4.3.0)
@ -475,6 +481,7 @@ DEPENDENCIES
sentry-rails
sentry-ruby
sidekiq
sidekiq-cron
simplecov
stimulus-rails
timecop (~> 0.9.4)

1
Procfile.dev
Unescape View File

@ -1,2 +1,3 @@
web: bin/rails server -p 3000
redis: redis-server
js: yarn build --watch

10
app/jobs/data_export_csv_job.rb
Unescape View File

@ -0,0 +1,10 @@
class DataExportCsvJob < ApplicationJob
queue_as :default
def perform
storage_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["EXPORT_PAAS_INSTANCE"])
export_service = Exports::LettingsLogExportService.new(storage_service)
export_service.export_csv_lettings_logs
end
end

10
app/jobs/data_export_xml_job.rb
Unescape View File

@ -0,0 +1,10 @@
class DataExportXmlJob < ApplicationJob
queue_as :default
def perform(full_update: false)
storage_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["EXPORT_PAAS_INSTANCE"])
export_service = Exports::LettingsLogExportService.new(storage_service)
export_service.export_xml_lettings_logs(full_update:)
end
end

10
config/initializers/sidekiq.rb
Unescape View File

@ -1,4 +1,5 @@
require "sidekiq/web"
require "sidekiq/cron/web"
if Rails.env.staging? || Rails.env.production?
redis_url = Configuration::PaasConfigurationService.new.redis_uris[:"dluhc-core-#{Rails.env}-redis"]
@ -11,3 +12,12 @@ if Rails.env.staging? || Rails.env.production?
config.redis = { url: redis_url }
end
end
# Until https://github.com/sidekiq-cron/sidekiq-cron/issues/357 is fixed.
Redis.silence_deprecations = true
Sidekiq.configure_server do |config|
config.on(:startup) do
Sidekiq::Cron::Job.load_from_hash YAML.load_file("config/sidekiq_cron_schedule.yml")
end
end

8
config/sidekiq_cron_schedule.yml
Unescape View File

@ -0,0 +1,8 @@
data_export_csv:
cron: "every day at 5am"
class: "DataExportCsvJob"
queue: default
data_export_xml:
cron: "every day at 5am"
class: "DataExportXmlJob"
queue: default

67
db/schema.rb
Unescape View File

@ -14,6 +14,14 @@ ActiveRecord::Schema[7.0].define(version: 2022_10_19_082625) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
create_table "collections", force: :cascade do |t|
t.datetime "start_date"
t.datetime "end_date"
t.string "type"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
end
create_table "data_protection_confirmations", force: :cascade do |t|
t.bigint "organisation_id"
t.bigint "data_protection_officer_id"
@ -237,12 +245,12 @@ ActiveRecord::Schema[7.0].define(version: 2022_10_19_082625) do
t.integer "void_date_value_check"
t.integer "housingneeds_type"
t.integer "housingneeds_other"
t.index ["created_by_id"], name: "index_lettings_logs_on_created_by_id"
t.index ["location_id"], name: "index_lettings_logs_on_location_id"
t.index ["managing_organisation_id"], name: "index_lettings_logs_on_managing_organisation_id"
t.index ["old_id"], name: "index_lettings_logs_on_old_id", unique: true
t.index ["owning_organisation_id"], name: "index_lettings_logs_on_owning_organisation_id"
t.index ["scheme_id"], name: "index_lettings_logs_on_scheme_id"
t.index ["created_by_id"], name: "index_case_logs_on_created_by_id"
t.index ["location_id"], name: "index_case_logs_on_location_id"
t.index ["managing_organisation_id"], name: "index_case_logs_on_managing_organisation_id"
t.index ["old_id"], name: "index_case_logs_on_old_id", unique: true
t.index ["owning_organisation_id"], name: "index_case_logs_on_owning_organisation_id"
t.index ["scheme_id"], name: "index_case_logs_on_scheme_id"
end
create_table "locations", force: :cascade do |t|
@ -319,6 +327,26 @@ ActiveRecord::Schema[7.0].define(version: 2022_10_19_082625) do
t.index ["old_visible_id"], name: "index_organisations_on_old_visible_id", unique: true
end
create_table "pages", force: :cascade do |t|
t.string "name"
t.string "header"
t.string "description"
t.integer "subsection_id"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
end
create_table "questions", force: :cascade do |t|
t.string "name"
t.string "check_answer_label"
t.string "header"
t.string "type"
t.integer "page_id"
t.integer "width"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
end
create_table "sales_logs", force: :cascade do |t|
t.integer "status", default: 0
t.datetime "saledate"
@ -334,15 +362,15 @@ ActiveRecord::Schema[7.0].define(version: 2022_10_19_082625) do
t.integer "jointmore"
t.integer "jointpur"
t.integer "beds"
t.integer "companybuy"
t.integer "age1"
t.integer "age1_known"
t.string "sex1"
t.integer "buy1livein"
t.integer "companybuy"
t.integer "national"
t.string "othernational"
t.integer "ethnic"
t.integer "ethnic_group"
t.integer "buy1livein"
t.integer "buylivein"
t.integer "builtype"
t.integer "proptype"
@ -355,18 +383,18 @@ ActiveRecord::Schema[7.0].define(version: 2022_10_19_082625) do
t.integer "ecstat2"
t.integer "privacynotice"
t.integer "ecstat1"
t.integer "wheel"
t.integer "hholdcount"
t.integer "wheel"
t.integer "age3"
t.integer "age3_known"
t.integer "income1"
t.integer "income1nk"
t.integer "age4"
t.integer "age4_known"
t.integer "age5"
t.integer "age5_known"
t.integer "age6"
t.integer "age6_known"
t.integer "income1"
t.integer "income1nk"
t.index ["created_by_id"], name: "index_sales_logs_on_created_by_id"
t.index ["managing_organisation_id"], name: "index_sales_logs_on_managing_organisation_id"
t.index ["owning_organisation_id"], name: "index_sales_logs_on_owning_organisation_id"
@ -396,6 +424,23 @@ ActiveRecord::Schema[7.0].define(version: 2022_10_19_082625) do
t.index ["owning_organisation_id"], name: "index_schemes_on_owning_organisation_id"
end
create_table "sections", force: :cascade do |t|
t.string "header"
t.string "description"
t.string "name"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.integer "collection_id"
end
create_table "subsections", force: :cascade do |t|
t.string "name"
t.string "label"
t.integer "section_id"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
end
create_table "users", force: :cascade do |t|
t.string "email", default: "", null: false
t.string "encrypted_password", default: "", null: false

9
lib/tasks/data_export.rake
Unescape View File

@ -4,13 +4,6 @@ namespace :core do
format = args[:format]
full_update = args[:full_update].present? && args[:full_update] == "true"
storage_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["EXPORT_PAAS_INSTANCE"])
export_service = Exports::LettingsLogExportService.new(storage_service)
if format.present? && format == "CSV"
export_service.export_csv_lettings_logs
else
export_service.export_xml_lettings_logs(full_update:)
end
DataExportJob.perform_later(format, full_update)
end
end

Write Preview
Loading…
Cancel
Save