diff --git a/Gemfile b/Gemfile index e6dac1995..c22504487 100644 --- a/Gemfile +++ b/Gemfile @@ -50,7 +50,7 @@ gem "paper_trail" gem "paper_trail-globalid" # Request rate limiting gem "rack-attack" -gem "redis" +gem "redis", "~> 4.8" # Receive exceptions and configure alerts gem "sentry-rails" gem "sentry-ruby" @@ -60,6 +60,7 @@ gem "possessive" gem "auto_strip_attributes" # Use sidekiq for background processing gem "sidekiq" +gem "sidekiq-cron" group :development, :test do # Check gems for known vulnerabilities diff --git a/Gemfile.lock b/Gemfile.lock index 2a9fb454f..340bdaaef 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -158,6 +158,8 @@ GEM rubocop smart_properties erubi (1.11.0) + et-orbi (1.2.7) + tzinfo excon (0.92.5) factory_bot (6.2.1) activesupport (>= 5.0.0) @@ -167,6 +169,9 @@ GEM faker (2.23.0) i18n (>= 1.8.11, < 2) ffi (1.15.5) + fugit (1.7.1) + et-orbi (~> 1, >= 1.2.7) + raabro (~> 1.4) globalid (1.0.0) activesupport (>= 5.0) govuk-components (3.2.1) @@ -267,6 +272,7 @@ GEM public_suffix (5.0.0) puma (5.6.5) nio4r (~> 2.0) + raabro (1.4.0) racc (1.6.0) rack (2.2.4) rack-attack (6.6.1) @@ -308,10 +314,7 @@ GEM rb-inotify (0.10.1) ffi (~> 1.0) redcarpet (3.5.1) - redis (5.0.5) - redis-client (>= 0.9.0) - redis-client (0.9.0) - connection_pool + redis (4.8.0) regexp_parser (2.5.0) request_store (1.5.1) rack (>= 1.4) @@ -385,6 +388,9 @@ GEM connection_pool (>= 2.2.2) rack (~> 2.0) redis (>= 4.5.0) + sidekiq-cron (1.8.0) + fugit (~> 1) + sidekiq (>= 4.2.1) simplecov (0.21.2) docile (~> 1.1) simplecov-html (~> 0.11) @@ -465,7 +471,7 @@ DEPENDENCIES rack-attack rack-mini-profiler (~> 2.0) rails (~> 7.0.2) - redis + redis (~> 4.8) roo rspec-rails rubocop-govuk (= 4.3.0) @@ -475,6 +481,7 @@ DEPENDENCIES sentry-rails sentry-ruby sidekiq + sidekiq-cron simplecov stimulus-rails timecop (~> 0.9.4) diff --git a/Procfile.dev b/Procfile.dev index 03c54b1d7..59b48dfe2 100644 --- a/Procfile.dev +++ b/Procfile.dev @@ -1,2 +1,3 @@ web: bin/rails server -p 3000 +redis: redis-server js: yarn build --watch diff --git a/app/jobs/data_export_csv_job.rb b/app/jobs/data_export_csv_job.rb new file mode 100644 index 000000000..8db19a202 --- /dev/null +++ b/app/jobs/data_export_csv_job.rb @@ -0,0 +1,10 @@ +class DataExportCsvJob < ApplicationJob + queue_as :default + + def perform + storage_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["EXPORT_PAAS_INSTANCE"]) + export_service = Exports::LettingsLogExportService.new(storage_service) + + export_service.export_csv_lettings_logs + end +end diff --git a/app/jobs/data_export_xml_job.rb b/app/jobs/data_export_xml_job.rb new file mode 100644 index 000000000..b26b65364 --- /dev/null +++ b/app/jobs/data_export_xml_job.rb @@ -0,0 +1,10 @@ +class DataExportXmlJob < ApplicationJob + queue_as :default + + def perform(full_update: false) + storage_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["EXPORT_PAAS_INSTANCE"]) + export_service = Exports::LettingsLogExportService.new(storage_service) + + export_service.export_xml_lettings_logs(full_update:) + end +end diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index fa377de99..4afb34d09 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -1,4 +1,5 @@ require "sidekiq/web" +require "sidekiq/cron/web" if Rails.env.staging? || Rails.env.production? redis_url = Configuration::PaasConfigurationService.new.redis_uris[:"dluhc-core-#{Rails.env}-redis"] @@ -11,3 +12,12 @@ if Rails.env.staging? || Rails.env.production? config.redis = { url: redis_url } end end + +# Until https://github.com/sidekiq-cron/sidekiq-cron/issues/357 is fixed. +Redis.silence_deprecations = true + +Sidekiq.configure_server do |config| + config.on(:startup) do + Sidekiq::Cron::Job.load_from_hash YAML.load_file("config/sidekiq_cron_schedule.yml") + end +end diff --git a/config/sidekiq_cron_schedule.yml b/config/sidekiq_cron_schedule.yml new file mode 100644 index 000000000..46cb10ff2 --- /dev/null +++ b/config/sidekiq_cron_schedule.yml @@ -0,0 +1,8 @@ +data_export_csv: + cron: "every day at 5am" + class: "DataExportCsvJob" + queue: default +data_export_xml: + cron: "every day at 5am" + class: "DataExportXmlJob" + queue: default diff --git a/db/schema.rb b/db/schema.rb index 80f37a6b0..868996566 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -14,6 +14,14 @@ ActiveRecord::Schema[7.0].define(version: 2022_10_19_082625) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" + create_table "collections", force: :cascade do |t| + t.datetime "start_date" + t.datetime "end_date" + t.string "type" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + end + create_table "data_protection_confirmations", force: :cascade do |t| t.bigint "organisation_id" t.bigint "data_protection_officer_id" @@ -237,12 +245,12 @@ ActiveRecord::Schema[7.0].define(version: 2022_10_19_082625) do t.integer "void_date_value_check" t.integer "housingneeds_type" t.integer "housingneeds_other" - t.index ["created_by_id"], name: "index_lettings_logs_on_created_by_id" - t.index ["location_id"], name: "index_lettings_logs_on_location_id" - t.index ["managing_organisation_id"], name: "index_lettings_logs_on_managing_organisation_id" - t.index ["old_id"], name: "index_lettings_logs_on_old_id", unique: true - t.index ["owning_organisation_id"], name: "index_lettings_logs_on_owning_organisation_id" - t.index ["scheme_id"], name: "index_lettings_logs_on_scheme_id" + t.index ["created_by_id"], name: "index_case_logs_on_created_by_id" + t.index ["location_id"], name: "index_case_logs_on_location_id" + t.index ["managing_organisation_id"], name: "index_case_logs_on_managing_organisation_id" + t.index ["old_id"], name: "index_case_logs_on_old_id", unique: true + t.index ["owning_organisation_id"], name: "index_case_logs_on_owning_organisation_id" + t.index ["scheme_id"], name: "index_case_logs_on_scheme_id" end create_table "locations", force: :cascade do |t| @@ -319,6 +327,26 @@ ActiveRecord::Schema[7.0].define(version: 2022_10_19_082625) do t.index ["old_visible_id"], name: "index_organisations_on_old_visible_id", unique: true end + create_table "pages", force: :cascade do |t| + t.string "name" + t.string "header" + t.string "description" + t.integer "subsection_id" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + end + + create_table "questions", force: :cascade do |t| + t.string "name" + t.string "check_answer_label" + t.string "header" + t.string "type" + t.integer "page_id" + t.integer "width" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + end + create_table "sales_logs", force: :cascade do |t| t.integer "status", default: 0 t.datetime "saledate" @@ -334,15 +362,15 @@ ActiveRecord::Schema[7.0].define(version: 2022_10_19_082625) do t.integer "jointmore" t.integer "jointpur" t.integer "beds" - t.integer "companybuy" t.integer "age1" t.integer "age1_known" t.string "sex1" + t.integer "buy1livein" + t.integer "companybuy" t.integer "national" t.string "othernational" t.integer "ethnic" t.integer "ethnic_group" - t.integer "buy1livein" t.integer "buylivein" t.integer "builtype" t.integer "proptype" @@ -355,18 +383,18 @@ ActiveRecord::Schema[7.0].define(version: 2022_10_19_082625) do t.integer "ecstat2" t.integer "privacynotice" t.integer "ecstat1" - t.integer "wheel" t.integer "hholdcount" + t.integer "wheel" t.integer "age3" t.integer "age3_known" - t.integer "income1" - t.integer "income1nk" t.integer "age4" t.integer "age4_known" t.integer "age5" t.integer "age5_known" t.integer "age6" t.integer "age6_known" + t.integer "income1" + t.integer "income1nk" t.index ["created_by_id"], name: "index_sales_logs_on_created_by_id" t.index ["managing_organisation_id"], name: "index_sales_logs_on_managing_organisation_id" t.index ["owning_organisation_id"], name: "index_sales_logs_on_owning_organisation_id" @@ -396,6 +424,23 @@ ActiveRecord::Schema[7.0].define(version: 2022_10_19_082625) do t.index ["owning_organisation_id"], name: "index_schemes_on_owning_organisation_id" end + create_table "sections", force: :cascade do |t| + t.string "header" + t.string "description" + t.string "name" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.integer "collection_id" + end + + create_table "subsections", force: :cascade do |t| + t.string "name" + t.string "label" + t.integer "section_id" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + end + create_table "users", force: :cascade do |t| t.string "email", default: "", null: false t.string "encrypted_password", default: "", null: false diff --git a/lib/tasks/data_export.rake b/lib/tasks/data_export.rake index 82433756c..4859103cf 100644 --- a/lib/tasks/data_export.rake +++ b/lib/tasks/data_export.rake @@ -4,13 +4,6 @@ namespace :core do format = args[:format] full_update = args[:full_update].present? && args[:full_update] == "true" - storage_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["EXPORT_PAAS_INSTANCE"]) - export_service = Exports::LettingsLogExportService.new(storage_service) - - if format.present? && format == "CSV" - export_service.export_csv_lettings_logs - else - export_service.export_xml_lettings_logs(full_update:) - end + DataExportJob.perform_later(format, full_update) end end