From a0fad5c8bb0eaf9fea619f94d102045f8c589c30 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 3 Aug 2023 11:04:05 +0200 Subject: [PATCH] Change indexing frequency from 5 minutes to 1 minute, add locks to schedulers (#26304) --- app/workers/scheduler/follow_recommendations_scheduler.rb | 2 +- app/workers/scheduler/indexing_scheduler.rb | 6 ++---- app/workers/scheduler/instance_refresh_scheduler.rb | 2 +- app/workers/scheduler/ip_cleanup_scheduler.rb | 2 +- app/workers/scheduler/pghero_scheduler.rb | 2 +- app/workers/scheduler/scheduled_statuses_scheduler.rb | 2 +- app/workers/scheduler/suspended_user_cleanup_scheduler.rb | 2 +- app/workers/scheduler/user_cleanup_scheduler.rb | 2 +- app/workers/scheduler/vacuum_scheduler.rb | 2 +- config/sidekiq.yml | 2 +- 10 files changed, 11 insertions(+), 13 deletions(-) diff --git a/app/workers/scheduler/follow_recommendations_scheduler.rb b/app/workers/scheduler/follow_recommendations_scheduler.rb index 17cf3f2cc..ea59ad986 100644 --- a/app/workers/scheduler/follow_recommendations_scheduler.rb +++ b/app/workers/scheduler/follow_recommendations_scheduler.rb @@ -4,7 +4,7 @@ class Scheduler::FollowRecommendationsScheduler include Sidekiq::Worker include Redisable - sidekiq_options retry: 0 + sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i # The maximum number of accounts that can be requested in one page from the # API is 80, and the suggestions API does not allow pagination. This number diff --git a/app/workers/scheduler/indexing_scheduler.rb b/app/workers/scheduler/indexing_scheduler.rb index d622f5586..2868a3b71 100644 --- a/app/workers/scheduler/indexing_scheduler.rb +++ b/app/workers/scheduler/indexing_scheduler.rb @@ -4,7 +4,7 @@ class Scheduler::IndexingScheduler include Sidekiq::Worker include Redisable - sidekiq_options retry: 0 + sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i IMPORT_BATCH_SIZE = 1000 SCAN_BATCH_SIZE = 10 * IMPORT_BATCH_SIZE @@ -16,9 +16,7 @@ class Scheduler::IndexingScheduler with_redis do |redis| redis.sscan_each("chewy:queue:#{type.name}", count: SCAN_BATCH_SIZE).each_slice(IMPORT_BATCH_SIZE) do |ids| type.import!(ids) - redis.pipelined do |pipeline| - pipeline.srem("chewy:queue:#{type.name}", ids) - end + redis.srem("chewy:queue:#{type.name}", ids) end end end diff --git a/app/workers/scheduler/instance_refresh_scheduler.rb b/app/workers/scheduler/instance_refresh_scheduler.rb index 59f91dc62..818682a5d 100644 --- a/app/workers/scheduler/instance_refresh_scheduler.rb +++ b/app/workers/scheduler/instance_refresh_scheduler.rb @@ -3,7 +3,7 @@ class Scheduler::InstanceRefreshScheduler include Sidekiq::Worker - sidekiq_options retry: 0 + sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i def perform Instance.refresh diff --git a/app/workers/scheduler/ip_cleanup_scheduler.rb b/app/workers/scheduler/ip_cleanup_scheduler.rb index 8f607db03..f78c0584d 100644 --- a/app/workers/scheduler/ip_cleanup_scheduler.rb +++ b/app/workers/scheduler/ip_cleanup_scheduler.rb @@ -6,7 +6,7 @@ class Scheduler::IpCleanupScheduler IP_RETENTION_PERIOD = ENV.fetch('IP_RETENTION_PERIOD', 1.year).to_i.seconds.freeze SESSION_RETENTION_PERIOD = ENV.fetch('SESSION_RETENTION_PERIOD', 1.year).to_i.seconds.freeze - sidekiq_options retry: 0 + sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i def perform clean_ip_columns! diff --git a/app/workers/scheduler/pghero_scheduler.rb b/app/workers/scheduler/pghero_scheduler.rb index a756b13b9..49bf2db7a 100644 --- a/app/workers/scheduler/pghero_scheduler.rb +++ b/app/workers/scheduler/pghero_scheduler.rb @@ -3,7 +3,7 @@ class Scheduler::PgheroScheduler include Sidekiq::Worker - sidekiq_options retry: 0 + sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i def perform PgHero.capture_space_stats diff --git a/app/workers/scheduler/scheduled_statuses_scheduler.rb b/app/workers/scheduler/scheduled_statuses_scheduler.rb index 3bf6300b3..b5801248f 100644 --- a/app/workers/scheduler/scheduled_statuses_scheduler.rb +++ b/app/workers/scheduler/scheduled_statuses_scheduler.rb @@ -3,7 +3,7 @@ class Scheduler::ScheduledStatusesScheduler include Sidekiq::Worker - sidekiq_options retry: 0 + sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i def perform publish_scheduled_statuses! diff --git a/app/workers/scheduler/suspended_user_cleanup_scheduler.rb b/app/workers/scheduler/suspended_user_cleanup_scheduler.rb index 87e22161b..90feead67 100644 --- a/app/workers/scheduler/suspended_user_cleanup_scheduler.rb +++ b/app/workers/scheduler/suspended_user_cleanup_scheduler.rb @@ -16,7 +16,7 @@ class Scheduler::SuspendedUserCleanupScheduler # has the capacity for it. MAX_DELETIONS_PER_JOB = 10 - sidekiq_options retry: 0 + sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i def perform return if Sidekiq::Queue.new('pull').size > MAX_PULL_SIZE diff --git a/app/workers/scheduler/user_cleanup_scheduler.rb b/app/workers/scheduler/user_cleanup_scheduler.rb index 4aee7935a..2d2efc731 100644 --- a/app/workers/scheduler/user_cleanup_scheduler.rb +++ b/app/workers/scheduler/user_cleanup_scheduler.rb @@ -3,7 +3,7 @@ class Scheduler::UserCleanupScheduler include Sidekiq::Worker - sidekiq_options retry: 0 + sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i def perform clean_unconfirmed_accounts! diff --git a/app/workers/scheduler/vacuum_scheduler.rb b/app/workers/scheduler/vacuum_scheduler.rb index 9c040f6e4..1c9a2aabe 100644 --- a/app/workers/scheduler/vacuum_scheduler.rb +++ b/app/workers/scheduler/vacuum_scheduler.rb @@ -3,7 +3,7 @@ class Scheduler::VacuumScheduler include Sidekiq::Worker - sidekiq_options retry: 0, lock: :until_executed + sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i def perform vacuum_operations.each do |operation| diff --git a/config/sidekiq.yml b/config/sidekiq.yml index 2278329a5..12c45c22a 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -23,7 +23,7 @@ class: Scheduler::Trends::ReviewNotificationsScheduler queue: scheduler indexing_scheduler: - every: '5m' + interval: 1 minute class: Scheduler::IndexingScheduler queue: scheduler vacuum_scheduler: