Resque in Production

Background Everything

@davetron5000

Resque is a Redis-backed Ruby library for creating background jobs, placing them on multiple queues, and processing them later.

STITCH FIX

Lead Engineer

class UsersController
  def create
    user = User.new(params[:user])
    if user.save
      UserMailer.welcome(user).deliver
    else
      render 'new'
    end
  end
end

class UsersController
  def create
    user = User.new(params[:user])
    if user.save
      UserMailer.welcome(user).deliver
    else
      render 'new'
    end
  end
end

Set Up Resque

Write the code
Configure it
Set up worker processes

class UsersController
  def create
    user = User.new(params[:user])
    if user.save
      Resque.enqueue(UserWelcomeMailerJob,user.id)
    else
      render 'new'
    end
  end
end

class UserWelcomeMailerJob
  @queue = :mail
  def self.perform(user_id)
    user = User.find(user_id)
    UserMailer.welcome(user).deliver
  end
end

class UserWelcomeMailerJob
  @queue = :mail
  def self.perform(user_id)
    user = User.find(user_id)
    UserMailer.welcome(user).deliver
  end
end

class UserWelcomeMailerJob
  @queue = :mail
  def self.perform(user_id)
    user = User.find(user_id)
    UserMailer.welcome(user).deliver
  end
end

# config/initializers/resque.rb
configuration = {
  host: ENV['RESQUE_REDIS_HOST'],
  port: ENV['RESQUE_REDIS_PORT'],
}
Resque.redis = Redis.new(configuration)

>  echo "require 'resque/tasks'" > lib/tasks/resque.rake
-> Rake task created
>  rake environment resque:work QUEUE=*
-> Worker is now running

New Problem

(the same one, actually)

class PurchasesController
  def create
    customer = current_customer
    purchase = Purchase.new(params[:purchase].merge(customer: customer))
    if purchase.valid?
      Purchase.transaction do
        customer.last_order_date = Time.now
        purchase.save!
        customer.save!
        PurchaseMailer.confirm_purchase(purchase).deliver
      end
    else
      render 'new'
    end
  end
end

class PurchasesController
  def create
    customer = current_customer
    purchase = Purchase.new(params[:purchase].merge(customer: customer))
    if purchase.valid?
      Purchase.transaction do
        customer.last_order_date = Time.now
        purchase.save!
        customer.save!
        PurchaseMailer.confirm_purchase(purchase).deliver
      end
    else
      render 'new'
    end
  end
end

class PurchasesController
  def create
    customer = current_customer
    purchase = Purchase.new(params[:purchase].merge(customer: customer))
    if purchase.valid?
      Purchase.transaction do
        customer.last_order_date = Time.now
        purchase.save!
        customer.save!
        Resque.enqueue(ConfirmPurhaseMailerJob,purchase.id)
      end
    else
      render 'new'
    end
  end
end

class ConfirmPurhaseMailerJob
  @queue = :mail
  def self.perform(purchase_id)
    purchase = Purchase.find(purchase_id)
    PurchaseMailer.confirm_purchase(purchase).deliver
  end
end

class UserWelcomeMailerJob
  @queue = :mail
  def self.perform(user_id)
    user = User.find(user_id)
    UserMailer.welcome(user).deliver
  end
end

class XXXMailerJob
  @queue = :mail
  def self.perform(some_stuff_id)
    some_stuff = SomeStuff.find(id)
    XXXMailer.send_email(some_stuff)
  end
end

>  echo "gem 'resque_mailer'" >> Gemfile
>  bundle install
-> gem installed

class PurchaseMailer < ActionMailer::Base


  def self.confirm_purchase(purchase)

    # ...    
  end
end

class UserMailer < ActionMailer::Base
  

  def self.welcome(user)

    # ...    
  end
end

class PurchaseMailer < ActionMailer::Base
  include Resque::Mailer

  def self.confirm_purchase(purchase_id)
    purchase = Purchase.find(purchase_id)
    # ...    
  end
end

class UserMailer < ActionMailer::Base
  include Resque::Mailer

  def self.welcome(user_id)
    user = User.find(user_id)
    # ...    
  end
end

class PurchaseMailer < ActionMailer::Base
  include Resque::Mailer

  def self.confirm_purchase(purchase_id)
    purchase = Purchase.find(purchase_id)
    # ...    
  end
end

class UserMailer < ActionMailer::Base
  include Resque::Mailer

  def self.welcome(user_id)
    user = User.find(user_id)
    # ...    
  end
end

class UsersController
  def create
    user = User.new(params[:user])
    if user.save
      UserMailer.welcome(user.id).deliver
    else
      render 'new'
    end
  end
end

class PurchasesController
  def create
    customer = current_customer
    purchase = Purchase.new(params[:purchase].merge(customer: customer))
    if purchase.valid?
      Purchase.transaction do
        customer.last_order_date = Time.now
        purchase.save!
        customer.save!
        PurchaseMailer.confirm_purchase(purchase).deliver
      end
    else
      render 'new'
    end
  end
end

Purchase emails still failing


mount Resque::Server, at: 'resque'

class PurchasesController
  def create
    customer = current_customer
    purchase = Purchase.new(params[:purchase].merge(customer: customer))
    if purchase.valid?
      Purchase.transaction do
        customer.last_order_date = Time.now
        purchase.save!
        customer.save!
        PurchaseMailer.confirm_purchase(purchase).deliver
      end
    else
      render 'new'
    end
  end
end

>  echo "gem 'resque-retry'" >> Gemfile
>  bundle install
-> gem installed

# config/initializers/resque.rb
require 'resque_scheduler'
require 'resque-retry'
require 'resque_scheduler/server'
require 'resque-retry/server'

configuration = {
  host: ENV['RESQUE_REDIS_HOST'],
  port: ENV['RESQUE_REDIS_PORT'],
}
Resque.redis = Redis.new(configuration)

require 'resque/failure/redis'
Resque::Failure::MultipleWithRetrySuppression.classes = [Resque::Failure::Redis]
Resque::Failure.backend = Resque::Failure::MultipleWithRetrySuppression

# config/initializers/resque.rb
require 'resque_scheduler'
require 'resque-retry'
require 'resque_scheduler/server'
require 'resque-retry/server'

configuration = {
  host: ENV['RESQUE_REDIS_HOST'],
  port: ENV['RESQUE_REDIS_PORT'],
}
Resque.redis = Redis.new(configuration)

require 'resque/failure/redis'
Resque::Failure::MultipleWithRetrySuppression.classes = [Resque::Failure::Redis]
Resque::Failure.backend = Resque::Failure::MultipleWithRetrySuppression

# config/initializers/resque.rb
require 'resque_scheduler'
require 'resque-retry'
require 'resque_scheduler/server'
require 'resque-retry/server'

configuration = {
  host: ENV['RESQUE_REDIS_HOST'],
  port: ENV['RESQUE_REDIS_PORT'],
}
Resque.redis = Redis.new(configuration)

require 'resque/failure/redis'
Resque::Failure::MultipleWithRetrySuppression.classes = [Resque::Failure::Redis]
Resque::Failure.backend = Resque::Failure::MultipleWithRetrySuppression

# Cribbed from https://blog.engineyard.com/2011/the-resque-way
class AsyncApplicationMailer < ActionMailer::Base
  include Resque::Mailer
  extend Resque::Plugins::Retry
end

class UserMailer < AsyncApplicationMailer

  # ...

end

>  rake environment resque:scheduler
-> retries can now be scheduled and processed

# config/initializers/resque.rb
require 'resque_scheduler'
require 'resque-retry'
require 'resque_scheduler/server'
require 'resque-retry/server'

configuration = {
  host: ENV['RESQUE_REDIS_HOST'],
  port: ENV['RESQUE_REDIS_PORT'],
}
Resque.redis = Redis.new(configuration)

require 'resque/failure/redis'
Resque::Failure::MultipleWithRetrySuppression.classes = [Resque::Failure::Redis]
Resque::Failure.backend = Resque::Failure::MultipleWithRetrySuppression

namespace :monitor do
  namespace :resque do
    task :failures do
      num_failures = Resque::Failure.count
      if num_failures > 0
        Rails.logger.error("#{num_failures} failures in the resque failed queue!")
        ResqueMailer.failed_jobs(num_failures).deliver! # <- bang!
      else
        Rails.logger.info("All clear!")
      end
    end
  end
end

namespace :monitor do
  namespace :resque do
    task :failures do
      num_failures = Resque::Failure.count
      if num_failures > 0
        Rails.logger.error("#{num_failures} failures in the resque failed queue!")
        ResqueMailer.failed_jobs(num_failures).deliver! # <- bang!
      else
        Rails.logger.info("All clear!")
      end
    end
  end
end

class PurchaseChargeJob
  @queue = :purchasing
  def self.perform(purchase_id)
    purchase = Purchase.find(purchase_id)
    purchase.capture_authorization!
    purchase.generate_pack_in_materials!
  end
end

class PurchaseChargeJob
  @queue = :purchasing
  def self.perform(purchase_id)
    purchase = Purchase.find(purchase_id)
    purchase.capture_authorization!
    Resque.enqueue(PackInMaterialsJob,purchase.id)
  end
end

class PackInMaterialsJob 
  @queue = :purchasing
  def self.perform(purchase_id)
    purchase = Purchase.find(purchase_id)
    purchase.generate_pack_in_materials!
  end
end

class PackInMaterialsJob 
  extend Resque::Plugins::Retry
  @queue = :purchasing
  @retry_limit = 5
  @retry_delay = 90 #seconds

  def self.perform(purchase_id)
    purchase = Purchase.find(purchase_id)
    purchase.generate_pack_in_materials!
  end
end

Jobs not being processed, but not failing

class AdminIndexJob
  @queue = :admin
  def self.perform(customer_id)
    # a whole lot of slow code
    # that builds a cached copy of
    # the customer and all his 
    # or her data
  end
end

>  rake environment resque:work QUEUE=mail,purchasing,admin
-> now purchasing takes a back seat
>  rake environment resque:work QUEUE=mail,purchasing,admin
>  rake environment resque:work QUEUE=purchasing,mail,admin
-> now things are getting confusing

>  rake environment resque:work QUEUE=mail
>  rake environment resque:work QUEUE=purchasing
>  rake environment resque:work QUEUE=admin
-> simple, assuming you can spare the CPU and memory

Resque Tips

Do not use 2.0 - use 1.x
All external contact would be run in a job
All email sent in a job
All jobs configured to retry
All jobs idempotent
Jobs are fine-grained
Each queue has its own worker
Each app has its own Redis
Everything monitored

THANKS!

Slides: http://bit.ly/resque-prod

Work For Stitch Fix: tech.stitchfix.com/blog

Buy The Senior Software Engineer: sweng.me

Buy Build Awesome Command-Line Apps in Ruby: pragprog.com/book/dccar2