feat: first attempt at improved http backoff (#164)

lib/unleash/metrics_reporter.rb

@@ -1,5 +1,6 @@
 require 'unleash/configuration'
 require 'unleash/metrics'
+require 'unleash/util/executor_result'
 require 'net/http'
 require 'json'
 require 'time'
@@ -35,6 +36,7 @@ def generate_report
  report
  end
 
+ # @return [integer]
  def post
  Unleash.logger.debug "post() Report"
 
@@ -48,8 +50,10 @@ def post
 
  if ['200', '202'].include? response.code
  Unleash.logger.debug "Report sent to unleash server successfully. Server responded with http code #{response.code}"
+ Unleash::Util::ExecutorResult::SUCCESS
  else
  Unleash.logger.error "Error when sending report to unleash server. Server responded with http code #{response.code}."
+ Unleash::Util::ExecutorResult::TEMPORARY_FAILURE
  end
  end
 

lib/unleash/scheduled_executor.rb

@@ -1,3 +1,5 @@
+require 'unleash/util/executor_result'
+
 module Unleash
  class ScheduledExecutor
  attr_accessor :name, :interval, :max_exceptions, :retry_count, :thread, :immediate_execution
@@ -53,8 +55,18 @@ def exit
  def run_blk(&blk)
  Unleash.logger.debug "thread #{name} starting execution"
 
- yield(blk)
- self.retry_count = 0
+ case yield(blk)
+ when Unleash::Util::ExecutorResult::SUCCESS
+ self.retry_count = 0
+ when Unleash::Util::ExecutorResult::TEMPORARY_FAILURE
+ Unleash.logger.warn "thread #{name} ended with temporary failure"
+ when Unleash::Util::ExecutorResult::PERMANENT_FAILURE
+ self.retry_count += 1
+ Unleash.logger.error "thread #{name} ended with permanent failure " \
+ " (#{self.retry_count}/#{self.max_exceptions}): '#{e}'"
+ else
+ Unleash.logger.warn "thread #{name} ended with unknown status, assuming temporary"
+ end
  rescue StandardError => e
  self.retry_count += 1
  Unleash.logger.error "thread #{name} threw exception #{e.class} " \

lib/unleash/toggle_fetcher.rb

@@ -1,5 +1,6 @@
 require 'unleash/configuration'
 require 'unleash/bootstrap/handler'
+require 'unleash/util/executor_result'
 require 'net/http'
 require 'json'
 
@@ -20,7 +21,9 @@ def initialize
  if Unleash.configuration.use_bootstrap?
  bootstrap
  else
- fetch
+ ret = fetch
+ # https://docs.ruby-lang.org/en/3.2/Exception.html#class-Exception-label-Custom+Exceptions
+ raise StandardError "Unable to fetch toggles from the server" unless ret == Unleash::Util::ExecutorResult::SUCCESS
  end
  rescue StandardError => e
  # fail back to reading the backup file
@@ -32,6 +35,7 @@ def initialize
  # once initialized, somewhere else you will want to start a loop with fetch()
  end
 
+ # @return [Hash]
  def toggles
  self.toggle_lock.synchronize do
  # wait for resource, only if it is null
@@ -41,17 +45,31 @@ def toggles
  end
 
  # rename to refresh_from_server! ??
+ # @return [integer]
  def fetch
  Unleash.logger.debug "fetch()"
- return if Unleash.configuration.disable_client
+ return Unleash::Util::ExecutorResult::SUCCESS if Unleash.configuration.disable_client
 
  response = Unleash::Util::Http.get(Unleash.configuration.fetch_toggles_uri, etag)
 
- if response.code == '304'
+ case response.code
+ when '304' # Net::HTTPNotModified
  Unleash.logger.debug "No changes according to the unleash server, nothing to do."
- return
- elsif response.code != '200'
- raise IOError, "Unleash server returned a non 200/304 HTTP result."
+ return Unleash::Util::ExecutorResult::SUCCESS
+ when '200' # Net::HTTPOK
+ Unleash.logger.debug "Received 200 OK from unleash server, will update local cache."
+ when '429' # Net::HTTPTooManyRequests
+ Unleash.logger.warn "Unleash server requested via HTTP result that we retry later. HTTP response code: #{response.code}"
+ return Unleash::Util::ExecutorResult::TEMPORARY_FAILURE
+ when '500'..'599' # Net::HTTPServerError
+ Unleash.logger.warn "Unleash server returned a server error. Consider it a permanent failure. HTTP response code: #{response.code}"
+ return Unleash::Util::ExecutorResult::TEMPORARY_FAILURE
+ when '401', '403', '404' # Net::HTTPUnauthorized, Net::HTTPForbidden, Net::HTTPNotFound
+ Unleash.logger.debug "Unleash server returned invalid code. Consider it a permanent failure. HTTP response code: #{response.code}"
+ return Unleash::Util::ExecutorResult::PERMANENT_FAILURE
+ else
+ Unleash.logger.error "Unleash server returned an unexpected result. Consider it a permanent failure. HTTP response code: #{response.code}"
+ return Unleash::Util::ExecutorResult::PERMANENT_FAILURE
  end
 
  self.etag = response['ETag']
@@ -62,6 +80,8 @@ def fetch
 
  update_running_client!
  save!
+
+ Unleash::Util::ExecutorResult::SUCCESS
  end
 
  def save!

lib/unleash/util/executor_result.rb

@@ -0,0 +1,13 @@
+module Unleash
+ module Util
+
+ module ExecutorResult
+ SUCCESS = 0
+ TEMPORARY_FAILURE = 1
+ PERMANENT_FAILURE = 2
+ end
+ # class RetriableError < ::StandardError
+ # end
+
+ end
+end