Skip to content

Commit

Permalink
Exclude deflate from Accept-Encoding by default
Browse files Browse the repository at this point in the history
Some web servers return a raw deflate stream instead of a zlib-wrapped
deflate stream defined in RFCs (called "DEFLATE") when they say
`Content-Encoding: deflate`.  However FaradayMiddleware::Gzip currently
only supports the latter, so we need to keep it from telling a server
that it accepts `deflate`.

See huginn#1018.
  • Loading branch information
knu committed Sep 11, 2015
1 parent 06b022f commit 7bcb5e1
Show file tree
Hide file tree
Showing 2 changed files with 35 additions and 0 deletions.
5 changes: 5 additions & 0 deletions app/concerns/web_request_concern.rb
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,11 @@ def faraday

builder.use FaradayMiddleware::Gzip

unless builder.headers.any? { |key,| /\Aaccept[-_]encoding\z/i =~ key }
# Exclude `deflate` by default. See #1018.
builder.headers[:accept_encoding] = 'gzip,identity'
end

case backend = faraday_backend
when :typhoeus
require 'typhoeus/adapters/faraday'
Expand Down
30 changes: 30 additions & 0 deletions spec/models/agents/website_agent_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,36 @@
event = Event.last
expect(event.payload['version']).to eq(2)
end

it 'should either avoid or support a raw deflate stream (#1018)' do
stub_request(:any, /deflate/).with(headers: { 'Accept-Encoding' => /\A(?!.*deflate)/ }).
to_return(body: 'hello',
status: 200)
stub_request(:any, /deflate/).with(headers: { 'Accept-Encoding' => /deflate/ }).
to_return(body: '\xcb\x48\xcd\xc9\xc9\x07\x00\x06\x2c'.force_encoding(Encoding::ASCII_8BIT),
headers: { 'Content-Encoding' => 'deflate' },
status: 200)

site = {
'name' => 'Some Response',
'expected_update_period_in_days' => '2',
'type' => 'text',
'url' => 'http://deflate',
'mode' => 'on_change',
'extract' => {
'content' => { 'regexp' => '.+', 'index' => 0 }
}
}
checker = Agents::WebsiteAgent.new(name: "Deflate Test", options: site)
checker.user = users(:bob)
checker.save!

expect {
checker.check
}.to change { Event.count }.by(1)
event = Event.last
expect(event.payload['content']).to eq('hello')
end
end

describe 'encoding' do
Expand Down

0 comments on commit 7bcb5e1

Please sign in to comment.