fix parsing of robots.txt

voicerepublic · Jul 23, 2014 · 70ebb03 · 70ebb03
1 parent 9401688
commit 70ebb03
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/app/middlewares/enforce_robots_txt.rb b/app/middlewares/enforce_robots_txt.rb
@@ -32,7 +32,9 @@ def call(env)
   private
 
   def robots_txt
-    @robots_txt ||= File.read(opts[:source]).split.inject([]) do |result, line|
+    return @robots_txt unless @robots_txt.nil?
+    lines = File.read(opts[:source]).split("\n")
+    @robots_txt = lines.inject([]) do |result, line|
       case line
       when /User-agent:\s*(.+)\s*$/ then result << [$1, []]
       when /Disallow:\s*(.+)\s*$/ then result.last.last << $1