Merge remote-tracking branch 'origin/master' into js_agent_can_be_dryrun

.env.example

@@ -147,7 +147,7 @@ ENABLE_SECOND_PRECISION_SCHEDULE=false
 # Specify the scheduler frequency in seconds (default: 0.3).
 # Increasing this value will help reduce the use of system resources
 # at the expense of time accuracy.
-#SCHEDULER_FREQUENCY=0.3
+SCHEDULER_FREQUENCY=0.3
 
 # Use Graphviz for generating diagrams instead of using Google Chart
 # Tools.  Specify a dot(1) command path built with SVG support
@@ -159,3 +159,9 @@ TIMEZONE="Pacific Time (US & Canada)"
 
 # Number of failed jobs to keep in the database
 FAILED_JOBS_TO_KEEP=100
+
+# Maximum runtime of background jobs in minutes
+DELAYED_JOB_MAX_RUNTIME=20
+
+# Amount of seconds for delayed_job to sleep before checking for new jobs
+DELAYED_JOB_SLEEP_DELAY=10

Gemfile

@@ -58,6 +58,7 @@ gem 'daemons', '~> 1.1.9'
 gem 'delayed_job', '~> 4.0.0'
 gem 'delayed_job_active_record', '~> 4.0.0'
 gem 'devise', '~> 3.4.0'
+gem 'dotenv-rails', '~> 2.0.1'
 gem 'em-http-request', '~> 1.1.2'
 gem 'faraday', '~> 0.9.0'
 gem 'faraday_middleware'
@@ -96,27 +97,25 @@ group :development do
   gem 'guard'
   gem 'guard-livereload'
   gem 'guard-rspec'
-end
 
-group :development, :test do
-  gem 'coveralls', require: false
-  gem 'delorean'
-  gem 'dotenv-rails'
-  gem 'pry'
-  gem 'rr'
-  gem 'rspec', '~> 3.2'
-  gem 'rspec-collection_matchers', '~> 1.1.0'
-  gem 'rspec-rails', '~> 3.1'
-  gem 'rspec-html-matchers', '~> 0.7'
-  gem 'shoulda-matchers'
-  gem 'spring', '~> 1.3.0'
-  gem 'spring-commands-rspec'
-  gem 'vcr'
-  gem 'webmock', '~> 1.17.4', require: false
+  group :test do
+    gem 'coveralls', require: false
+    gem 'delorean'
+    gem 'pry'
+    gem 'rr'
+    gem 'rspec', '~> 3.2'
+    gem 'rspec-collection_matchers', '~> 1.1.0'
+    gem 'rspec-rails', '~> 3.1'
+    gem 'rspec-html-matchers', '~> 0.7'
+    gem 'shoulda-matchers'
+    gem 'spring', '~> 1.3.0'
+    gem 'spring-commands-rspec'
+    gem 'vcr'
+    gem 'webmock', '~> 1.17.4', require: false
+  end
 end
 
 group :production do
-  gem 'dotenv-deployment'
   gem 'rack'
 end
 
@@ -126,15 +125,23 @@ gem 'tzinfo', '>= 1.2.0'	# required by rails; 1.2.0 has support for *BSD and Sol
 # Windows does not have zoneinfo files, so bundle the tzinfo-data gem.
 gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw]
 
-# This hack needs some explanation.  When on Heroku, use the pg, unicorn, and rails12factor gems.
-# When not on Heroku, we still want our Gemfile.lock to include these gems, so we scope them to
-# an unsupported platform.
-if ENV['ON_HEROKU'] || ENV['HEROKU_POSTGRESQL_ROSE_URL'] || ENV['HEROKU_POSTGRESQL_GOLD_URL'] || File.read(File.join(File.dirname(__FILE__), 'Procfile')) =~ /intended for Heroku/
+# Introduces a scope for Heroku specific gems.
+def on_heroku
+  if ENV['ON_HEROKU'] ||
+     ENV['HEROKU_POSTGRESQL_ROSE_URL'] ||
+     ENV['HEROKU_POSTGRESQL_GOLD_URL'] ||
+     File.read(File.join(File.dirname(__FILE__), 'Procfile')) =~ /intended for Heroku/
+    yield
+  else
+    # When not on Heroku, we still want our Gemfile.lock to include
+    # Heroku specific gems, so we scope them to an unsupported
+    # platform.
+    platform :ruby_18, &proc
+  end
+end
+
+on_heroku do
   gem 'pg'
   gem 'unicorn'
   gem 'rails_12factor', group: :production
-else
-  gem 'pg', platform: :ruby_18
-  gem 'unicorn', platform: :ruby_18
-  gem 'rails_12factor', platform: :ruby_18
 end

Gemfile.lock

@@ -124,11 +124,9 @@ GEM
     docile (1.1.5)
     domain_name (0.5.24)
       unf (>= 0.0.5, < 1.0.0)
-    dotenv (0.11.1)
-      dotenv-deployment (~> 0.0.2)
-    dotenv-deployment (0.0.2)
-    dotenv-rails (0.11.1)
-      dotenv (= 0.11.1)
+    dotenv (2.0.1)
+    dotenv-rails (2.0.1)
+      dotenv (= 2.0.1)
     dropbox-api (0.4.2)
       hashie
       multi_json
@@ -507,8 +505,7 @@ DEPENDENCIES
   delayed_job_active_record (~> 4.0.0)
   delorean
   devise (~> 3.4.0)
-  dotenv-deployment
-  dotenv-rails
+  dotenv-rails (~> 2.0.1)
   dropbox-api
   em-http-request (~> 1.1.2)
   faraday (~> 0.9.0)

app/assets/javascripts/components/utils.js.coffee

@@ -33,3 +33,29 @@ class @Utils
       onHide?()
     body?(modal.querySelector('.modal-body'))
     $(modal).modal('show')
+
+  @handleDryRunButton: (button, data = $(button.form).serialize()) ->
+    $(button).prop('disabled', true)
+    $('body').css(cursor: 'progress')
+    $.ajax type: 'POST', url: $(button).data('action-url'), dataType: 'json', data: data
+      .always =>
+        $('body').css(cursor: 'auto')
+      .done (json) =>
+        Utils.showDynamicModal """
+          <h5>Log</h5>
+          <pre class="agent-dry-run-log"></pre>
+          <h5>Events</h5>
+          <pre class="agent-dry-run-events"></pre>
+          <h5>Memory</h5>
+          <pre class="agent-dry-run-memory"></pre>
+          """,
+          body: (body) ->
+            $(body).
+              find('.agent-dry-run-log').text(json.log).end().
+              find('.agent-dry-run-events').text(json.events).end().
+              find('.agent-dry-run-memory').text(json.memory)
+          title: 'Dry Run Results',
+          onHide: -> $(button).prop('disabled', false)
+      .fail (xhr, status, error) ->
+        alert('Error: ' + error)
+        $(button).prop('disabled', false)

app/assets/javascripts/pages/agent-edit-page.js.coffee

@@ -174,32 +174,7 @@ class @AgentEditPage
 
   invokeDryRun: (e) =>
     e.preventDefault()
-    button = e.target
-    $(button).prop('disabled', true)
-    $('body').css(cursor: 'progress')
-    @updateFromEditors()
-    $.ajax type: 'POST', url: $(button).data('action-url'), dataType: 'json', data: $(button.form).serialize()
-      .always =>
-        $("body").css(cursor: 'auto')
-      .done (json) =>
-        Utils.showDynamicModal """
-          <h5>Log</h5>
-          <pre class="agent-dry-run-log"></pre>
-          <h5>Events</h5>
-          <pre class="agent-dry-run-events"></pre>
-          <h5>Memory</h5>
-          <pre class="agent-dry-run-memory"></pre>
-          """,
-          body: (body) ->
-            $(body).
-              find('.agent-dry-run-log').text(json.log).end().
-              find('.agent-dry-run-events').text(json.events).end().
-              find('.agent-dry-run-memory').text(json.memory)
-          title: 'Dry Run Results',
-          onHide: -> $(button).prop('disabled', false)
-      .fail (xhr, status, error) ->
-        alert('Error: ' + error)
-        $(button).prop('disabled', false)
+    Utils.handleDryRunButton(this)
 
 $ ->
   Utils.registerPage(AgentEditPage, forPathsMatching: /^agents/)

app/assets/stylesheets/diagram.css.scss

@@ -8,13 +8,9 @@
   }
 
   .overlay-container {
-    position: absolute;
-    top: 0;
-    left: 0;
     z-index: auto;
 
     .overlay {
-      position: relative;
       z-index: auto;
       width: 100%;
       height: 100%;

app/concerns/dry_runnable.rb

@@ -25,6 +25,10 @@ class << self
     )
   end
 
+  def dry_run?
+    is_a? Sandbox
+  end
+
   module Sandbox
     attr_accessor :results
 

app/controllers/agents_controller.rb

@@ -35,15 +35,18 @@ def run
   end
 
   def dry_run
-    attrs = params[:agent]
+    attrs = params[:agent] || {}
     if agent = current_user.agents.find_by(id: params[:id])
       # PUT /agents/:id/dry_run
-      type = agent.type
+      if attrs.present?
+        type = agent.type
+        agent = Agent.build_for_type(type, current_user, attrs)
+      end
     else
       # POST /agents/dry_run
       type = attrs.delete(:type)
+      agent = Agent.build_for_type(type, current_user, attrs)
     end
-    agent = Agent.build_for_type(type, current_user, attrs)
     agent.name ||= '(Untitled)'
 
     if agent.valid?

app/controllers/diagrams_controller.rb

@@ -1,9 +1,13 @@
 class DiagramsController < ApplicationController
   def show
-    @agents = if params[:scenario_id].present?
-                current_user.scenarios.find(params[:scenario_id]).agents.includes(:receivers)
-              else
-                current_user.agents.includes(:receivers)
-              end
+    if params[:scenario_id].present?
+      @scenario = current_user.scenarios.find(params[:scenario_id])
+      agents = @scenario.agents
+    else
+      agents = current_user.agents
+    end
+    @disabled_agents = agents.inactive
+    agents = agents.active if params[:exclude_disabled].present?
+    @agents = agents.includes(:receivers)
   end
 end

app/models/agent.rb

@@ -60,7 +60,8 @@ class Agent < ActiveRecord::Base
   has_many :scenario_memberships, :dependent => :destroy, :inverse_of => :agent
   has_many :scenarios, :through => :scenario_memberships, :inverse_of => :agents
 
-  scope :active, -> { where(disabled: false) }
+  scope :active,   -> { where(disabled: false) }
+  scope :inactive, -> { where(disabled: true) }
 
   scope :of_type, lambda { |type|
     type = case type

app/models/agents/imap_folder_agent.rb

@@ -6,13 +6,15 @@ module Agents
   class ImapFolderAgent < Agent
     cannot_receive_events!
 
+    can_dry_run!
+
     default_schedule "every_30m"
 
     description <<-MD
 
       The ImapFolderAgent checks an IMAP server in specified folders
       and creates Events based on new mails found since the last run.
-      In the first visit to a foler, this agent only checks for the
+      In the first visit to a folder, this agent only checks for the
       initial status and does not create events.
 
       Specify an IMAP server to connect with `host`, and set `ssl` to
@@ -45,8 +47,8 @@ class ImapFolderAgent < Agent
           specified, will be chosen as the "body" value in a created
           event.
 
-          Named captues will appear in the "matches" hash in a created
-          event.
+          Named captures will appear in the "matches" hash in a
+          created event.
 
       - "from", "to", "cc"
 
@@ -311,7 +313,7 @@ def check
 
         if boolify(interpolated['mark_as_read'])
           log 'Marking as read'
-          mail.mark_as_read
+          mail.mark_as_read unless dry_run?
         end
       }
     end
@@ -322,7 +324,7 @@ def each_unread_mail
       port = (Integer(port) if port.present?)
 
       log "Connecting to #{host}#{':%d' % port if port}#{' via SSL' if ssl}"
-      Client.open(host, port, ssl) { |imap|
+      Client.open(host, port: port, ssl: ssl) { |imap|
         log "Logging in as #{username}"
         imap.login(username, interpolated[:password])
 
@@ -437,8 +439,8 @@ def pluralize(count, noun)
 
     class Client < ::Net::IMAP
       class << self
-        def open(host, port, ssl)
-          imap = new(host, port, ssl)
+        def open(host, *args)
+          imap = new(host, *args)
           yield imap
         ensure
           imap.disconnect unless imap.nil?
@@ -525,17 +527,19 @@ def initialize(client, fetch_data, props = {})
 
       def has_attachment?
         @has_attachment ||=
-          begin
-            data = @client.uid_fetch(@uid, 'BODYSTRUCTURE').first
+          if data = @client.uid_fetch(@uid, 'BODYSTRUCTURE').first
             struct_has_attachment?(data.attr['BODYSTRUCTURE'])
+          else
+            false
           end
       end
 
       def fetch
         @parsed ||=
-          begin
-            data = @client.uid_fetch(@uid, 'BODY.PEEK[]').first
+          if data = @client.uid_fetch(@uid, 'BODY.PEEK[]').first
             Mail.read_from_string(data.attr['BODY[]'])
+          else
+            Mail.read_from_string('')
           end
       end
 

app/views/agents/_action_menu.html.erb

@@ -5,6 +5,12 @@
     </li>
   <% end %>
 
+  <% if agent.can_dry_run? %>
+    <li>
+      <%= link_to icon_tag('glyphicon-refresh') + ' Dry Run', '#', 'data-action-url' => dry_run_agent_path(agent), tabindex: "-1", onclick: "Utils.handleDryRunButton(this, '_method=PUT')" %>
+    </li>
+  <% end %>
+
   <li>
     <%= link_to icon_tag('glyphicon-eye-open') + ' Show'.html_safe, agent_path(agent) %>
   </li>

app/views/diagrams/show.html.erb

@@ -9,7 +9,14 @@
         <h2>Agent Event Flow</h2>
       </div>
       <div class="btn-group">
-        <%= link_to icon_tag('glyphicon-chevron-left') + ' Back'.html_safe, (params[:scenario_id] ? scenario_path(params[:scenario_id]) : agents_path), class: "btn btn-default" %>
+        <%= link_to icon_tag('glyphicon-chevron-left') + ' Back'.html_safe, (@scenario ? scenario_path(@scenario) : agents_path), class: "btn btn-default" %>
+        <% if (num_disabled = @disabled_agents.count).nonzero? -%>
+          <% if params[:exclude_disabled] %>
+            <%= link_to @scenario ? scenario_diagram_path(@scenario) : diagram_path, class: 'btn btn-default' do %><%= icon_tag('glyphicon-eye-open') %> Show <%= pluralize(num_disabled, 'disabled Agent') %><% end %>
+          <% else %>
+            <%= link_to @scenario ? scenario_diagram_path(@scenario, exclude_disabled: true) : diagram_path(exclude_disabled: true), class: 'btn btn-default' do %><%= icon_tag('glyphicon-eye-close') %> Hide <%= pluralize(num_disabled, 'disabled Agent') %><% end %>
+          <% end %>
+        <% end %>
       </div>
 
       <div class='digraph'>

bin/threaded.rb

@@ -2,6 +2,8 @@
 require 'huginn_scheduler'
 require 'twitter_stream'
 
+Rails.configuration.cache_classes = true
+
 STDOUT.sync = true
 STDERR.sync = true
 

config/application.rb

@@ -4,10 +4,10 @@
 
 Bundler.require(:default, Rails.env)
 
-Dotenv.overload File.expand_path('../../spec/env.test', __FILE__) if Rails.env.test?
-
 module Huginn
   class Application < Rails::Application
+    Dotenv.overload File.expand_path('../../spec/env.test', __FILE__) if Rails.env.test?
+
     # Settings in config/environments/* take precedence over those specified here.
     # Application configuration should go into files in config/initializers
     # -- all .rb files in that directory are automatically loaded.