diff --git a/.env.example b/.env.example
index 3a487249be..28f6df2fb2 100644
--- a/.env.example
+++ b/.env.example
@@ -147,7 +147,7 @@ ENABLE_SECOND_PRECISION_SCHEDULE=false
# Specify the scheduler frequency in seconds (default: 0.3).
# Increasing this value will help reduce the use of system resources
# at the expense of time accuracy.
-#SCHEDULER_FREQUENCY=0.3
+SCHEDULER_FREQUENCY=0.3
# Use Graphviz for generating diagrams instead of using Google Chart
# Tools. Specify a dot(1) command path built with SVG support
@@ -159,3 +159,9 @@ TIMEZONE="Pacific Time (US & Canada)"
# Number of failed jobs to keep in the database
FAILED_JOBS_TO_KEEP=100
+
+# Maximum runtime of background jobs in minutes
+DELAYED_JOB_MAX_RUNTIME=20
+
+# Amount of seconds for delayed_job to sleep before checking for new jobs
+DELAYED_JOB_SLEEP_DELAY=10
\ No newline at end of file
diff --git a/Gemfile b/Gemfile
index b958f7b8d1..4c24f9c2bc 100644
--- a/Gemfile
+++ b/Gemfile
@@ -58,6 +58,7 @@ gem 'daemons', '~> 1.1.9'
gem 'delayed_job', '~> 4.0.0'
gem 'delayed_job_active_record', '~> 4.0.0'
gem 'devise', '~> 3.4.0'
+gem 'dotenv-rails', '~> 2.0.1'
gem 'em-http-request', '~> 1.1.2'
gem 'faraday', '~> 0.9.0'
gem 'faraday_middleware'
@@ -96,27 +97,25 @@ group :development do
gem 'guard'
gem 'guard-livereload'
gem 'guard-rspec'
-end
-group :development, :test do
- gem 'coveralls', require: false
- gem 'delorean'
- gem 'dotenv-rails'
- gem 'pry'
- gem 'rr'
- gem 'rspec', '~> 3.2'
- gem 'rspec-collection_matchers', '~> 1.1.0'
- gem 'rspec-rails', '~> 3.1'
- gem 'rspec-html-matchers', '~> 0.7'
- gem 'shoulda-matchers'
- gem 'spring', '~> 1.3.0'
- gem 'spring-commands-rspec'
- gem 'vcr'
- gem 'webmock', '~> 1.17.4', require: false
+ group :test do
+ gem 'coveralls', require: false
+ gem 'delorean'
+ gem 'pry'
+ gem 'rr'
+ gem 'rspec', '~> 3.2'
+ gem 'rspec-collection_matchers', '~> 1.1.0'
+ gem 'rspec-rails', '~> 3.1'
+ gem 'rspec-html-matchers', '~> 0.7'
+ gem 'shoulda-matchers'
+ gem 'spring', '~> 1.3.0'
+ gem 'spring-commands-rspec'
+ gem 'vcr'
+ gem 'webmock', '~> 1.17.4', require: false
+ end
end
group :production do
- gem 'dotenv-deployment'
gem 'rack'
end
@@ -126,15 +125,23 @@ gem 'tzinfo', '>= 1.2.0' # required by rails; 1.2.0 has support for *BSD and Sol
# Windows does not have zoneinfo files, so bundle the tzinfo-data gem.
gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw]
-# This hack needs some explanation. When on Heroku, use the pg, unicorn, and rails12factor gems.
-# When not on Heroku, we still want our Gemfile.lock to include these gems, so we scope them to
-# an unsupported platform.
-if ENV['ON_HEROKU'] || ENV['HEROKU_POSTGRESQL_ROSE_URL'] || ENV['HEROKU_POSTGRESQL_GOLD_URL'] || File.read(File.join(File.dirname(__FILE__), 'Procfile')) =~ /intended for Heroku/
+# Introduces a scope for Heroku specific gems.
+def on_heroku
+ if ENV['ON_HEROKU'] ||
+ ENV['HEROKU_POSTGRESQL_ROSE_URL'] ||
+ ENV['HEROKU_POSTGRESQL_GOLD_URL'] ||
+ File.read(File.join(File.dirname(__FILE__), 'Procfile')) =~ /intended for Heroku/
+ yield
+ else
+ # When not on Heroku, we still want our Gemfile.lock to include
+ # Heroku specific gems, so we scope them to an unsupported
+ # platform.
+ platform :ruby_18, &proc
+ end
+end
+
+on_heroku do
gem 'pg'
gem 'unicorn'
gem 'rails_12factor', group: :production
-else
- gem 'pg', platform: :ruby_18
- gem 'unicorn', platform: :ruby_18
- gem 'rails_12factor', platform: :ruby_18
end
diff --git a/Gemfile.lock b/Gemfile.lock
index 2e989c72be..34c3da2b11 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -124,11 +124,9 @@ GEM
docile (1.1.5)
domain_name (0.5.24)
unf (>= 0.0.5, < 1.0.0)
- dotenv (0.11.1)
- dotenv-deployment (~> 0.0.2)
- dotenv-deployment (0.0.2)
- dotenv-rails (0.11.1)
- dotenv (= 0.11.1)
+ dotenv (2.0.1)
+ dotenv-rails (2.0.1)
+ dotenv (= 2.0.1)
dropbox-api (0.4.2)
hashie
multi_json
@@ -507,8 +505,7 @@ DEPENDENCIES
delayed_job_active_record (~> 4.0.0)
delorean
devise (~> 3.4.0)
- dotenv-deployment
- dotenv-rails
+ dotenv-rails (~> 2.0.1)
dropbox-api
em-http-request (~> 1.1.2)
faraday (~> 0.9.0)
diff --git a/app/assets/javascripts/components/utils.js.coffee b/app/assets/javascripts/components/utils.js.coffee
index cf29e5c46f..7fb2b3c258 100644
--- a/app/assets/javascripts/components/utils.js.coffee
+++ b/app/assets/javascripts/components/utils.js.coffee
@@ -33,3 +33,29 @@ class @Utils
onHide?()
body?(modal.querySelector('.modal-body'))
$(modal).modal('show')
+
+ @handleDryRunButton: (button, data = $(button.form).serialize()) ->
+ $(button).prop('disabled', true)
+ $('body').css(cursor: 'progress')
+ $.ajax type: 'POST', url: $(button).data('action-url'), dataType: 'json', data: data
+ .always =>
+ $('body').css(cursor: 'auto')
+ .done (json) =>
+ Utils.showDynamicModal """
+ Log
+
+ Events
+
+ Memory
+
+ """,
+ body: (body) ->
+ $(body).
+ find('.agent-dry-run-log').text(json.log).end().
+ find('.agent-dry-run-events').text(json.events).end().
+ find('.agent-dry-run-memory').text(json.memory)
+ title: 'Dry Run Results',
+ onHide: -> $(button).prop('disabled', false)
+ .fail (xhr, status, error) ->
+ alert('Error: ' + error)
+ $(button).prop('disabled', false)
diff --git a/app/assets/javascripts/pages/agent-edit-page.js.coffee b/app/assets/javascripts/pages/agent-edit-page.js.coffee
index 62642de3cd..9adfe820d9 100644
--- a/app/assets/javascripts/pages/agent-edit-page.js.coffee
+++ b/app/assets/javascripts/pages/agent-edit-page.js.coffee
@@ -174,32 +174,7 @@ class @AgentEditPage
invokeDryRun: (e) =>
e.preventDefault()
- button = e.target
- $(button).prop('disabled', true)
- $('body').css(cursor: 'progress')
- @updateFromEditors()
- $.ajax type: 'POST', url: $(button).data('action-url'), dataType: 'json', data: $(button.form).serialize()
- .always =>
- $("body").css(cursor: 'auto')
- .done (json) =>
- Utils.showDynamicModal """
- Log
-
- Events
-
- Memory
-
- """,
- body: (body) ->
- $(body).
- find('.agent-dry-run-log').text(json.log).end().
- find('.agent-dry-run-events').text(json.events).end().
- find('.agent-dry-run-memory').text(json.memory)
- title: 'Dry Run Results',
- onHide: -> $(button).prop('disabled', false)
- .fail (xhr, status, error) ->
- alert('Error: ' + error)
- $(button).prop('disabled', false)
+ Utils.handleDryRunButton(this)
$ ->
Utils.registerPage(AgentEditPage, forPathsMatching: /^agents/)
diff --git a/app/assets/stylesheets/diagram.css.scss b/app/assets/stylesheets/diagram.css.scss
index c068ca62c2..ba5acb4dfd 100644
--- a/app/assets/stylesheets/diagram.css.scss
+++ b/app/assets/stylesheets/diagram.css.scss
@@ -8,13 +8,9 @@
}
.overlay-container {
- position: absolute;
- top: 0;
- left: 0;
z-index: auto;
.overlay {
- position: relative;
z-index: auto;
width: 100%;
height: 100%;
diff --git a/app/concerns/dry_runnable.rb b/app/concerns/dry_runnable.rb
index 165573eb96..d3951635f8 100644
--- a/app/concerns/dry_runnable.rb
+++ b/app/concerns/dry_runnable.rb
@@ -25,6 +25,10 @@ class << self
)
end
+ def dry_run?
+ is_a? Sandbox
+ end
+
module Sandbox
attr_accessor :results
diff --git a/app/controllers/agents_controller.rb b/app/controllers/agents_controller.rb
index d656b371bc..d19f7cd25c 100644
--- a/app/controllers/agents_controller.rb
+++ b/app/controllers/agents_controller.rb
@@ -35,15 +35,18 @@ def run
end
def dry_run
- attrs = params[:agent]
+ attrs = params[:agent] || {}
if agent = current_user.agents.find_by(id: params[:id])
# PUT /agents/:id/dry_run
- type = agent.type
+ if attrs.present?
+ type = agent.type
+ agent = Agent.build_for_type(type, current_user, attrs)
+ end
else
# POST /agents/dry_run
type = attrs.delete(:type)
+ agent = Agent.build_for_type(type, current_user, attrs)
end
- agent = Agent.build_for_type(type, current_user, attrs)
agent.name ||= '(Untitled)'
if agent.valid?
diff --git a/app/controllers/diagrams_controller.rb b/app/controllers/diagrams_controller.rb
index 6772ea4101..9092e0b204 100644
--- a/app/controllers/diagrams_controller.rb
+++ b/app/controllers/diagrams_controller.rb
@@ -1,9 +1,13 @@
class DiagramsController < ApplicationController
def show
- @agents = if params[:scenario_id].present?
- current_user.scenarios.find(params[:scenario_id]).agents.includes(:receivers)
- else
- current_user.agents.includes(:receivers)
- end
+ if params[:scenario_id].present?
+ @scenario = current_user.scenarios.find(params[:scenario_id])
+ agents = @scenario.agents
+ else
+ agents = current_user.agents
+ end
+ @disabled_agents = agents.inactive
+ agents = agents.active if params[:exclude_disabled].present?
+ @agents = agents.includes(:receivers)
end
end
diff --git a/app/models/agent.rb b/app/models/agent.rb
index c0aa3d8ab9..2d893d2573 100644
--- a/app/models/agent.rb
+++ b/app/models/agent.rb
@@ -60,7 +60,8 @@ class Agent < ActiveRecord::Base
has_many :scenario_memberships, :dependent => :destroy, :inverse_of => :agent
has_many :scenarios, :through => :scenario_memberships, :inverse_of => :agents
- scope :active, -> { where(disabled: false) }
+ scope :active, -> { where(disabled: false) }
+ scope :inactive, -> { where(disabled: true) }
scope :of_type, lambda { |type|
type = case type
diff --git a/app/models/agents/imap_folder_agent.rb b/app/models/agents/imap_folder_agent.rb
index c39abbb7e4..87959634b5 100644
--- a/app/models/agents/imap_folder_agent.rb
+++ b/app/models/agents/imap_folder_agent.rb
@@ -6,13 +6,15 @@ module Agents
class ImapFolderAgent < Agent
cannot_receive_events!
+ can_dry_run!
+
default_schedule "every_30m"
description <<-MD
The ImapFolderAgent checks an IMAP server in specified folders
and creates Events based on new mails found since the last run.
- In the first visit to a foler, this agent only checks for the
+ In the first visit to a folder, this agent only checks for the
initial status and does not create events.
Specify an IMAP server to connect with `host`, and set `ssl` to
@@ -45,8 +47,8 @@ class ImapFolderAgent < Agent
specified, will be chosen as the "body" value in a created
event.
- Named captues will appear in the "matches" hash in a created
- event.
+ Named captures will appear in the "matches" hash in a
+ created event.
- "from", "to", "cc"
@@ -311,7 +313,7 @@ def check
if boolify(interpolated['mark_as_read'])
log 'Marking as read'
- mail.mark_as_read
+ mail.mark_as_read unless dry_run?
end
}
end
@@ -322,7 +324,7 @@ def each_unread_mail
port = (Integer(port) if port.present?)
log "Connecting to #{host}#{':%d' % port if port}#{' via SSL' if ssl}"
- Client.open(host, port, ssl) { |imap|
+ Client.open(host, port: port, ssl: ssl) { |imap|
log "Logging in as #{username}"
imap.login(username, interpolated[:password])
@@ -437,8 +439,8 @@ def pluralize(count, noun)
class Client < ::Net::IMAP
class << self
- def open(host, port, ssl)
- imap = new(host, port, ssl)
+ def open(host, *args)
+ imap = new(host, *args)
yield imap
ensure
imap.disconnect unless imap.nil?
@@ -525,17 +527,19 @@ def initialize(client, fetch_data, props = {})
def has_attachment?
@has_attachment ||=
- begin
- data = @client.uid_fetch(@uid, 'BODYSTRUCTURE').first
+ if data = @client.uid_fetch(@uid, 'BODYSTRUCTURE').first
struct_has_attachment?(data.attr['BODYSTRUCTURE'])
+ else
+ false
end
end
def fetch
@parsed ||=
- begin
- data = @client.uid_fetch(@uid, 'BODY.PEEK[]').first
+ if data = @client.uid_fetch(@uid, 'BODY.PEEK[]').first
Mail.read_from_string(data.attr['BODY[]'])
+ else
+ Mail.read_from_string('')
end
end
diff --git a/app/views/agents/_action_menu.html.erb b/app/views/agents/_action_menu.html.erb
index d251acde2e..5a8a59e8da 100644
--- a/app/views/agents/_action_menu.html.erb
+++ b/app/views/agents/_action_menu.html.erb
@@ -5,6 +5,12 @@
<% end %>
+ <% if agent.can_dry_run? %>
+
+ <%= link_to icon_tag('glyphicon-refresh') + ' Dry Run', '#', 'data-action-url' => dry_run_agent_path(agent), tabindex: "-1", onclick: "Utils.handleDryRunButton(this, '_method=PUT')" %>
+
+ <% end %>
+
<%= link_to icon_tag('glyphicon-eye-open') + ' Show'.html_safe, agent_path(agent) %>
diff --git a/app/views/diagrams/show.html.erb b/app/views/diagrams/show.html.erb
index f09b0d0c1e..f106fb7367 100644
--- a/app/views/diagrams/show.html.erb
+++ b/app/views/diagrams/show.html.erb
@@ -9,7 +9,14 @@
Agent Event Flow
- <%= link_to icon_tag('glyphicon-chevron-left') + ' Back'.html_safe, (params[:scenario_id] ? scenario_path(params[:scenario_id]) : agents_path), class: "btn btn-default" %>
+ <%= link_to icon_tag('glyphicon-chevron-left') + ' Back'.html_safe, (@scenario ? scenario_path(@scenario) : agents_path), class: "btn btn-default" %>
+ <% if (num_disabled = @disabled_agents.count).nonzero? -%>
+ <% if params[:exclude_disabled] %>
+ <%= link_to @scenario ? scenario_diagram_path(@scenario) : diagram_path, class: 'btn btn-default' do %><%= icon_tag('glyphicon-eye-open') %> Show <%= pluralize(num_disabled, 'disabled Agent') %><% end %>
+ <% else %>
+ <%= link_to @scenario ? scenario_diagram_path(@scenario, exclude_disabled: true) : diagram_path(exclude_disabled: true), class: 'btn btn-default' do %><%= icon_tag('glyphicon-eye-close') %> Hide <%= pluralize(num_disabled, 'disabled Agent') %><% end %>
+ <% end %>
+ <% end %>
diff --git a/bin/threaded.rb b/bin/threaded.rb
index 56c99c46f7..9e68d825f6 100644
--- a/bin/threaded.rb
+++ b/bin/threaded.rb
@@ -2,6 +2,8 @@
require 'huginn_scheduler'
require 'twitter_stream'
+Rails.configuration.cache_classes = true
+
STDOUT.sync = true
STDERR.sync = true
diff --git a/config/application.rb b/config/application.rb
index f4156e403a..91e82fe8da 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -4,10 +4,10 @@
Bundler.require(:default, Rails.env)
-Dotenv.overload File.expand_path('../../spec/env.test', __FILE__) if Rails.env.test?
-
module Huginn
class Application < Rails::Application
+ Dotenv.overload File.expand_path('../../spec/env.test', __FILE__) if Rails.env.test?
+
# Settings in config/environments/* take precedence over those specified here.
# Application configuration should go into files in config/initializers
# -- all .rb files in that directory are automatically loaded.
diff --git a/config/initializers/delayed_job.rb b/config/initializers/delayed_job.rb
index cf2d29c1f3..5a55bdc2a8 100644
--- a/config/initializers/delayed_job.rb
+++ b/config/initializers/delayed_job.rb
@@ -1,9 +1,10 @@
Delayed::Worker.destroy_failed_jobs = false
Delayed::Worker.max_attempts = 5
-Delayed::Worker.max_run_time = 20.minutes
+Delayed::Worker.max_run_time = (ENV['DELAYED_JOB_MAX_RUNTIME'].presence || 20).to_i.minutes
Delayed::Worker.read_ahead = 5
Delayed::Worker.default_priority = 10
Delayed::Worker.delay_jobs = !Rails.env.test?
+Delayed::Worker.sleep_delay = (ENV['DELAYED_JOB_SLEEP_DELAY'].presence || 10).to_f
# Delayed::Worker.logger = Logger.new(Rails.root.join('log', 'delayed_job.log'))
# Delayed::Worker.logger.level = Logger::DEBUG
diff --git a/spec/concerns/dry_runnable_spec.rb b/spec/concerns/dry_runnable_spec.rb
index bfba97fe61..4eaf3f89a0 100644
--- a/spec/concerns/dry_runnable_spec.rb
+++ b/spec/concerns/dry_runnable_spec.rb
@@ -8,10 +8,10 @@ class Agents::SandboxedAgent < Agent
def check
log "Logging"
- create_event payload: { test: "foo" }
+ create_event payload: { 'test' => 'foo' }
error "Recording error"
- create_event payload: { test: "bar" }
- self.memory = { last_status: "ok" }
+ create_event payload: { 'test' => 'bar' }
+ self.memory = { 'last_status' => 'ok', 'dry_run' => dry_run? }
save!
end
end
@@ -24,18 +24,41 @@ def check
}
end
- it "traps logging, event emission and memory updating" do
+ def counts
+ [users(:bob).agents.count, users(:bob).events.count, users(:bob).logs.count]
+ end
+
+ it "does not affect normal run, with dry_run? returning false" do
+ before = counts
+ after = before.zip([0, 2, 2]).map { |x, d| x + d }
+
+ expect {
+ @agent.check
+ @agent.reload
+ }.to change { counts }.from(before).to(after)
+
+ expect(@agent.memory).to eq({ 'last_status' => 'ok', 'dry_run' => false })
+
+ payloads = @agent.events.reorder(:id).last(2).map(&:payload)
+ expect(payloads).to eq([{ 'test' => 'foo' }, { 'test' => 'bar' }])
+
+ messages = @agent.logs.reorder(:id).last(2).map(&:message)
+ expect(messages).to eq(['Logging', 'Recording error'])
+ end
+
+ it "traps logging, event emission and memory updating, with dry_run? returning true" do
results = nil
expect {
results = @agent.dry_run!
+ @agent.reload
}.not_to change {
- [users(:bob).agents.count, users(:bob).events.count, users(:bob).logs.count]
+ [@agent.memory, counts]
}
expect(results[:log]).to match(/\AI, .+ INFO -- : Logging\nE, .+ ERROR -- : Recording error\n/)
- expect(results[:events]).to eq([{ test: 'foo' }, { test: 'bar' }])
- expect(results[:memory]).to eq({ "last_status" => "ok" })
+ expect(results[:events]).to eq([{ 'test' => 'foo' }, { 'test' => 'bar' }])
+ expect(results[:memory]).to eq({ 'last_status' => 'ok', 'dry_run' => true })
end
it "does not perform dry-run if Agent does not support dry-run" do
@@ -45,8 +68,9 @@ def check
expect {
results = @agent.dry_run!
+ @agent.reload
}.not_to change {
- [users(:bob).agents.count, users(:bob).events.count, users(:bob).logs.count]
+ [@agent.memory, counts]
}
expect(results[:log]).to match(/\AE, .+ ERROR -- : Exception during dry-run. SandboxedAgent does not support dry-run: /)