diff --git a/.env.example b/.env.example index 3235504ce4..3a487249be 100644 --- a/.env.example +++ b/.env.example @@ -19,7 +19,7 @@ DATABASE_ADAPTER=mysql2 DATABASE_ENCODING=utf8 DATABASE_RECONNECT=true DATABASE_NAME=huginn_development -DATABASE_POOL=5 +DATABASE_POOL=10 DATABASE_USERNAME=root DATABASE_PASSWORD="" #DATABASE_HOST=your-domain-here.com @@ -101,6 +101,9 @@ TUMBLR_OAUTH_SECRET= DROPBOX_OAUTH_KEY= DROPBOX_OAUTH_SECRET= +WUNDERLIST_OAUTH_KEY= +WUNDERLIST_OAUTH_SECRET= + ############################# # AWS and Mechanical Turk # ############################# diff --git a/Gemfile b/Gemfile index b6cbc7d1d5..b958f7b8d1 100644 --- a/Gemfile +++ b/Gemfile @@ -8,19 +8,21 @@ gem 'net-ftp-list', '~> 3.2.8' # FtpsiteAgent gem 'wunderground', '~> 1.2.0' # WeatherAgent gem 'forecast_io', '~> 2.0.0' # WeatherAgent gem 'rturk', '~> 2.12.1' # HumanTaskAgent -gem 'weibo_2', '~> 0.1' # Weibo Agents gem 'hipchat', '~> 1.2.0' # HipchatAgent gem 'xmpp4r', '~> 0.5.6' # JabberAgent gem 'mqtt' # MQTTAgent gem 'slack-notifier', '~> 1.0.0' # SlackAgent gem 'hypdf', '~> 1.0.7' # PDFInfoAgent +# Weibo Agents +gem 'weibo_2', github: 'cantino/weibo_2', branch: 'master' + # GoogleCalendarPublishAgent gem "google-api-client", require: 'google/api_client' # Twitter Agents -gem 'twitter', '~> 5.8.0' # Must to be loaded before cantino-twitter-stream. -gem 'twitter-stream', github: 'dsander/twitter-stream', branch: 'huginn' +gem 'twitter', '~> 5.14.0' # Must to be loaded before cantino-twitter-stream. +gem 'twitter-stream', github: 'cantino/twitter-stream', branch: 'huginn' gem 'omniauth-twitter' # Tumblr Agents @@ -37,6 +39,7 @@ gem 'haversine' # Optional Services. gem 'omniauth-37signals' # BasecampAgent # gem 'omniauth-github' +gem 'omniauth-wunderlist', github: 'wunderlist/omniauth-wunderlist', ref: 'd0910d0396107b9302aa1bc50e74bb140990ccb8' # Bundler <1.5 does not recognize :x64_mingw as a valid platform name. # Unfortunately, it can't self-update because it errors when encountering :x64_mingw. diff --git a/Gemfile.lock b/Gemfile.lock index e0ad71c70a..2e989c72be 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,12 +1,32 @@ GIT - remote: git://github.com/dsander/twitter-stream.git - revision: 1713b4fe5b387580364b39716bb5c26d6601c50f + remote: git://github.com/cantino/twitter-stream.git + revision: f7e7edb0bae013bffabf3598e7147773d9fd370f branch: huginn specs: twitter-stream (0.1.15) eventmachine (~> 1.0.7) http_parser.rb (~> 0.6.0) - simple_oauth (~> 0.2.0) + simple_oauth (~> 0.3.0) + +GIT + remote: git://github.com/cantino/weibo_2.git + revision: 00e57d29d8252126014b038cd738b02e05e4cfc5 + branch: master + specs: + weibo_2 (0.1.7) + hashie (~> 2.0.4) + multi_json (~> 1) + oauth2 (~> 0.9.1) + rest-client (~> 1.8) + +GIT + remote: git://github.com/wunderlist/omniauth-wunderlist.git + revision: d0910d0396107b9302aa1bc50e74bb140990ccb8 + ref: d0910d0396107b9302aa1bc50e74bb140990ccb8 + specs: + omniauth-wunderlist (0.0.1) + omniauth (~> 1.0) + omniauth-oauth2 (~> 1.1) GEM remote: https://rubygems.org/ @@ -47,7 +67,7 @@ GEM minitest (~> 5.1) thread_safe (~> 0.3, >= 0.3.4) tzinfo (~> 1.1) - addressable (2.3.7) + addressable (2.3.8) arel (6.0.0) autoparse (0.3.3) addressable (>= 2.3.1) @@ -102,6 +122,8 @@ GEM warden (~> 1.2.3) diff-lcs (1.2.5) docile (1.1.5) + domain_name (0.5.24) + unf (>= 0.0.5, < 1.0.0) dotenv (0.11.1) dotenv-deployment (~> 0.0.2) dotenv-deployment (0.0.2) @@ -122,7 +144,7 @@ GEM em-websocket (0.5.1) eventmachine (>= 0.12.9) http_parser.rb (~> 0.6.0) - equalizer (0.0.9) + equalizer (0.0.11) erector (0.10.0) treetop (>= 1.2.3) erubis (2.7.0) @@ -190,8 +212,10 @@ GEM httmultiparty (0.3.10) httparty (>= 0.7.3) multipart-post - http (0.5.1) - http_parser.rb + http (0.6.4) + http_parser.rb (~> 0.6.0) + http-cookie (1.0.2) + domain_name (~> 0.5) http_parser.rb (0.6.0) httparty (0.13.1) json (~> 1.8) @@ -205,7 +229,7 @@ GEM json (1.8.2) jsonpath (0.5.6) multi_json - jwt (1.3.0) + jwt (1.4.1) kaminari (0.16.1) actionpack (>= 3.0.0) activesupport (>= 3.0.0) @@ -229,7 +253,7 @@ GEM memoizable (0.4.2) thread_safe (~> 0.3, >= 0.3.1) method_source (0.8.2) - mime-types (2.4.3) + mime-types (2.5) mini_portile (0.6.2) minitest (5.5.1) mqtt (0.3.1) @@ -281,7 +305,7 @@ GEM slop (~> 3.4) quiet_assets (1.1.0) railties (>= 3.1, < 5.0) - rack (1.6.0) + rack (1.6.1) rack-test (0.6.3) rack (>= 1.0) rails (4.2.1) @@ -321,7 +345,8 @@ GEM ref (1.0.5) responders (2.1.0) railties (>= 4.2.0, < 5) - rest-client (1.7.3) + rest-client (1.8.0) + http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 3.0) netrc (~> 0.7) retriable (2.0.2) @@ -378,7 +403,7 @@ GEM jwt (>= 0.1.5) multi_json (>= 1.0.0) simple-rss (1.3.1) - simple_oauth (0.2.0) + simple_oauth (0.3.1) simplecov (0.9.2) docile (~> 1.1.0) multi_json (~> 1.0) @@ -426,17 +451,17 @@ GEM builder (>= 2.1.2) jwt (>= 0.1.2) multi_json (>= 1.3.0) - twitter (5.8.0) + twitter (5.14.0) addressable (~> 2.3) buftok (~> 0.2.0) equalizer (~> 0.0.9) faraday (~> 0.9.0) - http (~> 0.5.0) + http (~> 0.6.0) http_parser.rb (~> 0.6.0) json (~> 1.8) memoizable (~> 0.4.0) naught (~> 1.0) - simple_oauth (~> 0.2.0) + simple_oauth (~> 0.3.0) typhoeus (0.6.9) ethon (>= 0.7.1) tzinfo (1.2.2) @@ -444,6 +469,9 @@ GEM uglifier (2.7.0) execjs (>= 0.3.0) json (>= 1.8.0) + unf (0.1.4) + unf_ext + unf_ext (0.0.7.1) unicorn (4.8.3) kgio (~> 2.6) rack @@ -457,11 +485,6 @@ GEM webmock (1.17.4) addressable (>= 2.2.7) crack (>= 0.3.2) - weibo_2 (0.1.7) - hashie (~> 2.0.4) - multi_json (~> 1) - oauth2 (~> 0.9.1) - rest-client (~> 1.7.3) wunderground (1.2.0) addressable httparty (> 0.6.0) @@ -521,6 +544,7 @@ DEPENDENCIES omniauth-dropbox omniauth-tumblr omniauth-twitter + omniauth-wunderlist! pg protected_attributes (~> 1.0.8) pry @@ -547,7 +571,7 @@ DEPENDENCIES therubyracer (~> 0.12.2) tumblr_client twilio-ruby (~> 3.11.5) - twitter (~> 5.8.0) + twitter (~> 5.14.0) twitter-stream! typhoeus (~> 0.6.3) tzinfo (>= 1.2.0) @@ -556,6 +580,6 @@ DEPENDENCIES unicorn vcr webmock (~> 1.17.4) - weibo_2 (~> 0.1) + weibo_2! wunderground (~> 1.2.0) xmpp4r (~> 0.5.6) diff --git a/README.md b/README.md index dc24ee7667..a6d4404f0c 100644 --- a/README.md +++ b/README.md @@ -8,32 +8,32 @@ Huginn is a system for building agents that perform automated tasks for you onli ![the origin of the name](doc/imgs/the-name.png) -#### Here are some of the things that you can do with Huginn right now: +#### Here are some of the things that you can do with Huginn: * Track the weather and get an email when it's going to rain (or snow) tomorrow ("Don't forget your umbrella!") -* List terms that you care about and receive emails when their occurrence on Twitter changes. (For example, want to know when something interesting has happened in the world of Machine Learning? Huginn will watch the term "machine learning" on Twitter and tell you when there is a large spike.) +* List terms that you care about and receive emails when their occurrence on Twitter changes. (For example, want to know when something interesting has happened in the world of Machine Learning? Huginn will watch the term "machine learning" on Twitter and tell you when there is a spike in discussion.) * Watch for air travel or shopping deals * Follow your project names on Twitter and get updates when people mention them * Scrape websites and receive emails when they change * Connect to Adioso, HipChat, Basecamp, Growl, FTP, IMAP, Jabber, JIRA, MQTT, nextbus, Pushbullet, Pushover, RSS, Bash, Slack, StubHub, translation APIs, Twilio, Twitter, Wunderground, and Weibo, to name a few. -* Compose digest emails about things you care about to be sent at specific times of the day +* Send digest emails with things that you care about at specific times during the day * Track counts of high frequency events and send an SMS within moments when they spike, such as the term "san francisco emergency" * Send and receive WebHooks -* Run arbitrary JavaScript Agents on the server +* Run custom JavaScript or CoffeeScript functions * Track your location over time * Create Amazon Mechanical Turk workflows as the inputs, or outputs, of agents (the Amazon Turk Agent is called the "HumanTaskAgent"). For example: "Once a day, ask 5 people for a funny cat photo; send the results to 5 more people to be rated; send the top-rated photo to 5 people for a funny caption; send to 5 final people to rate for funniest caption; finally, post the best captioned photo on my blog." [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/cantino/huginn?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) -Follow [@tectonic](https://twitter.com/tectonic) for updates as Huginn evolves, and join us in our [Gitter room](https://gitter.im/cantino/huginn) to discuss the project. +Join us in our [Gitter room](https://gitter.im/cantino/huginn) to discuss the project and follow [@tectonic](https://twitter.com/tectonic) for updates as Huginn evolves. -### We need your help! +### Join us! -Want to help with Huginn? All contributions are encouraged! You could make UI improvements, [add new Agents](https://github.com/cantino/huginn/wiki/Creating-a-new-agent), write documentation and tutorials, or try tackling [issues tagged with #help-wanted](https://github.com/cantino/huginn/issues?direction=desc&labels=help-wanted&page=1&sort=created&state=open). +Want to help with Huginn? All contributions are encouraged! You could make UI improvements, [add new Agents](https://github.com/cantino/huginn/wiki/Creating-a-new-agent), write [documentation and tutorials](https://github.com/cantino/huginn/wiki), or try tackling [issues tagged with #help-wanted](https://github.com/cantino/huginn/issues?direction=desc&labels=help-wanted&page=1&sort=created&state=open). Please fork, add specs, and send pull requests! -Really want an issue fixed/feature implemented? Or maybe you just want to solve some community issues and earn some extra coffee money? Then you should take a look at the [current bounties on Bountysource](https://www.bountysource.com/trackers/282580-huginn). +Really want a fix or feature? Want to solve some community issues and earn some extra coffee money? Take a look at the [current bounties on Bountysource](https://www.bountysource.com/trackers/282580-huginn). -Have an awesome an idea but not feeling quite up to contributing yet? Head over to our [Official 'suggest an agent' thread ](https://github.com/cantino/huginn/issues/353) and tell us about your cool idea! +Have an awesome idea but not feeling quite up to contributing yet? Head over to our [Official 'suggest an agent' thread ](https://github.com/cantino/huginn/issues/353) and tell us! ## Examples @@ -102,15 +102,5 @@ We assume your deployment will run over SSL. This is a very good idea! However, Huginn is provided under the MIT License. -## Community -Huginn has its own IRC channel on freenode: #huginn. -Some of us are hanging out there, come and say hello. - -## Contribution - -Huginn is a work in progress and is just getting started. Please get involved! You can [add new Agents](https://github.com/cantino/huginn/wiki/Creating-a-new-agent), expand the [Wiki](https://github.com/cantino/huginn/wiki), or help us simplify and strengthen the Agent API or core application. - -Please fork, add specs, and send pull requests! - [![Build Status](https://travis-ci.org/cantino/huginn.png)](https://travis-ci.org/cantino/huginn) [![Coverage Status](https://coveralls.io/repos/cantino/huginn/badge.png)](https://coveralls.io/r/cantino/huginn) [![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/cantino/huginn/trend.png)](https://bitdeli.com/free "Bitdeli Badge") [![Dependency Status](https://gemnasium.com/cantino/huginn.svg)](https://gemnasium.com/cantino/huginn) [![Bountysource](https://www.bountysource.com/badge/tracker?tracker_id=282580)](https://www.bountysource.com/trackers/282580-huginn?utm_source=282580&utm_medium=shield&utm_campaign=TRACKER_BADGE) diff --git a/app.json b/app.json index 790bed82bc..6525572f99 100644 --- a/app.json +++ b/app.json @@ -4,7 +4,7 @@ "website": "https://github.com/cantino/huginn", "repository": "https://github.com/cantino/huginn", "env": { - "BUILDPACK_URL": "https://github.com/ddollar/heroku-buildpack-multi.git", + "BUILDPACK_URL": "https://github.com/heroku/heroku-buildpack-multi.git", "APP_SECRET_TOKEN": { "generator": "secret" }, @@ -19,5 +19,6 @@ "scripts": { "postdeploy": "bundle exec rake db:migrate" }, + "addons": ["heroku-postgresql"], "success_url": "/users/sign_up" } diff --git a/app/assets/javascripts/pages/agent-edit-page.js.coffee b/app/assets/javascripts/pages/agent-edit-page.js.coffee index 62dcb77ebc..62642de3cd 100644 --- a/app/assets/javascripts/pages/agent-edit-page.js.coffee +++ b/app/assets/javascripts/pages/agent-edit-page.js.coffee @@ -4,6 +4,14 @@ class @AgentEditPage @showCorrectRegionsOnStartup() $("form.agent-form").on "submit", => @updateFromEditors() + $("#agent_name").each -> + # Select the number suffix if this is a cloned agent. + if matches = this.value.match(/ \(\d+\)$/) + this.focus() + if this.selectionStart? + this.selectionStart = matches.index + this.selectionEnd = this.value.length + # The type selector is only available on the new agent form. if $("#agent_type").length $("#agent_type").on "change", => @handleTypeChange(false) diff --git a/app/assets/javascripts/pages/agent-show-page.js.coffee b/app/assets/javascripts/pages/agent-show-page.js.coffee index 3b9ef02ff9..bfa6c2be17 100644 --- a/app/assets/javascripts/pages/agent-show-page.js.coffee +++ b/app/assets/javascripts/pages/agent-show-page.js.coffee @@ -2,6 +2,7 @@ class @AgentShowPage constructor: -> $(".agent-show #show-tabs a[href='#logs'], #logs .refresh").on "click", @fetchLogs $(".agent-show #logs .clear").on "click", @clearLogs + $(".agent-show #memory .clear").on "click", @clearMemory # Trigger tabs when navigated to. if tab = window.location.href.match(/tab=(\w+)\b/i)?[1] @@ -39,6 +40,20 @@ class @AgentShowPage $("#logs .spinner").stop(true, true).fadeOut -> $("#logs .refresh, #logs .clear").show() + clearMemory: (e) -> + if confirm("Are you sure you want to clear memory of this Agent?") + agentId = $(e.target).closest("[data-agent-id]").data("agent-id") + e.preventDefault() + $("#memory .spinner").css(display: 'inline-block') + $("#memory .clear").hide() + $.post "/agents/#{agentId}/memory", { "_method": "DELETE" } + .done -> + $("#memory .spinner").fadeOut -> + $("#memory + .memory").text "{\n}\n" + .fail -> + $("#memory .spinner").fadeOut -> + $("#memory .clear").css(display: 'inline-block') + $ -> Utils.registerPage(AgentShowPage, forPathsMatching: /^agents\/\d+/) diff --git a/app/assets/stylesheets/application.css.scss.erb b/app/assets/stylesheets/application.css.scss.erb index 9119d9235b..d00936a6cd 100644 --- a/app/assets/stylesheets/application.css.scss.erb +++ b/app/assets/stylesheets/application.css.scss.erb @@ -61,7 +61,7 @@ img.odin { display: none; } -img.spinner { +.spinner { display: none; vertical-align: bottom; } @@ -172,6 +172,13 @@ span.not-applicable:after { font-weight: bold; } +// Memory + +#memory .action-icon { + display: inline-block; + cursor: pointer; +} + // Credentials and Ace Editor #ace-credential-value { @@ -251,8 +258,8 @@ h2 .scenario, a span.label.scenario { width: 200px; } -$services: twitter 37signals github tumblr dropbox; -$service-colors: #55acee #8fc857 #444444 #2c4762 #007EE5; +$services: twitter 37signals github tumblr dropbox wunderlist; +$service-colors: #55acee #8fc857 #444444 #2c4762 #007EE5 #ED5F27; @mixin services { @each $service in $services { diff --git a/app/concerns/liquid_droppable.rb b/app/concerns/liquid_droppable.rb index f6d210ede3..bf8adb5fb1 100644 --- a/app/concerns/liquid_droppable.rb +++ b/app/concerns/liquid_droppable.rb @@ -21,7 +21,12 @@ def each end included do - const_set :Drop, Kernel.const_set("#{name}Drop", Class.new(Drop)) unless const_defined?("#{name}Drop") + const_set :Drop, + if Kernel.const_defined?(drop_name = "#{name}Drop") + Kernel.const_get(drop_name) + else + Kernel.const_set(drop_name, Class.new(Drop)) + end end def to_liquid diff --git a/app/controllers/agents_controller.rb b/app/controllers/agents_controller.rb index a1cf386b91..d656b371bc 100644 --- a/app/controllers/agents_controller.rb +++ b/app/controllers/agents_controller.rb @@ -112,6 +112,16 @@ def propagate end end + def destroy_memory + @agent = current_user.agents.find(params[:id]) + @agent.update!(memory: {}) + + respond_to do |format| + format.html { redirect_back "Memory erased for '#{@agent.name}'" } + format.json { head :ok } + end + end + def show @agent = current_user.agents.find(params[:id]) diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index d400d049f8..77fcbc071a 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -57,6 +57,8 @@ def omniauth_provider_icon(provider) case provider.to_sym when :twitter, :tumblr, :github, :dropbox icon_tag("fa-#{provider}") + when :wunderlist + icon_tag("fa-list") else icon_tag("fa-lock") end diff --git a/app/jobs/agent_check_job.rb b/app/jobs/agent_check_job.rb new file mode 100644 index 0000000000..0603bdd84e --- /dev/null +++ b/app/jobs/agent_check_job.rb @@ -0,0 +1,15 @@ +class AgentCheckJob < ActiveJob::Base + # Given an Agent id, load the Agent, call #check on it, and then save it with an updated `last_check_at` timestamp. + def perform(agent_id) + agent = Agent.find(agent_id) + begin + return if agent.unavailable? + agent.check + agent.last_check_at = Time.now + agent.save! + rescue => e + agent.error "Exception during check. #{e.message}: #{e.backtrace.join("\n")}" + raise + end + end +end \ No newline at end of file diff --git a/app/jobs/agent_receive_job.rb b/app/jobs/agent_receive_job.rb new file mode 100644 index 0000000000..385c3b8c76 --- /dev/null +++ b/app/jobs/agent_receive_job.rb @@ -0,0 +1,16 @@ +class AgentReceiveJob < ActiveJob::Base + # Given an Agent id and an array of Event ids, load the Agent, call #receive on it with the Event objects, and then + # save it with an updated `last_receive_at` timestamp. + def perform(agent_id, event_ids) + agent = Agent.find(agent_id) + begin + return if agent.unavailable? + agent.receive(Event.where(:id => event_ids).order(:id)) + agent.last_receive_at = Time.now + agent.save! + rescue => e + agent.error "Exception during receive. #{e.message}: #{e.backtrace.join("\n")}" + raise + end + end +end \ No newline at end of file diff --git a/app/models/agent.rb b/app/models/agent.rb index 2d1314506f..c0aa3d8ab9 100644 --- a/app/models/agent.rb +++ b/app/models/agent.rb @@ -387,24 +387,11 @@ def receive!(options={}) end end - # Given an Agent id and an array of Event ids, load the Agent, call #receive on it with the Event objects, and then - # save it with an updated `last_receive_at` timestamp. - # - # This method is tagged with `handle_asynchronously` and will be delayed and run with delayed_job. It accepts Agent - # and Event ids instead of a literal ActiveRecord models because it is preferable to serialize delayed_jobs with ids. + # This method will enqueue an AgentReceiveJob job. It accepts Agent and Event ids instead of a literal ActiveRecord + # models because it is preferable to serialize jobs with ids. def async_receive(agent_id, event_ids) - agent = Agent.find(agent_id) - begin - return if agent.unavailable? - agent.receive(Event.where(:id => event_ids)) - agent.last_receive_at = Time.now - agent.save! - rescue => e - agent.error "Exception during receive. #{e.message}: #{e.backtrace.join("\n")}" - raise - end + AgentReceiveJob.perform_later(agent_id, event_ids) end - handle_asynchronously :async_receive # Given a schedule name, run `check` via `bulk_check` on all Agents with that schedule. # This is called by bin/schedule.rb for each schedule in `SCHEDULES`. @@ -425,24 +412,11 @@ def bulk_check(schedule) end end - # Given an Agent id, load the Agent, call #check on it, and then save it with an updated `last_check_at` timestamp. - # - # This method is tagged with `handle_asynchronously` and will be delayed and run with delayed_job. It accepts an Agent - # id instead of a literal Agent because it is preferable to serialize delayed_jobs with ids, instead of with the full - # Agents. + # This method will enqueue an AgentCheckJob job. It accepts an Agent id instead of a literal Agent because it is + # preferable to serialize job with ids, instead of with the full Agents. def async_check(agent_id) - agent = Agent.find(agent_id) - begin - return if agent.unavailable? - agent.check - agent.last_check_at = Time.now - agent.save! - rescue => e - agent.error "Exception during check. #{e.message}: #{e.backtrace.join("\n")}" - raise - end + AgentCheckJob.perform_later(agent_id) end - handle_asynchronously :async_check end end diff --git a/app/models/agents/email_agent.rb b/app/models/agents/email_agent.rb index d3397dd739..be1068189f 100644 --- a/app/models/agents/email_agent.rb +++ b/app/models/agents/email_agent.rb @@ -35,7 +35,7 @@ def receive(incoming_events) incoming_events.each do |event| log "Sending digest mail to #{user.email} with event #{event.id}" recipients(event.payload).each do |recipient| - SystemMailer.delay.send_message(:to => recipient, :subject => interpolated(event)['subject'], :headline => interpolated(event)['headline'], :body => interpolated(event)['body'], :groups => [present(event.payload)]) + SystemMailer.send_message(:to => recipient, :subject => interpolated(event)['subject'], :headline => interpolated(event)['headline'], :body => interpolated(event)['body'], :groups => [present(event.payload)]).deliver_later end end end diff --git a/app/models/agents/email_digest_agent.rb b/app/models/agents/email_digest_agent.rb index 3ee5e76b3d..1a89b5f1f8 100644 --- a/app/models/agents/email_digest_agent.rb +++ b/app/models/agents/email_digest_agent.rb @@ -42,7 +42,7 @@ def check groups = self.memory['queue'].map { |payload| present(payload) } log "Sending digest mail to #{user.email} with events [#{ids}]" recipients.each do |recipient| - SystemMailer.delay.send_message(:to => recipient, :subject => interpolated['subject'], :headline => interpolated['headline'], :groups => groups) + SystemMailer.send_message(:to => recipient, :subject => interpolated['subject'], :headline => interpolated['headline'], :groups => groups).deliver_later end self.memory['queue'] = [] self.memory['events'] = [] diff --git a/app/models/agents/google_calendar_publish_agent.rb b/app/models/agents/google_calendar_publish_agent.rb index 18a35d5ee3..5ad4e3333a 100644 --- a/app/models/agents/google_calendar_publish_agent.rb +++ b/app/models/agents/google_calendar_publish_agent.rb @@ -18,7 +18,7 @@ class GoogleCalendarPublishAgent < Agent 2. New project -> Huginn 3. APIs & Auth -> Enable google calendar 4. Credentials -> Create new Client ID -> Service Account - 5. Persist the generated private key to a path, ie: `/home/hugin/a822ccdefac89fac6330f95039c492dfa3ce6843.p12` + 5. Persist the generated private key to a path, ie: `/home/huginn/a822ccdefac89fac6330f95039c492dfa3ce6843.p12` 6. Grant access via google calendar UI to the service account email address for each calendar you wish to manage. For a whole google apps domain, you can [delegate authority](https://developers.google.com/+/domains/authentication/delegation) diff --git a/app/models/agents/java_script_agent.rb b/app/models/agents/java_script_agent.rb index c330e7aa0b..70dc082a07 100644 --- a/app/models/agents/java_script_agent.rb +++ b/app/models/agents/java_script_agent.rb @@ -25,6 +25,8 @@ class JavaScriptAgent < Agent * `this.options(key)` * `this.log(message)` * `this.error(message)` + * `this.escapeHtml(htmlToEscape)` + * `this.unescapeHtml(htmlToUnescape)` MD form_configurable :language, type: :array, values: %w[JavaScript CoffeeScript] @@ -116,6 +118,8 @@ def execute_js(js_function, incoming_events = []) memory.to_json end end + context["escapeHtml"] = lambda { |a, x| CGI.escapeHTML(x) } + context["unescapeHtml"] = lambda { |a, x| CGI.unescapeHTML(x) } if (options['language'] || '').downcase == 'coffeescript' context.eval(CoffeeScript.compile code) @@ -176,6 +180,14 @@ def setup_javascript doError(message); } + Agent.escapeHtml = function(html) { + return escapeHtml(html); + } + + Agent.unescapeHtml = function(html) { + return unescapeHtml(html); + } + Agent.check = function(){}; Agent.receive = function(){}; JS diff --git a/app/models/agents/webhook_agent.rb b/app/models/agents/webhook_agent.rb index fc227ef9a6..92f38ebad7 100644 --- a/app/models/agents/webhook_agent.rb +++ b/app/models/agents/webhook_agent.rb @@ -12,21 +12,20 @@ class WebhookAgent < Agent https://#{ENV['DOMAIN']}/users/#{user.id}/web_requests/#{id || ''}/:secret ``` where `:secret` is specified in your options. - The - Options: * `secret` - A token that the host will provide for authentication. * `expected_receive_period_in_days` - How often you expect to receive events this way. Used to determine if the agent is working. * `payload_path` - JSONPath of the attribute in the POST body to be - used as the Event payload. + used as the Event payload. If `payload_path` points to an array, + Events will be created for each element. MD end event_description do <<-MD - The event payload is base on the value of the `payload_path` option, + The event payload is based on the value of the `payload_path` option, which is set to `#{interpolated['payload_path']}`. MD end @@ -34,7 +33,8 @@ class WebhookAgent < Agent def default_options { "secret" => "supersecretstring", "expected_receive_period_in_days" => 1, - "payload_path" => "payload"} + "payload_path" => "some_key" + } end def receive_web_request(params, method, format) @@ -42,7 +42,9 @@ def receive_web_request(params, method, format) return ["Please use POST requests only", 401] unless method == "post" return ["Not Authorized", 401] unless secret == interpolated['secret'] - create_event(:payload => payload_for(params)) + [payload_for(params)].flatten.each do |payload| + create_event(payload: payload) + end ['Event Created', 201] end diff --git a/app/models/agents/website_agent.rb b/app/models/agents/website_agent.rb index f86c9c7c6c..aad791ee27 100644 --- a/app/models/agents/website_agent.rb +++ b/app/models/agents/website_agent.rb @@ -78,6 +78,8 @@ class WebsiteAgent < Agent Set `disable_ssl_verification` to `true` to disable ssl verification. + Set `unzip` to `gzip` to inflate the resource using gzip. + The WebsiteAgent can also scrape based on incoming events. It will scrape the url contained in the `url` key of the incoming event payload. If you specify `merge` as the mode, it will retain the old payload and update it with the new values. In Liquid templating, the following variable is available: @@ -86,7 +88,7 @@ class WebsiteAgent < Agent * `status`: HTTP status as integer. (Almost always 200) - * `headers`: Reponse headers; for example, `{{ _response_.headers.Content-Type }}` expands to the value of the Content-Type header. Keys are insentitive to cases and -/_. + * `headers`: Response headers; for example, `{{ _response_.headers.Content-Type }}` expands to the value of the Content-Type header. Keys are insensitive to cases and -/_. MD event_description do @@ -174,6 +176,9 @@ def check_url(url, payload = {}) if (encoding = interpolated['force_encoding']).present? body = body.encode(Encoding::UTF_8, encoding) end + if interpolated['unzip'] == "gzip" + body = ActiveSupport::Gzip.decompress(body) + end doc = parse(body) if extract_full_json? @@ -240,14 +245,12 @@ def store_payload!(old_events, result) case interpolated['mode'].presence when 'on_change' result_json = result.to_json - old_events.each do |old_event| - if old_event.payload.to_json == result_json - old_event.expires_at = new_event_expiration_date - old_event.save! - return false - end + if found = old_events.find { |event| event.payload.to_json == result_json } + found.update!(expires_at: new_event_expiration_date) + false + else + true end - true when 'all', 'merge', '' true else diff --git a/app/models/agents/wunderlist_agent.rb b/app/models/agents/wunderlist_agent.rb new file mode 100644 index 0000000000..cc5d72b4e3 --- /dev/null +++ b/app/models/agents/wunderlist_agent.rb @@ -0,0 +1,78 @@ +module Agents + class WunderlistAgent < Agent + include FormConfigurable + include Oauthable + valid_oauth_providers :wunderlist + + cannot_be_scheduled! + + gem_dependency_check { Devise.omniauth_providers.include?(:wunderlist) } + + description <<-MD + #{'## Include the `omniauth-wunderlist` gem in your `Gemfile` and set `WUNDERLIST_OAUTH_KEY` and `WUNDERLIST_OAUTH_SECRET` in your environment to use this Agent' if dependencies_missing?} + + The WunderlistAgent creates new new tasks based on the incoming event. + + To be able to use this Agent you need to authenticate with Wunderlist in the [Services](/services) section first. + + MD + + def default_options + { + 'list_id' => '', + 'title' => '{{title}}' + } + end + + form_configurable :list_id, roles: :completable + form_configurable :title + + def complete_list_id + response = request_guard do + HTTParty.get lists_url, request_options + end + response.map { |p| {text: "#{p['title']} (#{p['id']})", id: p['id']}} + end + + def validate_options + errors.add(:base, "you need to specify the list you want to add tasks to") unless options['list_id'].present? + errors.add(:base, "you need to specify the title of the task to create") unless options['title'].present? + end + + def working? + !recent_error_logs? + end + + def receive(incoming_events) + incoming_events.each do |event| + mo = interpolated(event) + title = mo[:title][0..244] + log("Creating new task '#{title}' on list #{mo[:list_id]}", inbound_event: event) + request_guard do + HTTParty.post tasks_url, request_options.merge(body: {title: title, list_id: mo[:list_id].to_i}.to_json) + end + end + end + private + def request_guard(&blk) + response = yield + error("Error during http request: #{response.body}") if response.code > 400 + response + end + + def lists_url + "https://a.wunderlist.com/api/v1/lists" + end + + def tasks_url + "https://a.wunderlist.com/api/v1/tasks" + end + + def request_options + {:headers => {'Content-Type' => 'application/json', + 'User-Agent' => 'Huginn (https://github.com/cantino/huginn)', + 'X-Access-Token' => service.token, + 'X-Client-ID' => ENV["WUNDERLIST_OAUTH_KEY"] }} + end + end +end \ No newline at end of file diff --git a/app/views/agents/show.html.erb b/app/views/agents/show.html.erb index 92c35b78eb..744d517dfa 100644 --- a/app/views/agents/show.html.erb +++ b/app/views/agents/show.html.erb @@ -162,9 +162,13 @@ 
<%= Utils.pretty_jsonify @agent.options || {} %>

-

+

Memory: - 

<%= Utils.pretty_jsonify @agent.memory || {} %>
+ <% if @agent.memory.present? %> + + + <% end %> + 
<%= Utils.pretty_jsonify @agent.memory || {} %>

diff --git a/bin/schedule.rb b/bin/schedule.rb index 1b077ad276..9546c3ec53 100755 --- a/bin/schedule.rb +++ b/bin/schedule.rb @@ -11,5 +11,5 @@ exit 1 end -scheduler = HuginnScheduler.new(frequency: ENV['SCHEDULER_FREQUENCY']) +scheduler = HuginnScheduler.new(frequency: ENV['SCHEDULER_FREQUENCY'].presence || 0.3) scheduler.run! diff --git a/bin/setup_heroku b/bin/setup_heroku index 2d16b89a59..afd7bfb1ec 100755 --- a/bin/setup_heroku +++ b/bin/setup_heroku @@ -81,10 +81,16 @@ unless $config['DOMAIN'] first_time = true end -set_value 'BUILDPACK_URL', "https://github.com/ddollar/heroku-buildpack-multi.git" +set_value 'BUILDPACK_URL', "https://github.com/heroku/heroku-buildpack-multi.git" set_value 'PROCFILE_PATH', "deployment/heroku/Procfile.heroku", force: false set_value 'ON_HEROKU', "true" +unless $config['DATABASE_URL'] + puts "Setting up the postgres addon" + puts capture("heroku addons:add heroku-postgresql") + puts +end + unless $config['SMTP_DOMAIN'] && $config['SMTP_USER_NAME'] && $config['SMTP_PASSWORD'] && $config['SMTP_SERVER'] && $config['EMAIL_FROM_ADDRESS'] puts "Okay, let's setup outgoing email settings. The simplest solution is to use the free sendgrid Heroku addon." puts "If you'd like to use your own server, or your Gmail account, please see .env.example and set" diff --git a/bin/threaded.rb b/bin/threaded.rb index 7cd503b9a3..56c99c46f7 100644 --- a/bin/threaded.rb +++ b/bin/threaded.rb @@ -1,5 +1,6 @@ require 'thread' require 'huginn_scheduler' +require 'twitter_stream' STDOUT.sync = true STDERR.sync = true @@ -33,7 +34,7 @@ def safely(&block) threads << Thread.new do safely do - @scheduler = HuginnScheduler.new(frequency: ENV['SCHEDULER_FREQUENCY']) + @scheduler = HuginnScheduler.new(frequency: ENV['SCHEDULER_FREQUENCY'].presence || 0.3) @scheduler.run! puts "Scheduler stopped ..." end diff --git a/config/application.rb b/config/application.rb index 0fed9f3374..f4156e403a 100644 --- a/config/application.rb +++ b/config/application.rb @@ -13,7 +13,7 @@ class Application < Rails::Application # -- all .rb files in that directory are automatically loaded. # Custom directories with classes and modules you want to be autoloadable. - config.autoload_paths += %W(#{config.root}/lib #{config.root}/app/presenters) + config.autoload_paths += %W(#{config.root}/lib #{config.root}/app/presenters #{config.root}/app/jobs) # Activate observers that should always be running. # config.active_record.observers = :cacher, :garbage_collector, :forum_observer @@ -52,5 +52,7 @@ class Application < Rails::Application # Do not swallow errors in after_commit/after_rollback callbacks. config.active_record.raise_in_transactional_callbacks = true + + config.active_job.queue_adapter = :delayed_job end end diff --git a/config/database.yml b/config/database.yml index 41e0072e65..996366dc2e 100644 --- a/config/database.yml +++ b/config/database.yml @@ -3,7 +3,7 @@ development: encoding: <%= ENV['DATABASE_ENCODING'].presence || "utf8" %> reconnect: <%= ENV['DATABASE_RECONNECT'].presence || "true" %> database: <%= ENV['DATABASE_NAME'].presence || "huginn_development" %> - pool: <%= ENV['DATABASE_POOL'].presence || "5" %> + pool: <%= ENV['DATABASE_POOL'].presence || "10" %> username: <%= ENV['DATABASE_USERNAME'].presence || "root" %> password: <%= ENV['DATABASE_PASSWORD'] || "" %> host: <%= ENV['DATABASE_HOST'] || "" %> @@ -29,7 +29,7 @@ production: encoding: <%= ENV['DATABASE_ENCODING'].presence || "utf8" %> reconnect: <%= ENV['DATABASE_RECONNECT'].presence || "true" %> database: <%= ENV['DATABASE_NAME'].presence || "huginn_production" %> - pool: <%= ENV['DATABASE_POOL'].presence || "5" %> + pool: <%= ENV['DATABASE_POOL'].presence || "10" %> username: <%= ENV['DATABASE_USERNAME'].presence || "root" %> password: <%= ENV['DATABASE_PASSWORD'].presence || "password" %> host: <%= ENV['DATABASE_HOST'] || "" %> diff --git a/config/initializers/ar_mysql_column_charset.rb b/config/initializers/ar_mysql_column_charset.rb index cade648ebf..aea6a68823 100644 --- a/config/initializers/ar_mysql_column_charset.rb +++ b/config/initializers/ar_mysql_column_charset.rb @@ -1,3 +1 @@ -ActiveSupport.on_load :active_record do - require 'ar_mysql_column_charset' -end +require 'ar_mysql_column_charset' diff --git a/config/initializers/devise.rb b/config/initializers/devise.rb index f806ef2385..6ae99abd33 100644 --- a/config/initializers/devise.rb +++ b/config/initializers/devise.rb @@ -263,6 +263,12 @@ config.omniauth :dropbox, key, secret end + if defined?(OmniAuth::Strategies::Wunderlist) && + (key = ENV["WUNDERLIST_OAUTH_KEY"]).present? && + (secret = ENV["WUNDERLIST_OAUTH_SECRET"]).present? + config.omniauth :wunderlist, key, secret + end + # ==> Warden configuration # If you want to use other strategies, that are not supported by Devise, or # change the failure app, you can configure them inside the config.warden block. diff --git a/config/locales/devise.en.yml b/config/locales/devise.en.yml index dd10d119e0..8af9503fc9 100644 --- a/config/locales/devise.en.yml +++ b/config/locales/devise.en.yml @@ -32,6 +32,7 @@ en: github: "GitHub" 37signals: "37Signals (Basecamp)" dropbox: "Dropbox" + wunderlist: 'Wunderlist' passwords: no_token: "You can't access this page without coming from a password reset email. If you do come from a password reset email, please make sure you used the full URL provided." send_instructions: "You will receive an email with instructions on how to reset your password in a few minutes." diff --git a/config/routes.rb b/config/routes.rb index 768a208523..c579b2b1f2 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -6,6 +6,7 @@ post :handle_details_post put :leave_scenario delete :remove_events + delete :memory, action: :destroy_memory end collection do diff --git a/docker/scripts/init b/docker/scripts/init index 934f6ffd24..934e9a3de7 100755 --- a/docker/scripts/init +++ b/docker/scripts/init @@ -150,7 +150,7 @@ source /app/.env # Fixup the Procfile and prepare the PORT [ -n "\${DO_NOT_RUN_JOBS}" ] && perl -pi -e 's/^jobs:/#jobs:/' /app/Procfile -perl -pi -e 's/rails server\$/rails server -p \\\$PORT/' /app/Procfile +perl -pi -e 's/rails server\$/rails server -b 0.0.0.0 -p \\\$PORT/' /app/Procfile export PORT # Start huginn diff --git a/lib/ar_mysql_column_charset.rb b/lib/ar_mysql_column_charset.rb index b93710461d..64c80648cd 100644 --- a/lib/ar_mysql_column_charset.rb +++ b/lib/ar_mysql_column_charset.rb @@ -1,121 +1,16 @@ -require 'active_record' - # Module#prepend support for Ruby 1.9 require 'prepend' unless Module.method_defined?(:prepend) -module ActiveRecord::ConnectionAdapters - class ColumnDefinition - module CharsetSupport - attr_accessor :charset, :collation - end - - prepend CharsetSupport - end - - class TableDefinition - module CharsetSupport - def new_column_definition(name, type, options) - column = super - column.charset = options[:charset] - column.collation = options[:collation] - column - end - end - - prepend CharsetSupport - end - - class AbstractMysqlAdapter - module CharsetSupport - def prepare_column_options(column, types) - spec = super - spec[:charset] = column.charset.inspect if column.charset && column.charset != charset - spec[:collation] = column.collation.inspect if column.collation && column.collation != collation - spec - end +require 'active_support' - def migration_keys - super + [:charset, :collation] - end - - def utf8mb4_supported? - if @utf8mb4_supported.nil? - @utf8mb4_supported = !select("show character set like 'utf8mb4'").empty? - else - @utf8mb4_supported +ActiveSupport.on_load :active_record do + class << ActiveRecord::Base + def establish_connection(spec = nil) + super.tap { |ret| + if /mysql/i === connection.adapter_name + require 'ar_mysql_column_charset/main' end - end - - def charset_collation(charset, collation) - [charset, collation].map { |name| - case name - when nil - nil - when /\A(utf8mb4(_\w*)?)\z/ - if utf8mb4_supported? - $1 - else - "utf8#{$2}" - end - else - name.to_s - end - } - end - - def create_database(name, options = {}) - # utf8mb4 is used in column definitions; use utf8 for - # databases. - [:charset, :collation].each { |key| - case options[key] - when /\A(utf8mb4(_\w*)?)\z/ - options = options.merge(key => "utf8#{$2}") - end - } - super(name, options) - end - end - - prepend CharsetSupport - - class SchemaCreation - module CharsetSupport - def column_options(o) - column_options = super - column_options[:charset] = o.charset unless o.charset.nil? - column_options[:collation] = o.collation unless o.collation.nil? - column_options - end - - def add_column_options!(sql, options) - charset, collation = @conn.charset_collation(options[:charset], options[:collation]) - - if charset - sql << " CHARACTER SET #{charset}" - end - - if collation - sql << " COLLATE #{collation}" - end - - super - end - end - - prepend CharsetSupport - end - - class Column - module CharsetSupport - attr_reader :charset - - def initialize(*args) - super - @charset = @collation[/\A[^_]+/] unless @collation.nil? - end - end - - prepend CharsetSupport + } end end end diff --git a/lib/ar_mysql_column_charset/main.rb b/lib/ar_mysql_column_charset/main.rb new file mode 100644 index 0000000000..4b3041f326 --- /dev/null +++ b/lib/ar_mysql_column_charset/main.rb @@ -0,0 +1,118 @@ +raise "Do not directly load this library." unless defined?(ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter) + +module ActiveRecord::ConnectionAdapters + class ColumnDefinition + module CharsetSupport + attr_accessor :charset, :collation + end + + prepend CharsetSupport + end + + class TableDefinition + module CharsetSupport + def new_column_definition(name, type, options) + column = super + column.charset = options[:charset] + column.collation = options[:collation] + column + end + end + + prepend CharsetSupport + end + + class AbstractMysqlAdapter + module CharsetSupport + def prepare_column_options(column, types) + spec = super + spec[:charset] = column.charset.inspect if column.charset && column.charset != charset + spec[:collation] = column.collation.inspect if column.collation && column.collation != collation + spec + end + + def migration_keys + super + [:charset, :collation] + end + + def utf8mb4_supported? + if @utf8mb4_supported.nil? + @utf8mb4_supported = !select("show character set like 'utf8mb4'").empty? + else + @utf8mb4_supported + end + end + + def charset_collation(charset, collation) + [charset, collation].map { |name| + case name + when nil + nil + when /\A(utf8mb4(_\w*)?)\z/ + if utf8mb4_supported? + $1 + else + "utf8#{$2}" + end + else + name.to_s + end + } + end + + def create_database(name, options = {}) + # utf8mb4 is used in column definitions; use utf8 for + # databases. + [:charset, :collation].each { |key| + case options[key] + when /\A(utf8mb4(_\w*)?)\z/ + options = options.merge(key => "utf8#{$2}") + end + } + super(name, options) + end + end + + prepend CharsetSupport + + class SchemaCreation + module CharsetSupport + def column_options(o) + column_options = super + column_options[:charset] = o.charset unless o.charset.nil? + column_options[:collation] = o.collation unless o.collation.nil? + column_options + end + + def add_column_options!(sql, options) + charset, collation = @conn.charset_collation(options[:charset], options[:collation]) + + if charset + sql << " CHARACTER SET #{charset}" + end + + if collation + sql << " COLLATE #{collation}" + end + + super + end + end + + prepend CharsetSupport + end + + class Column + module CharsetSupport + attr_reader :charset + + def initialize(*args) + super + @charset = @collation[/\A[^_]+/] unless @collation.nil? + end + end + + prepend CharsetSupport + end + end +end diff --git a/lib/tasks/ar_mysql_column_charset.rake b/lib/tasks/ar_mysql_column_charset.rake new file mode 100644 index 0000000000..aea6a68823 --- /dev/null +++ b/lib/tasks/ar_mysql_column_charset.rake @@ -0,0 +1 @@ +require 'ar_mysql_column_charset' diff --git a/spec/controllers/agents_controller_spec.rb b/spec/controllers/agents_controller_spec.rb index fa7106286e..b7335b15dc 100644 --- a/spec/controllers/agents_controller_spec.rb +++ b/spec/controllers/agents_controller_spec.rb @@ -374,4 +374,24 @@ def valid_attributes(options = {}) } end end + + describe "DELETE memory" do + it "clears memory of the agent" do + agent = agents(:bob_website_agent) + agent.update!(memory: { "test" => 42 }) + sign_in users(:bob) + delete :destroy_memory, id: agent.to_param + expect(agent.reload.memory).to eq({}) + end + + it "does not clear memory of an agent not owned by the current user" do + agent = agents(:jane_website_agent) + agent.update!(memory: { "test" => 42 }) + sign_in users(:bob) + expect { + delete :destroy_memory, id: agent.to_param + }.to raise_error(ActiveRecord::RecordNotFound) + expect(agent.reload.memory).to eq({ "test" => 42}) + end + end end diff --git a/spec/env.test b/spec/env.test index dfc9fcc819..4f567ded96 100644 --- a/spec/env.test +++ b/spec/env.test @@ -7,4 +7,5 @@ THIRTY_SEVEN_SIGNALS_OAUTH_KEY=TESTKEY THIRTY_SEVEN_SIGNALS_OAUTH_SECRET=TESTSECRET DROPBOX_OAUTH_KEY=dropboxoauthkey DROPBOX_OAUTH_SECRET=dropboxoauthsecret +WUNDERLIST_OAUTH_KEY=wunderoauthkey FAILED_JOBS_TO_KEEP=2 \ No newline at end of file diff --git a/spec/models/agents/java_script_agent_spec.rb b/spec/models/agents/java_script_agent_spec.rb index f9476997c4..d6b9380892 100644 --- a/spec/models/agents/java_script_agent_spec.rb +++ b/spec/models/agents/java_script_agent_spec.rb @@ -176,6 +176,20 @@ end end + describe "escaping and unescaping HTML" do + it "can escape and unescape html with this.escapeHtml and this.unescapeHtml in the javascript environment" do + @agent.options['code'] = 'Agent.check = function() { this.createEvent({ escaped: this.escapeHtml(\'test \"escaping\" \'), unescaped: this.unescapeHtml(\'test "unescaping" <characters>\')}); };' + @agent.save! + expect { + expect { + @agent.check + }.not_to change { AgentLog.count } + }.to change { Event.count}.by(1) + created_event = @agent.events.last + expect(created_event.payload).to eq({ 'escaped' => 'test "escaping" <characters>', 'unescaped' => 'test "unescaping" '}) + end + end + describe "getting incoming events" do it "can access incoming events in the JavaScript enviroment via this.incomingEvents" do event = Event.new diff --git a/spec/models/agents/webhook_agent_spec.rb b/spec/models/agents/webhook_agent_spec.rb index 39318d6a6a..25f2ec6615 100644 --- a/spec/models/agents/webhook_agent_spec.rb +++ b/spec/models/agents/webhook_agent_spec.rb @@ -3,27 +3,37 @@ describe Agents::WebhookAgent do let(:agent) do _agent = Agents::WebhookAgent.new(:name => 'webhook', - :options => { 'secret' => 'foobar', 'payload_path' => 'payload' }) + :options => { 'secret' => 'foobar', 'payload_path' => 'some_key' }) _agent.user = users(:bob) _agent.save! _agent end - let(:payload) { {'some' => 'info'} } + let(:payload) { {'people' => [{ 'name' => 'bob' }, { 'name' => 'jon' }] } } describe 'receive_web_request' do it 'should create event if secret matches' do out = nil expect { - out = agent.receive_web_request({ 'secret' => 'foobar', 'payload' => payload }, "post", "text/html") + out = agent.receive_web_request({ 'secret' => 'foobar', 'some_key' => payload }, "post", "text/html") }.to change { Event.count }.by(1) expect(out).to eq(['Event Created', 201]) expect(Event.last.payload).to eq(payload) end + it 'should be able to create multiple events when given an array' do + out = nil + agent.options['payload_path'] = 'some_key.people' + expect { + out = agent.receive_web_request({ 'secret' => 'foobar', 'some_key' => payload }, "post", "text/html") + }.to change { Event.count }.by(2) + expect(out).to eq(['Event Created', 201]) + expect(Event.last.payload).to eq({ 'name' => 'jon' }) + end + it 'should not create event if secrets dont match' do out = nil expect { - out = agent.receive_web_request({ 'secret' => 'bazbat', 'payload' => payload }, "post", "text/html") + out = agent.receive_web_request({ 'secret' => 'bazbat', 'some_key' => payload }, "post", "text/html") }.to change { Event.count }.by(0) expect(out).to eq(['Not Authorized', 401]) end @@ -31,7 +41,7 @@ it "should only accept POSTs" do out = nil expect { - out = agent.receive_web_request({ 'secret' => 'foobar', 'payload' => payload }, "get", "text/html") + out = agent.receive_web_request({ 'secret' => 'foobar', 'some_key' => payload }, "get", "text/html") }.to change { Event.count }.by(0) expect(out).to eq(['Please use POST requests only', 401]) end diff --git a/spec/models/agents/website_agent_spec.rb b/spec/models/agents/website_agent_spec.rb index e2e2af012d..7f45cb9095 100644 --- a/spec/models/agents/website_agent_spec.rb +++ b/spec/models/agents/website_agent_spec.rb @@ -152,6 +152,38 @@ end end + describe 'unzipping' do + it 'should unzip with unzip option' do + json = { + 'response' => { + 'version' => 2, + 'title' => "hello!" + } + } + zipped = ActiveSupport::Gzip.compress(json.to_json) + stub_request(:any, /gzip/).to_return(:body => zipped, :status => 200) + site = { + 'name' => "Some JSON Response", + 'expected_update_period_in_days' => "2", + 'type' => "json", + 'url' => "http://gzip.com", + 'mode' => 'on_change', + 'extract' => { + 'version' => { 'path' => 'response.version' }, + }, + 'unzip' => 'gzip', + } + checker = Agents::WebsiteAgent.new(:name => "Weather Site", :options => site) + checker.user = users(:bob) + checker.save! + + checker.check + event = Event.last + puts event.payload + expect(event.payload['version']).to eq(2) + end + end + describe 'encoding' do it 'should be forced with force_encoding option' do huginn = "\u{601d}\u{8003}" diff --git a/spec/models/agents/wunderlist_agent_spec.rb b/spec/models/agents/wunderlist_agent_spec.rb new file mode 100644 index 0000000000..4a57c3fccb --- /dev/null +++ b/spec/models/agents/wunderlist_agent_spec.rb @@ -0,0 +1,74 @@ +require 'spec_helper' +require 'models/concerns/oauthable' + +describe Agents::WunderlistAgent do + it_behaves_like Oauthable + + before(:each) do + + @valid_params = { + 'list_id' => '12345', + 'title' => '{{title}}: {{url}}', + } + + @checker = Agents::WunderlistAgent.new(:name => "somename", :options => @valid_params) + @checker.user = users(:jane) + @checker.service = services(:generic) + @checker.save! + + @event = Event.new + @event.agent = agents(:bob_weather_agent) + @event.payload = { title: 'hello', url: 'www.example.com'} + @event.save! + end + + describe "validating" do + before do + expect(@checker).to be_valid + end + + it "should require the title" do + @checker.options['title'] = nil + expect(@checker).not_to be_valid + end + + it "should require the list_id" do + @checker.options['list_id'] = nil + expect(@checker).not_to be_valid + end + end + + it "should generate the request_options" do + expect(@checker.send(:request_options)).to eq({:headers=>{"Content-Type"=>"application/json", "User-Agent"=>"Huginn (https://github.com/cantino/huginn)", "X-Access-Token"=>"1234token", "X-Client-ID"=>"wunderoauthkey"}}) + end + + describe "#complete_list_id" do + it "should return a array of hashes" do + stub_request(:get, 'https://a.wunderlist.com/api/v1/lists').to_return( + :body => JSON.dump([{title: 'test', id: 12345}]), + :headers => {"Content-Type" => "text/json"} + ) + expect(@checker.complete_list_id).to eq([{:text=>"test (12345)", :id=>12345}]) + end + end + + describe "#receive" do + it "send a message to the hipchat" do + stub_request(:post, 'https://a.wunderlist.com/api/v1/tasks').with { |request| request.body == 'abc'} + @checker.receive([@event]) + end + end + + describe "#working?" do + it "should be working with no entry in the error log" do + expect(@checker).to be_working + end + + it "should not be working with a recent entry in the error log" do + @checker.error("test") + @checker.reload + @checker.last_event_at = Time.now + expect(@checker).to_not be_working + end + end +end