diff --git a/.env.example b/.env.example index 743028655a..b588b13907 100644 --- a/.env.example +++ b/.env.example @@ -108,6 +108,9 @@ DROPBOX_OAUTH_SECRET= WUNDERLIST_OAUTH_KEY= WUNDERLIST_OAUTH_SECRET= +EVERNOTE_OAUTH_KEY= +EVERNOTE_OAUTH_SECRET= + ############################# # AWS and Mechanical Turk # ############################# diff --git a/Gemfile b/Gemfile index d9fdb0519f..aa0653a125 100644 --- a/Gemfile +++ b/Gemfile @@ -39,6 +39,10 @@ gem 'omniauth-dropbox' # UserLocationAgent gem 'haversine' +# EvernoteAgent +gem 'omniauth-evernote' +gem 'evernote_oauth' + # Optional Services. gem 'omniauth-37signals' # BasecampAgent gem 'omniauth-wunderlist', github: 'wunderlist/omniauth-wunderlist', ref: 'd0910d0396107b9302aa1bc50e74bb140990ccb8' diff --git a/Gemfile.lock b/Gemfile.lock index 8a9e0d5ec6..7a3f836a60 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -179,6 +179,10 @@ GEM ethon (0.7.1) ffi (>= 1.3.0) eventmachine (1.0.7) + evernote-thrift (1.25.1) + evernote_oauth (0.2.3) + evernote-thrift + oauth (>= 0.4.1) execjs (2.3.0) extlib (0.9.16) faraday (0.9.1) @@ -318,6 +322,10 @@ GEM omniauth-oauth2 (~> 1.0) omniauth-dropbox (0.2.0) omniauth-oauth (~> 1.0) + omniauth-evernote (1.2.1) + evernote-thrift + multi_json (~> 1.0) + omniauth-oauth (~> 1.0) omniauth-oauth (1.0.1) oauth omniauth (~> 1.0) @@ -543,6 +551,7 @@ DEPENDENCIES dotenv-rails (~> 2.0.1) dropbox-api em-http-request (~> 1.1.2) + evernote_oauth faraday (~> 0.9.0) faraday_middleware (>= 0.10.0) feed-normalizer @@ -576,6 +585,7 @@ DEPENDENCIES omniauth omniauth-37signals omniauth-dropbox + omniauth-evernote omniauth-tumblr omniauth-twitter omniauth-wunderlist! diff --git a/app/concerns/evernote_concern.rb b/app/concerns/evernote_concern.rb new file mode 100644 index 0000000000..ff6ba94146 --- /dev/null +++ b/app/concerns/evernote_concern.rb @@ -0,0 +1,48 @@ +module EvernoteConcern + extend ActiveSupport::Concern + + included do + include Oauthable + + validate :validate_evernote_options + + valid_oauth_providers :evernote + + gem_dependency_check { defined?(EvernoteOAuth) && Devise.omniauth_providers.include?(:evernote) } + end + + def evernote_client + EvernoteOAuth::Client.new( + token: evernote_oauth_token, + consumer_key: evernote_consumer_key, + consumer_secret: evernote_consumer_secret, + sandbox: false + ) + end + + private + + def validate_evernote_options + unless evernote_consumer_key.present? && + evernote_consumer_secret.present? && + evernote_oauth_token.present? + errors.add(:base, "Evernote consumer_key, consumer_secret, oauth_token, and oauth_token_secret are required to authenticate with the Twitter API. You can provide these as options to this Agent, or as Credentials with the same names, but starting with 'evernote_'.") + end + end + + def evernote_consumer_key + (config = Devise.omniauth_configs[:evernote]) && config.strategy.consumer_key + end + + def evernote_consumer_secret + (config = Devise.omniauth_configs[:evernote]) && config.strategy.consumer_secret + end + + def evernote_oauth_token + service && service.token + end + + def evernote_oauth_token_secret + service && service.secret + end +end diff --git a/app/models/agents/evernote_agent.rb b/app/models/agents/evernote_agent.rb new file mode 100644 index 0000000000..cd0adc1335 --- /dev/null +++ b/app/models/agents/evernote_agent.rb @@ -0,0 +1,373 @@ +module Agents + class EvernoteAgent < Agent + include EvernoteConcern + + description <<-MD + The Evernote Agent connects with a user's Evernote note store. + + To be able to use this agent with your account you need to authenticate with Evernote in the [Services](/services) section. + + Options: + + * `mode` - Two possible values: + + - `update` Based on events it receives, the agent will create notes + or update notes with the same `title` and `notebook` + + - `read` On a schedule, it will generate events containing data for newly + added or updated notes + + * `include_xhtml_content` - Set to `true` to include the content in ENML (Evernote Markup Language) of the note + + * `note` + + - When `mode` is `update` the parameters of `note` are the attributes of the note to be added/edited. + To edit a note, both `title` and `notebook` must be set. + + For example, to add the tags 'comic' and 'CS' to a note titled 'xkcd Survey' in the notebook 'xkcd', use: + + "notes": { + "title": "xkcd Survey", + "content": "", + "notebook": "xkcd", + "tagNames": "comic, CS" + } + + If a note with the above title and notebook did note exist already, one would be created. + + - When `mode` is `read` the values are search parameters. + Note: The `content` parameter is not used for searching. + + For example, to find all notes with tag 'CS' in the notebook 'xkcd', use: + + "notes": { + "title": "", + "content": "", + "notebook": "xkcd", + "tagNames": "CS" + } + MD + + event_description <<-MD + When `mode` is `update`, events look like: + + { + "title": "...", + "content": "...", + "notebook": "...", + "tags": "...", + "source": "...", + "sourceURL": "..." + } + + When `mode` is `read`, events look like: + + { + "title": "...", + "content": "...", + "notebook": "...", + "tags": "...", + "source": "...", + "sourceURL": "...", + "resources" : [ + { + "url": "resource1_url", + "name": "resource1_name", + "mime_type": "resource1_mime_type" + } + ... + ] + } + MD + + default_schedule "never" + + def working? + event_created_within?(interpolated['expected_update_period_in_days']) && !recent_error_logs? + end + + def default_options + { + "expected_update_period_in_days" => "2", + "mode" => "update", + "include_xhtml_content" => "false", + "note" => { + "title" => "{{title}}", + "content" => "{{content}}", + "notebook" => "{{notebook}}", + "tagNames" => "{{tag1}}, {{tag2}}" + } + } + end + + def validate_options + errors.add(:base, "mode must be 'update' or 'read'") unless %w(read update).include?(options[:mode]) + + if options[:mode] == "update" && schedule != "never" + errors.add(:base, "when mode is set to 'update', schedule must be 'never'") + end + + if options[:mode] == "read" && schedule == "never" + errors.add(:base, "when mode is set to 'read', agent must have a schedule") + end + + errors.add(:base, "expected_update_period_in_days is required") unless options['expected_update_period_in_days'].present? + + if options[:mode] == "update" && options[:note].values.all?(&:empty?) + errors.add(:base, "you must specify at least one note parameter to create or update a note") + end + end + + def include_xhtml_content? + options[:include_xhtml_content] == "true" + end + + def receive(incoming_events) + if options[:mode] == "update" + incoming_events.each do |event| + note = note_store.create_or_update_note(note_params(event)) + create_event :payload => note.attr(include_content: include_xhtml_content?) + end + end + end + + def check + if options[:mode] == "read" + opts = note_params(options) + + # convert time to evernote timestamp format: + # https://dev.evernote.com/doc/reference/Types.html#Typedef_Timestamp + opts.merge!(agent_created_at: created_at.to_i * 1000) + opts.merge!(last_checked_at: (memory[:last_checked_at] ||= created_at.to_i * 1000)) + + if opts[:tagNames] + opts.merge!(notes_with_tags: (memory[:notes_with_tags] ||= + NoteStore::Search.new(note_store, {tagNames: opts[:tagNames]}).note_guids)) + end + + notes = NoteStore::Search.new(note_store, opts).notes + notes.each do |note| + memory[:notes_with_tags] << note.guid unless memory[:notes_with_tags].include?(note.guid) + + create_event :payload => note.attr(include_resources: true, include_content: include_xhtml_content?) + end + + memory[:last_checked_at] = Time.now.to_i * 1000 + save! + end + end + + private + + def note_params(options) + params = interpolated(options)[:note] + errors.add(:base, "only one notebook allowed") unless params[:notebook].to_s.split(", ") == 1 + params[:tagNames] = params[:tagNames].to_s.split(", ") + params + end + + def evernote_note_store + evernote_client.note_store + end + + def note_store + NoteStore.new(evernote_note_store) + end + + # wrapper for evernote api NoteStore + # https://dev.evernote.com/doc/reference/ + class NoteStore + attr_reader :en_note_store + delegate :createNote, :updateNote, :getNote, :listNotebooks, :listTags, :getNotebook, + :createNotebook, :findNotesMetadata, :getNoteTagNames, :to => :en_note_store + + def initialize(en_note_store) + @en_note_store = en_note_store + end + + def create_or_update_note(params) + search = Search.new(self, {title: params[:title], notebook: params[:notebook]}) + # evernote search can only filter notes with titles containing a substring; + # this finds a note with the exact title + note = search.notes.detect {|note| note.title == params[:title]} + if note + # a note with specified title and notebook exists, so update it + update_note(params.merge(guid: note.guid, notebookGuid: note.notebookGuid)) + else + # create the notebook unless it already exists + notebook = find_notebook(name: params[:notebook]) + notebook_guid = + notebook ? notebook.guid : create_notebook(params[:notebook]).guid + + create_note(params.merge(notebookGuid: notebook_guid)) + end + end + + def create_note(params) + note = Evernote::EDAM::Type::Note.new(with_wrapped_content(params)) + en_note = createNote(note) + find_note(en_note.guid) + end + + def update_note(params) + # do not empty note properties that have not been set in `params` + params.keys.each { |key| params.delete(key) unless params[key].present? } + params = with_wrapped_content(params) + + # append specified tags instead of replacing current tags + tags = getNoteTagNames(params[:guid]) + tags.each { |tag| + params[:tagNames] << tag unless params[:tagNames].include?(tag) } + + note = Evernote::EDAM::Type::Note.new(params) + updateNote(note) + find_note(params[:guid]) + end + + def find_note(guid) + # https://dev.evernote.com/doc/reference/NoteStore.html#Fn_NoteStore_getNote + en_note = getNote(guid, true, false, false, false) + build_note(en_note) + end + + def build_note(en_note) + notebook = find_notebook(guid: en_note.notebookGuid).name + tags = en_note.tagNames || find_tags(en_note.tagGuids.to_a).map(&:name) + Note.new(en_note, notebook, tags) + end + + def find_tags(guids) + listTags.select {|tag| guids.include?(tag.guid)} + end + + def find_notebook(params) + if params[:guid] + listNotebooks.detect {|notebook| notebook.guid == params[:guid]} + elsif params[:name] + listNotebooks.detect {|notebook| notebook.name == params[:name]} + end + end + + def create_notebook(name) + notebook = Evernote::EDAM::Type::Notebook.new(name: name) + createNotebook(notebook) + end + + def with_wrapped_content(params) + params.delete(:notebook) + + if params[:content] + params[:content] = + "" \ + "" \ + "#{params[:content]}" + end + + params + end + + class Search + attr_reader :note_store, :opts + def initialize(note_store, opts) + @note_store = note_store + @opts = opts + end + + def filtered_metadata + filter, spec = create_filter, create_spec + metadata = note_store.findNotesMetadata(filter, 0, 100, spec).notes + end + + def note_guids + filtered_metadata.map(&:guid) + end + + def notes + metadata = filtered_metadata + + if opts[:last_checked_at] && opts[:tagNames] + + # evernote does note change Note#updated timestamp when a tag is added to a note + # the following selects recently updated notes + # and notes that recently had the specified tags added + metadata.select! do |note_data| + note_data.updated > opts[:last_checked_at] || + (!opts[:notes_with_tags].include?(note_data.guid) && note_data.created > opts[:agent_created_at]) + end + + elsif opts[:last_checked_at] + metadata.select! { |note_data| note_data.updated > opts[:last_checked_at] } + end + + metadata.map! { |note_data| note_store.find_note(note_data.guid) } + metadata + end + + private + + def create_filter + filter = Evernote::EDAM::NoteStore::NoteFilter.new + + # evernote search grammar: + # https://dev.evernote.com/doc/articles/search_grammar.php#Search_Terms + query_terms = [] + query_terms << "notebook:\"#{opts[:notebook]}\"" if opts[:notebook].present? + query_terms << "intitle:\"#{opts[:title]}\"" if opts[:title].present? + query_terms << "updated:day-1" if opts[:last_checked_at].present? + opts[:tagNames].to_a.each { |tag| query_terms << "tag:#{tag}" } + + filter.words = query_terms.join(" ") + filter + end + + def create_spec + Evernote::EDAM::NoteStore::NotesMetadataResultSpec.new( + includeTitle: true, + includeAttributes: true, + includeNotebookGuid: true, + includeTagGuids: true, + includeUpdated: true, + includeCreated: true + ) + end + end + end + + class Note + attr_accessor :en_note + attr_reader :notebook, :tags + delegate :guid, :notebookGuid, :title, :tagGuids, :content, :resources, + :attributes, :to => :en_note + + def initialize(en_note, notebook, tags) + @en_note = en_note + @notebook = notebook + @tags = tags + end + + def attr(opts = {}) + return_attr = { + title: title, + notebook: notebook, + tags: tags, + source: attributes.source, + source_url: attributes.sourceURL + } + + return_attr[:content] = content if opts[:include_content] + + if opts[:include_resources] && resources + return_attr[:resources] = [] + resources.each do |resource| + return_attr[:resources] << { + url: resource.attributes.sourceURL, + name: resource.attributes.fileName, + mime_type: resource.mime + } + end + end + return_attr + end + end + end +end diff --git a/config/initializers/devise.rb b/config/initializers/devise.rb index 6b533cbea8..f3ca53509b 100644 --- a/config/initializers/devise.rb +++ b/config/initializers/devise.rb @@ -263,6 +263,16 @@ config.omniauth :wunderlist, key, secret end + if defined?(OmniAuth::Strategies::Evernote) && + (key = ENV["EVERNOTE_OAUTH_KEY"]).present? && + (secret = ENV["EVERNOTE_OAUTH_SECRET"]).present? + # for production: + config.omniauth :evernote, key, secret + + # for development: + # config.omniauth :evernote, key, secret, client_options: { :site => 'https://sandbox.evernote.com' } + end + # ==> Warden configuration # If you want to use other strategies, that are not supported by Devise, or # change the failure app, you can configure them inside the config.warden block. diff --git a/config/locales/devise.en.yml b/config/locales/devise.en.yml index 8af9503fc9..5600612d84 100644 --- a/config/locales/devise.en.yml +++ b/config/locales/devise.en.yml @@ -33,6 +33,7 @@ en: 37signals: "37Signals (Basecamp)" dropbox: "Dropbox" wunderlist: 'Wunderlist' + evernote: "Evernote" passwords: no_token: "You can't access this page without coming from a password reset email. If you do come from a password reset email, please make sure you used the full URL provided." send_instructions: "You will receive an email with instructions on how to reset your password in a few minutes." diff --git a/spec/models/agents/evernote_agent_spec.rb b/spec/models/agents/evernote_agent_spec.rb new file mode 100644 index 0000000000..a9ef547705 --- /dev/null +++ b/spec/models/agents/evernote_agent_spec.rb @@ -0,0 +1,297 @@ +require 'spec_helper' + +describe Agents::EvernoteAgent do + + let(:note_store) do + class FakeEvernoteNoteStore + attr_accessor :notes, :tags, :notebooks + def initialize + @notes, @tags, @notebooks = [], [], [] + end + + def createNote(note) + note.attributes = OpenStruct.new(source: nil, sourceURL: nil) + note.guid = @notes.length + 1 + @notes << note + note + end + + def updateNote(note) + note.attributes = OpenStruct.new(source: nil, sourceURL: nil) + old_note = @notes.find {|en_note| en_note.guid == note.guid} + @notes[@notes.index(old_note)] = note + note + end + + def getNote(guid, *other_args) + @notes.find {|note| note.guid == guid} + end + + def createNotebook(notebook) + notebook.guid = @notebooks.length + 1 + @notebooks << notebook + notebook + end + + def createTag(tag) + tag.guid = @tags.length + 1 + @tags << tag + tag + end + + def listNotebooks; @notebooks; end + + def listTags; @tags; end + + def getNoteTagNames(guid) + getNote(guid).try(:tagNames) || [] + end + + def findNotesMetadata(*args); end + end + + note_store = FakeEvernoteNoteStore.new + stub.any_instance_of(Agents::EvernoteAgent).evernote_note_store { note_store } + note_store + end + + describe "#receive" do + context "when mode is set to 'update'" do + before do + @options = { + :mode => "update", + :include_xhtml_content => "false", + :expected_update_period_in_days => "2", + :note => { + :title => "{{title}}", + :content => "{{content}}", + :notebook => "{{notebook}}", + :tagNames => "{{tag1}}, {{tag2}}" + } + } + @agent = Agents::EvernoteAgent.new(:name => "evernote updater", :options => @options) + @agent.service = services(:generic) + @agent.user = users(:bob) + @agent.save! + + @event = Event.new + @event.agent = agents(:bob_website_agent) + @event.payload = { :title => "xkcd Survey", + :content => "The xkcd Survey: Big Data for a Big Planet", + :notebook => "xkcd", + :tag1 => "funny", + :tag2 => "data" } + @event.save! + + tag1 = OpenStruct.new(name: "funny") + tag2 = OpenStruct.new(name: "data") + [tag1, tag2].each { |tag| note_store.createTag(tag) } + end + + it "adds a note for any payload it receives" do + stub(note_store).findNotesMetadata { OpenStruct.new(notes: []) } + Agents::EvernoteAgent.async_receive(@agent.id, [@event.id]) + + expect(note_store.notes.size).to eq(1) + expect(note_store.notes.first.title).to eq("xkcd Survey") + expect(note_store.notebooks.size).to eq(1) + expect(note_store.tags.size).to eq(2) + + expect(@agent.events.count).to eq(1) + expect(@agent.events.first.payload).to eq({ + "title" => "xkcd Survey", + "notebook" => "xkcd", + "tags" => ["funny", "data"], + "source" => nil, + "source_url" => nil + }) + end + + context "a note with the same title and notebook exists" do + before do + note1 = OpenStruct.new(title: "xkcd Survey", notebookGuid: 1) + note2 = OpenStruct.new(title: "Footprints", notebookGuid: 1) + [note1, note2].each { |note| note_store.createNote(note) } + note_store.createNotebook(OpenStruct.new(name: "xkcd")) + + stub(note_store).findNotesMetadata { + OpenStruct.new(notes: [note1]) } + end + + it "updates the existing note" do + Agents::EvernoteAgent.async_receive(@agent.id, [@event.id]) + + expect(note_store.notes.size).to eq(2) + expect(note_store.getNote(1).tagNames).to eq(["funny", "data"]) + expect(@agent.events.count).to eq(1) + end + end + + context "include_xhtml_content is set to 'true'" do + before do + @agent.options[:include_xhtml_content] = "true" + @agent.save! + end + + it "creates an event with note content wrapped in ENML" do + stub(note_store).findNotesMetadata { OpenStruct.new(notes: []) } + Agents::EvernoteAgent.async_receive(@agent.id, [@event.id]) + + payload = @agent.events.first.payload + + expect(payload[:content]).to eq( + "" \ + "" \ + "The xkcd Survey: Big Data for a Big Planet" + ) + end + end + end + end + + describe "#check" do + context "when mode is set to 'read'" do + before do + @options = { + :mode => "read", + :include_xhtml_content => "false", + :expected_update_period_in_days => "2", + :note => { + :title => "", + :content => "", + :notebook => "xkcd", + :tagNames => "funny, comic" + } + } + @checker = Agents::EvernoteAgent.new(:name => "evernote reader", :options => @options) + + @checker.service = services(:generic) + @checker.user = users(:bob) + @checker.schedule = "every_2h" + + @checker.save! + @checker.created_at = 1.minute.ago + + note_store.createNote( + OpenStruct.new(title: "xkcd Survey", + notebookGuid: 1, + updated: 2.minutes.ago.to_i * 1000, + tagNames: ["funny", "comic"]) + ) + note_store.createNotebook(OpenStruct.new(name: "xkcd")) + tag1 = OpenStruct.new(name: "funny") + tag2 = OpenStruct.new(name: "comic") + [tag1, tag2].each { |tag| note_store.createTag(tag) } + + stub(note_store).findNotesMetadata { + notes = note_store.notes.select do |note| + note.notebookGuid == 1 && + %w(funny comic).all? { |tag_name| note.tagNames.include?(tag_name) } + end + OpenStruct.new(notes: notes) + } + end + + context "the first time it checks" do + it "returns only notes created/updated since it was created" do + expect { @checker.check }.to change { Event.count }.by(0) + end + end + + context "on subsequent checks" do + it "returns notes created/updated since the last time it checked" do + expect { @checker.check }.to change { Event.count }.by(0) + + future_time = (Time.now + 1.minute).to_i * 1000 + note_store.createNote( + OpenStruct.new(title: "Footprints", + notebookGuid: 1, + tagNames: ["funny", "comic", "recent"], + updated: future_time)) + + note_store.createNote( + OpenStruct.new(title: "something else", + notebookGuid: 2, + tagNames: ["funny", "comic"], + updated: future_time)) + + expect { @checker.check }.to change { Event.count }.by(1) + end + + it "returns notes tagged since the last time it checked" do + note_store.createNote( + OpenStruct.new(title: "Footprints", + notebookGuid: 1, + tagNames: [], + created: Time.now.to_i * 1000, + updated: Time.now.to_i * 1000)) + @checker.check + + note_store.getNote(2).tagNames = ["funny", "comic"] + + expect { @checker.check }.to change { Event.count }.by(1) + end + end + end + end + + describe "#validation" do + before do + @options = { + :mode => "update", + :include_xhtml_content => "false", + :expected_update_period_in_days => "2", + :note => { + :title => "{{title}}", + :content => "{{content}}", + :notebook => "{{notebook}}", + :tagNames => "{{tag1}}, {{tag2}}" + } + } + @agent = Agents::EvernoteAgent.new(:name => "evernote updater", :options => @options) + @agent.service = services(:generic) + @agent.user = users(:bob) + @agent.save! + + expect(@agent).to be_valid + end + + it "requires the mode to be 'update' or 'read'" do + @agent.options[:mode] = "" + expect(@agent).not_to be_valid + end + + context "mode is set to 'update'" do + before do + @agent.options[:mode] = "update" + end + + it "requires some note parameter to be present" do + @agent.options[:note].keys.each { |k| @agent.options[:note][k] = "" } + expect(@agent).not_to be_valid + end + + it "requires schedule to be 'never'" do + @agent.schedule = 'never' + expect(@agent).to be_valid + + @agent.schedule = 'every_1m' + expect(@agent).not_to be_valid + end + end + + context "mode is set to 'read'" do + before do + @agent.options[:mode] = "read" + end + + it "requires a schedule to be set" do + @agent.schedule = 'every_1m' + expect(@agent).to be_valid + + @agent.schedule = 'never' + expect(@agent).not_to be_valid + end + end + end +end