From e4e726bd64601d582c2859360bab3e7300b3e127 Mon Sep 17 00:00:00 2001 From: kev Date: Tue, 31 Oct 2023 14:51:34 +0800 Subject: [PATCH] add ./data to repo --- Dockerfile | 1 + README.md | 4 +- data/hooks.py | 119 +++++++++++++++++++++++++++++++++++++++++++++ data/urls.yaml | 36 ++++++++++++++ data/urlwatch.yaml | 12 +++++ 5 files changed, 170 insertions(+), 2 deletions(-) create mode 100644 data/hooks.py create mode 100644 data/urls.yaml create mode 100644 data/urlwatch.yaml diff --git a/Dockerfile b/Dockerfile index 5f71f7b..325984a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,7 @@ ARG URLWATCH_VERSION RUN set -xe \ && apk add --no-cache ca-certificates \ bash \ + bind-tools \ build-base \ curl \ jq \ diff --git a/README.md b/README.md index c171e3b..d69995d 100644 --- a/README.md +++ b/README.md @@ -78,8 +78,6 @@ v3.3.5 >>> exit ``` -[1]: https://thp.io/2008/urlwatch/ - ## customizing cron schedule ### Create a crontab file @@ -103,3 +101,5 @@ services: - ./data/crontab:/etc/crontabs/root restart: unless-stopped ``` + +[1]: https://github.com/thp/urlwatch diff --git a/data/hooks.py b/data/hooks.py new file mode 100644 index 0000000..4482f96 --- /dev/null +++ b/data/hooks.py @@ -0,0 +1,119 @@ +# +# Example hooks file for urlwatch +# +# Copyright (c) 2008-2023 Thomas Perl +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. The name of the author may not be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +import re + +from urlwatch import filters +from urlwatch import jobs +from urlwatch import reporters + + +#class CustomLoginJob(jobs.UrlJob): +# """Custom login for my webpage""" +# +# __kind__ = 'custom-login' +# __required__ = ('username', 'password') +# +# def retrieve(self, job_state): +# return 'Would log in to {} with {} and {}\n'.format(self.url, self.username, self.password) + + +#class CaseFilter(filters.FilterBase): +# """Custom filter for changing case, needs to be selected manually""" +# +# __kind__ = 'case' +# +# def filter(self, data, subfilter): +# # The subfilter is specified using a colon, for example the "case" +# # filter here can be specified as "case:upper" and "case:lower" +# +# if subfilter is None: +# subfilter = 'upper' +# +# if subfilter == 'upper': +# return data.upper() +# elif subfilter == 'lower': +# return data.lower() +# else: +# raise ValueError('Unknown case subfilter: %r' % (subfilter,)) + + +#class IndentFilter(filters.FilterBase): +# """Custom filter for indenting, needs to be selected manually""" +# +# __kind__ = 'indent' +# +# def filter(self, data, subfilter): +# # The subfilter here is a number of characters to indent +# +# if subfilter is None: +# indent = 8 +# else: +# indent = int(subfilter) +# +# return '\n'.join((' '*indent) + line for line in data.splitlines()) + + + +class CustomMatchUrlFilter(filters.AutoMatchFilter): + # The AutoMatchFilter will apply automatically to all filters + # that have the given properties set + MATCH = {'url': 'http://example.org/'} + + # An auto-match filter does not have any subfilters + def filter(self, data, subfilter): + return data.replace('foo', 'bar') + +class CustomRegexMatchUrlFilter(filters.RegexMatchFilter): + # Similar to AutoMatchFilter + MATCH = {'url': re.compile('http://example.org/.*')} + + # An auto-match filter does not have any subfilters + def filter(self, data, subfilter): + return data.replace('foo', 'bar') + + +class CustomTextFileReporter(reporters.TextReporter): + """Custom reporter that writes the text-only report to a file""" + + __kind__ = 'custom_file' + + def submit(self): + with open(self.config['filename'], 'w') as fp: + fp.write('\n'.join(super().submit())) + + +class CustomHtmlFileReporter(reporters.HtmlReporter): + """Custom reporter that writes the HTML report to a file""" + + __kind__ = 'custom_html' + + def submit(self): + with open(self.config['filename'], 'w') as fp: + fp.write('\n'.join(super().submit())) diff --git a/data/urls.yaml b/data/urls.yaml new file mode 100644 index 0000000..acc34d0 --- /dev/null +++ b/data/urls.yaml @@ -0,0 +1,36 @@ +--- + +name: urlwatch +url: https://github.com/thp/urlwatch/tags +user_visible_url: https://github.com/thp/urlwatch +filter: +- xpath: '(//h4[@data-test-selector="tag-title"]/a)[1]' +- html2text: re +- strip: + +--- + +name: shadowsocks-libev +url: https://api.github.com/repos/shadowsocks/shadowsocks-libev/releases/latest +user_visible_url: https://github.com/shadowsocks/shadowsocks-libev +filter: +- shellpipe: 'jq -r .tag_name' +- strip: + +--- + +name: easypi +url: https://www.nslookup.io/api/v1/records +user_visible_url: https://www.nslookup.io/domains/easypi.duckdns.org/dns-records/#google +method: POST +headers: + Content-Type: application/json +data: '{"domain":"easypi.duckdns.org","dnsServer":"google"}' +filter: +- jq: + query: '.records.a.response.answer[0].ipInfo.query' +- re.sub: + pattern: '"' + repl: '' + +... diff --git a/data/urlwatch.yaml b/data/urlwatch.yaml new file mode 100644 index 0000000..78bf11c --- /dev/null +++ b/data/urlwatch.yaml @@ -0,0 +1,12 @@ +job_defaults: + url: + ignore_connection_errors: true + ignore_http_error_codes: 4xx, 5xx + treat_new_as_changed: true + +report: + text: + footer: false + slack: + webhook_url: https://hooks.slack.com/services/XXXXXXXXX/YYYYYYYYY/ZZZZZZZZZZZZZZZZZZZZZZZZ + enabled: true