1 # -*- coding: utf-8 -*-
3 # Copyright (C) 2011, 2014, 2020 kaliko <kaliko@azylum.org>
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation, version 3 only.
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
14 # You should have received a copy of the GNU General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
22 from urllib.error import URLError
24 from feedparser import parse as feed_parse
26 from .plugin import Plugin, botcmd
38 def html_escape(text):
39 """Produce entities within text."""
40 return ''.join(html_escape_table.get(c, c) for c in text)
43 def strtm_to_dtm(struc_time):
44 return datetime.datetime(*struc_time[:6])
47 class FeedMonitor(threading.Thread):
48 def __init__(self, plugin):
49 threading.Thread.__init__(self)
50 self.feeds_list = plugin.FEEDS
51 self.tempo = plugin.TEMPO
53 self.last_check = datetime.datetime.utcnow()
55 self.thread_killed = False
57 def _update_cache(self, feed, parsed):
58 self.seen[feed].update({'ids': {p.id for p in parsed.entries} or {}})
60 if parsed.get('etag', False):
61 self.seen[feed].update({'cache': {'etag': parsed.etag}})
62 if parsed.get('modified', False):
63 self.seen[feed].update({'cache': {'modified': parsed.modified}})
65 def new_posts(self, feed):
66 """Send new posts in feed"""
67 self.plugin.log.debug('feed: : "%s"', feed)
68 if self.seen.get(feed) and self.seen.get(feed).get('cache'):
69 parsed_feed = feed_parse(feed, **self.seen[feed]['cache'])
71 if self.seen.get(feed):
72 self.plugin.log.debug('No cache headers set (etag/modified)')
73 parsed_feed = feed_parse(feed)
74 # Cannot resolve address
75 if 'status' not in parsed_feed:
76 self.plugin.log.error('Error from "%s": %s.',
77 feed, parsed_feed.bozo_exception.__repr__())
80 if parsed_feed.status == 304:
81 self.plugin.log.debug('Got 304 not modified')
83 # unusual return http code
84 if parsed_feed.status != 200:
85 self.plugin.log.warning(
86 'Got code %(status)d from "%(href)s" (please update).',
89 if not self.seen.setdefault(feed):
90 # Fills with post id when first started (prevent from posting all
92 self.seen[feed] = {'cache': None}
93 self._update_cache(feed, parsed_feed)
95 title = '"%s":' % parsed_feed.feed.get('title', 'n/a')
96 xtitle = '<strong>%s</strong>:' % html_escape(
97 parsed_feed.feed.get('title', 'n/a'))
102 entries = {p.id for p in parsed_feed.entries}
103 seen_ids = self.seen.get(feed).get('ids')
104 new_entries = [p for p in parsed_feed.entries
105 if p.id in entries - seen_ids]
106 for post in new_entries:
107 self.plugin.log.info(post.title)
108 body = '%(title)s %(link)s' % post
110 xpost = {'title': html_escape(post.get('title', 'n/a'))}
111 xpost['link'] = html_escape(post.get('link',))
112 xbody = '<a href="{link}">{title}</a>'.format(**xpost)
114 # Updating self.seen, entries and cache headers
115 self._update_cache(feed, parsed_feed)
117 self.plugin.send(self.plugin.bot.room,
118 {'mhtml': '<br />'.join(xhtml), 'mbody': '\n'.join(text)},
122 while not self.thread_killed:
123 self.plugin.log.debug('feeds check')
124 for feed in self.feeds_list:
127 except URLError as err: # Non fatal exception
128 self.plugin.log.error(f'error for {feed}: {err.reason}')
129 except Exception as err: # Unknown execption, killing thread anyway
130 self.plugin.log.error('feeds thread crashed: %s', err)
131 self.plugin.log.error(''.join(traceback.format_exc()))
132 self.thread_killed = True
133 self.last_check = datetime.datetime.utcnow()
134 for _ in list(range(self.tempo)):
136 if self.thread_killed:
143 Feeds plugin depends on external module: **feedparser**
146 #: Time between feeds check
148 #: Default feeds to monitor
150 'https://www.debian.org/security/dsa',
151 'https://www.debian.org/News/news',
153 'https://tracker.debian.org/pkg/prosody/rss',
154 'https://tracker.debian.org/pkg/ejabberd/rss',
156 'https://planet.debian.org/atom.xml',
159 def __init__(self, bot):
160 Plugin.__init__(self, bot)
161 self.last_check = None
162 self.th_mon = FeedMonitor(self)
166 self.th_mon.thread_killed = True
169 def feeds(self, rcv, args):
170 """Monitors debian project related feeds.
172 * ``!feeds`` : registred feeds list
173 * ``!feeds last`` : last check time"""
175 self.reply(rcv, 'Last feeds check: %s' % self.th_mon.last_check)
177 html = ['<a href="{0}">{1}</a>'.format(html_escape(u),
179 ) for u in Feeds.FEEDS]
180 msg = {'mbody': 'Feeds:\n' + '\n'.join(Feeds.FEEDS),
181 'mhtml': 'Feeds:<br />' + '<br />'.join(html)}