X-Git-Url: https://git.kaliko.me/?a=blobdiff_plain;f=sid%2Ffeeds.py;h=5a33d6d5ccb40dbaa2c1f88d3c5a1795d02e16aa;hb=8d02f7db31a8587f1cd9bb31bcdffd517f6c144e;hp=043c784dcb23f4cf0484bee46badee0f2e370e27;hpb=1d5d7c371bdecaf42cd6785f8c8ab152f0eaec48;p=sid.git
diff --git a/sid/feeds.py b/sid/feeds.py
index 043c784..5a33d6d 100644
--- a/sid/feeds.py
+++ b/sid/feeds.py
@@ -17,6 +17,7 @@
import datetime
import threading
import time
+import traceback
from feedparser import parse as feed_parse
@@ -46,41 +47,34 @@ class FeedMonitor(threading.Thread):
threading.Thread.__init__(self)
self.feeds_list = plugin.FEEDS
self.tempo = plugin.TEMPO
- self.bot = plugin.bot
+ self.plugin = plugin
self.last_check = datetime.datetime.utcnow()
self.seen = dict()
self.thread_killed = False
- def send(self, message):
- """simple wrapper around bot send_message method"""
- self.bot.send_message(mto=self.bot.room,
- mbody=message[1],
- mhtml=message[0],
- mtype='groupchat')
-
def new_posts(self, feed):
"""Send new posts in feed"""
parsed_feed = feed_parse(feed)
# Cannot resolve address
if 'status' not in parsed_feed:
- self.bot.log.error('Error from "%s": %s.' %
- (feed, parsed_feed.bozo_exception.__repr__()))
+ self.plugin.log.error('Error from "%s": %s.',
+ feed, parsed_feed.bozo_exception.__repr__())
return
# unusual return http code
if parsed_feed.status != 200:
- self.bot.log.error(
- 'Got code %(status)d from "%(href)s" (please update).' %
- parsed_feed)
+ self.plugin.log.warning(
+ 'Got code %(status)d from "%(href)s" (please update).',
+ parsed_feed)
return
feed_updated = parsed_feed.feed.get('updated_parsed', None)
# Avoid looping over all posts if possible
if feed_updated and strtm_to_dtm(feed_updated) < self.last_check:
- self.bot.log.debug('updated : %s' % strtm_to_dtm(feed_updated))
- self.bot.log.debug('last check: %s' % self.last_check)
+ self.plugin.log.debug('updated : %s', strtm_to_dtm(feed_updated))
+ self.plugin.log.debug('last check: %s', self.last_check)
return
title = '"%s":' % parsed_feed.feed.get('title', 'n/a')
@@ -92,34 +86,39 @@ class FeedMonitor(threading.Thread):
if not self.seen.setdefault(feed_id):
# Fills with post id when first started (prevent from posting all
# entries at startup)
- self.seen[feed_id] = [post.id for post in parsed_feed.entries]
+ self.seen[feed_id] = {p.id for p in parsed_feed.entries}
return
- for post in parsed_feed.entries:
- if post.id not in self.seen.get(feed_id):
- self.seen[feed_id].append(post.id)
- self.bot.log.info(post.title)
-
- body = '%(title)s %(link)s' % post
- text.append(body)
-
- xpost = dict(**post)
- xpost['title'] = html_escape(xpost.get('title', 'n/a'))
- xbody = '%(title)s' % xpost
- xhtml.append(xbody)
-
+ # Detecting new post
+ entries = {p.id for p in parsed_feed.entries}
+ new_entries = [p for p in parsed_feed.entries
+ if p.id in entries - self.seen.get(feed_id)]
+ for post in new_entries:
+ self.plugin.log.info(post.title)
+
+ body = '%(title)s %(link)s' % post
+ text.append(body)
+
+ xpost = {'title': html_escape(post.get('title', 'n/a'))}
+ xpost['link'] = html_escape(post.get('link',))
+ xbody = '{title}'.format(**xpost)
+ xhtml.append(xbody)
+ # Updating self.seen
+ self.seen[feed_id] = entries
if len(text) > 1:
- self.send(('
'.join(xhtml), '\n'.join(text)))
+ self.plugin.send(self.plugin.bot.room,
+ {'mhtml':'
'.join(xhtml), 'mbody':'\n'.join(text)},
+ mtype='groupchat')
def run(self):
while not self.thread_killed:
- self.bot.log.debug('feeds check')
+ self.plugin.log.debug('feeds check')
for feed in self.feeds_list:
try:
self.new_posts(feed)
except Exception as err:
- self.bot.log.error('feeds thread crashed')
- self.bot.log.error(err)
+ self.plugin.log.error('feeds thread crashed: %s', err)
+ self.plugin.log.error(''.join(traceback.format_exc()))
self.thread_killed = True
self.last_check = datetime.datetime.utcnow()
for _ in list(range(self.tempo)):
@@ -138,13 +137,13 @@ class Feeds(Plugin):
# 'http://www.debian.org/News/news',
# DPN in french
- 'http://www.debian.org/News/weekly/dwn.fr.rdf',
+ 'http://www.debian.org/News/weekly/dwn.fr.rdf',
# Misc
- 'http://rss.gmane.org/topics/excerpts/gmane.linux.debian.devel.announce',
- 'http://rss.gmane.org/gmane.linux.debian.user.security.announce',
- 'http://planet-fr.debian.net/users/rss20.xml',
- 'http://planet.debian.org/atom.xml',
+ 'http://rss.gmane.org/topics/excerpts/gmane.linux.debian.devel.announce',
+ 'http://rss.gmane.org/gmane.linux.debian.user.security.announce',
+ 'http://planet-fr.debian.net/users/rss20.xml',
+ 'http://planet.debian.org/atom.xml',
]
def __init__(self, bot):
@@ -157,10 +156,16 @@ class Feeds(Plugin):
self.th_mon.thread_killed = True
@botcmd
- def feeds(self, message, args):
+ def feeds(self, rcv, args):
"""feeds monitors debian project related feeds.
!feeds : registred feeds list
!feeds last : last check time"""
if 'last' in args:
- return 'Last feeds check: %s' % self.th_mon.last_check
- return '\n'.join(Feeds.FEEDS)
+ self.reply(rcv, 'Last feeds check: %s' % self.th_mon.last_check)
+ return
+ html = ['{1}'.format(html_escape(u),
+ html_escape(u[7:])
+ ) for u in Feeds.FEEDS]
+ msg = {'mbody': 'Feeds:\n' + '\n'.join(Feeds.FEEDS),
+ 'mhtml': 'Feeds:
' + '
'.join(html),}
+ self.reply(rcv, msg)