58 lines
1.8 KiB
Python
58 lines
1.8 KiB
Python
|
"""RSS feed aggregator. This module contains no user-facing commands."""
|
||
|
|
|
||
|
|
import sys
|
||
|
|
import os
|
||
|
|
import threading
|
||
|
|
import traceback
|
||
|
|
import feedparser
|
||
|
|
|
||
|
|
def on_connected(tenquestionmarks):
|
||
|
|
_rss_loop(tenquestionmarks,tenquestionmarks.config()["rss"]["frequency"])
|
||
|
|
|
||
|
|
def _rss_loop(tenquestionmarks, frequency=900.0):
|
||
|
|
try:
|
||
|
|
old_entries_file = os.path.join(tenquestionmarks.directory(),"old-feed-entries")
|
||
|
|
FILE = open(old_entries_file, "r")
|
||
|
|
filetext = FILE.read()
|
||
|
|
FILE.close()
|
||
|
|
except IOError:
|
||
|
|
filetext = ""
|
||
|
|
open(old_entries_file, "w").close()
|
||
|
|
|
||
|
|
filetext = filetext.decode("UTF-8")
|
||
|
|
for feed in tenquestionmarks.config()["rss"]["feeds"]:
|
||
|
|
feedname = ""
|
||
|
|
if isinstance(tenquestionmarks.config()["rss"]["feeds"],dict):
|
||
|
|
feedname = feed
|
||
|
|
feed = tenquestionmarks.config()["rss"]["feeds"][feed]
|
||
|
|
|
||
|
|
NextFeed = False
|
||
|
|
tenquestionmarks.log("refresh","Refreshing feed %s" % (feed))
|
||
|
|
d = feedparser.parse(feed)
|
||
|
|
for entry in d.entries:
|
||
|
|
title = entry.title
|
||
|
|
try:
|
||
|
|
title = title.encode("ascii")
|
||
|
|
except UnicodeEncodeError, uee:
|
||
|
|
title = tenquestionmarks.degrade_to_ascii(title)
|
||
|
|
except UnicodeDecodeError, ude:
|
||
|
|
title = tenquestionmarks.degrade_to_ascii(title)
|
||
|
|
if title in filetext:
|
||
|
|
tenquestionmarks.log("refresh","Old entry: %s" % (title))
|
||
|
|
NextFeed = True
|
||
|
|
else:
|
||
|
|
FILE = open(old_entries_file, "a")
|
||
|
|
try:
|
||
|
|
FILE.write(title + u"\n")
|
||
|
|
except Exception, e:
|
||
|
|
traceback.print_exc(file=sys.stdout)
|
||
|
|
tenquestionmarks.log("Error","%s %s" % (e,title))
|
||
|
|
FILE.close()
|
||
|
|
tenquestionmarks.queue(tenquestionmarks.config()["rss"]["format"] % ({"title": title, "link": entry.link, "feedname": feedname}))
|
||
|
|
if NextFeed:
|
||
|
|
break
|
||
|
|
|
||
|
|
def refresher(): _rss_loop(tenquestionmarks,frequency)
|
||
|
|
t = threading.Timer(frequency, refresher) # TODO: make this static
|
||
|
|
t.start()
|