Commit 42de2e66 by murtaugh

feedr using updated_parsed ipv published_parsed, good for all feeds?

parent 6c2e9681
Showing with 5 additions and 4 deletions
......@@ -141,7 +141,7 @@ class SitePage (object):
else:
print ("Creating wiki page {0}".format(self.page.name), file=sys.stderr)
ts = ts2dt(item.published_parsed)
ts = ts2dt(item.updated_parsed)
# ts = datetime.datetime(*item.published_parsed[:6])
# print ("published:", ts.strftime("%Y/%m/%d %H:%M:%S"), file=sys.stderr)
......@@ -275,7 +275,8 @@ class SiteWebPage (object):
def process_feed(self, cachedir, limit=None, phpuploader=None, user=None, force=False, pretend=False):
feed = feedparser.parse(self.feed_url)
# ensure in reverse chronological order (probably redundant but important for the processing)
feed.entries.sort(key=lambda x: x.published_parsed, reverse=True)
# feed.entries.sort(key=lambda x: x.published_parsed, reverse=True)
feed.entries.sort(key=lambda x: x.updated_parsed, reverse=True)
# print (feed.entries[0].published_parsed, "to", feed.entries[-1].published_parsed)
count = 0
# Process in CHRONOLOGICAL order... skipping elements that OLDER than / equal to feed's last updated timestamp
......@@ -287,7 +288,7 @@ class SiteWebPage (object):
use_all_entries = []
# print ("processing feed, last_updated {0}".format(self.last_updated.strftime(DATETIME_STRF)), file=sys.stderr)
for item in all_entries:
item_dt = ts2dt(item.published_parsed)
item_dt = ts2dt(item.updated_parsed)
if item_dt <= self.last_updated:
# print ("Skipping older item {0}".format(item.title), file=sys.stderr)
skipped += 1
......@@ -300,7 +301,7 @@ class SiteWebPage (object):
print ("No new items since feed last updated", file=sys.stderr)
for item in all_entries:
item_dt = ts2dt(item.published_parsed)
item_dt = ts2dt(item.updated_parsed)
sitepage = SitePage(self.wiki, item.link)
sitepage.process_feed_item(item, source=self.pagename, cachedir=cachedir, phpuploader=phpuploader, user=user, pretend=pretend)
if self.last_updated == None or item_dt > self.last_updated:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment