+ foreach my $feed (@_) {
+ $feed->{lastupdate}=time;
+ $feed->{newposts}=0;
+ $feed->{message}=sprintf(gettext("processed ok at %s"),
+ displaytime($feed->{lastupdate}));
+ $feed->{error}=0;
+
+ debug(sprintf(gettext("checking feed %s ..."), $feed->{name}));
+
+ if (! length $feed->{feedurl}) {
+ my @urls=XML::Feed->find_feeds($feed->{url});
+ if (! @urls) {
+ $feed->{message}=sprintf(gettext("could not find feed at %s"), $feed->{url});
+ $feed->{error}=1;
+ debug($feed->{message});
+ next;
+ }
+ $feed->{feedurl}=pop @urls;
+ }
+ my $res=URI::Fetch->fetch($feed->{feedurl});
+ if (! $res) {
+ $feed->{message}=URI::Fetch->errstr;
+ $feed->{error}=1;
+ debug($feed->{message});
+ next;
+ }
+ if ($res->status == URI::Fetch::URI_GONE()) {
+ $feed->{message}=gettext("feed not found");
+ $feed->{error}=1;
+ debug($feed->{message});
+ next;
+ }
+ my $content=$res->content;
+ my $f=eval{XML::Feed->parse(\$content)};
+ if ($@) {
+ # One common cause of XML::Feed crashing is a feed
+ # that contains invalid UTF-8 sequences. Convert
+ # feed to ascii to try to work around.
+ $feed->{message}.=" ".sprintf(gettext("(invalid UTF-8 stripped from feed)"));
+ $content=Encode::decode_utf8($content);
+ $f=eval{XML::Feed->parse(\$content)};
+ }
+ if ($@) {
+ # Another possibility is badly escaped entities.
+ $feed->{message}.=" ".sprintf(gettext("(feed entities escaped)"));
+ $content=~s/\&(?!amp)(\w+);/&$1;/g;
+ $content=Encode::decode_utf8($content);
+ $f=eval{XML::Feed->parse(\$content)};
+ }
+ if ($@) {
+ $feed->{message}=gettext("feed crashed XML::Feed!")." ($@)";
+ $feed->{error}=1;
+ debug($feed->{message});
+ next;
+ }
+ if (! $f) {
+ $feed->{message}=XML::Feed->errstr;
+ $feed->{error}=1;
+ debug($feed->{message});
+ next;
+ }
+
+ foreach my $entry ($f->entries) {
+ add_page(
+ feed => $feed,
+ copyright => $f->copyright,
+ title => defined $entry->title ? decode_entities($entry->title) : "untitled",
+ link => $entry->link,
+ content => defined $entry->content->body ? $entry->content->body : "",
+ guid => defined $entry->id ? $entry->id : time."_".$feed->{name},
+ ctime => $entry->issued ? ($entry->issued->epoch || time) : time,
+ );
+ }
+ }