hook(type => "getopt", id => "aggregate", call => \&getopt);
hook(type => "getsetup", id => "aggregate", call => \&getsetup);
hook(type => "checkconfig", id => "aggregate", call => \&checkconfig);
hook(type => "getopt", id => "aggregate", call => \&getopt);
hook(type => "getsetup", id => "aggregate", call => \&getsetup);
hook(type => "checkconfig", id => "aggregate", call => \&checkconfig);
if (exists $config{aggregate_webtrigger} && $config{aggregate_webtrigger}) {
hook(type => "cgi", id => "aggregate", call => \&cgi);
}
if (exists $config{aggregate_webtrigger} && $config{aggregate_webtrigger}) {
hook(type => "cgi", id => "aggregate", call => \&cgi);
}
"aggregate" => \$config{aggregate},
"aggregateinternal!" => \$config{aggregateinternal},
);
"aggregate" => \$config{aggregate},
"aggregateinternal!" => \$config{aggregateinternal},
);
description => "enable aggregation to internal pages?",
safe => 0, # enabling needs manual transition
rebuild => 0,
description => "enable aggregation to internal pages?",
safe => 0, # enabling needs manual transition
rebuild => 0,
my $oldoutput = $config{destdir}."/".IkiWiki::htmlpage($data->{page});
$config{aggregateinternal} = 1;
my $oldoutput = $config{destdir}."/".IkiWiki::htmlpage($data->{page});
$config{aggregateinternal} = 1;
$feed->{template}=$params{template} . ".tmpl";
delete $feed->{unseen};
$feed->{lastupdate}=0 unless defined $feed->{lastupdate};
$feed->{template}=$params{template} . ".tmpl";
delete $feed->{unseen};
$feed->{lastupdate}=0 unless defined $feed->{lastupdate};
$feed->{numposts}=0 unless defined $feed->{numposts};
$feed->{newposts}=0 unless defined $feed->{newposts};
$feed->{message}=gettext("new feed") unless defined $feed->{message};
$feed->{numposts}=0 unless defined $feed->{numposts};
$feed->{newposts}=0 unless defined $feed->{newposts};
$feed->{message}=gettext("new feed") unless defined $feed->{message};
close OUT || error("save $newfile: $!", $cleanup);
rename($newfile, "$config{wikistatedir}/aggregate") ||
error("rename $newfile: $!", $cleanup);
close OUT || error("save $newfile: $!", $cleanup);
rename($newfile, "$config{wikistatedir}/aggregate") ||
error("rename $newfile: $!", $cleanup);
foreach my $name (keys %feeds) {
# remove any feeds that were not seen while building the pages
# that used to contain them
foreach my $name (keys %feeds) {
# remove any feeds that were not seen while building the pages
# that used to contain them
foreach my $guid (values %guids) {
# any guid whose feed is gone should be removed
if (! exists $feeds{$guid->{feed}}) {
foreach my $guid (values %guids) {
# any guid whose feed is gone should be removed
if (! exists $feeds{$guid->{feed}}) {
# Load the current state in from disk, and merge into it
# values from the state in memory that might have changed
# during aggregation.
# Load the current state in from disk, and merge into it
# values from the state in memory that might have changed
# during aggregation.
# It's also possible that guids were removed from the on-disk state
# while the aggregation was in process. That would only happen if
# their feed was also removed, so any removed guids added back here
# It's also possible that guids were removed from the on-disk state
# while the aggregation was in process. That would only happen if
# their feed was also removed, so any removed guids added back here
return values %feeds if $config{rebuild};
return grep { time - $_->{lastupdate} >= $_->{updateinterval} } values %feeds;
return values %feeds if $config{rebuild};
return grep { time - $_->{lastupdate} >= $_->{updateinterval} } values %feeds;
- $feed->{message}=sprintf(gettext("processed ok at %s"),
- displaytime($feed->{lastupdate}));
+ $feed->{message}=sprintf(gettext("last checked %s"),
+ displaytime($feed->{lasttry}));
# that contains invalid UTF-8 sequences. Convert
# feed to ascii to try to work around.
$feed->{message}.=" ".sprintf(gettext("(invalid UTF-8 stripped from feed)"));
# that contains invalid UTF-8 sequences. Convert
# feed to ascii to try to work around.
$feed->{message}.=" ".sprintf(gettext("(invalid UTF-8 stripped from feed)"));
- $content=Encode::decode_utf8($content, 0);
- $f=eval{XML::Feed->parse(\$content)};
+ $f=eval {
+ $content=Encode::decode_utf8($content, 0);
+ XML::Feed->parse(\$content)
+ };
}
if ($@) {
# Another possibility is badly escaped entities.
$feed->{message}.=" ".sprintf(gettext("(feed entities escaped)"));
$content=~s/\&(?!amp)(\w+);/&$1;/g;
}
if ($@) {
# Another possibility is badly escaped entities.
$feed->{message}.=" ".sprintf(gettext("(feed entities escaped)"));
$content=~s/\&(?!amp)(\w+);/&$1;/g;
- $content=Encode::decode_utf8($content, 0);
- $f=eval{XML::Feed->parse(\$content)};
+ $f=eval {
+ $content=Encode::decode_utf8($content, 0);
+ XML::Feed->parse(\$content)
+ };
- my $content=$content=$entry->content->body;
+ # XML::Feed doesn't work around XML::Atom's bizarre
+ # API, so we will. Real unicode strings? Yes please.
+ # See [[bugs/Aggregated_Atom_feeds_are_double-encoded]]
+ local $XML::Atom::ForceUnicode = 1;
+
+ my $c=$entry->content;
copyright => $f->copyright,
title => defined $entry->title ? decode_entities($entry->title) : "untitled",
link => $entry->link,
copyright => $f->copyright,
title => defined $entry->title ? decode_entities($entry->title) : "untitled",
link => $entry->link,
guid => defined $entry->id ? $entry->id : time."_".$feed->{name},
ctime => $entry->issued ? ($entry->issued->epoch || time) : time,
guid => defined $entry->id ? $entry->id : time."_".$feed->{name},
ctime => $entry->issued ? ($entry->issued->epoch || time) : time,
# escape slashes and periods in title so it doesn't specify
# directory name or trigger ".." disallowing code.
$page=~s!([/.])!"__".ord($1)."__"!eg;
# escape slashes and periods in title so it doesn't specify
# directory name or trigger ".." disallowing code.
$page=~s!([/.])!"__".ord($1)."__"!eg;
my $template=template($feed->{template}, blind_cache => 1);
$template->param(title => $params{title})
if defined $params{title} && length($params{title});
my $template=template($feed->{template}, blind_cache => 1);
$template->param(title => $params{title})
if defined $params{title} && length($params{title});
- $template->param(content => htmlescape(htmlabs($params{content}, $feed->{feedurl})));
+ $template->param(content => wikiescape(htmlabs($params{content},
+ defined $params{base} ? $params{base} : $feed->{feedurl})));
$template->param(name => $feed->{name});
$template->param(url => $feed->{url});
$template->param(copyright => $params{copyright})
$template->param(name => $feed->{name});
$template->param(url => $feed->{url});
$template->param(copyright => $params{copyright})
if (defined $mtime && $mtime <= time) {
# Set the mtime, this lets the build process get the right
# creation time on record for the new page.
if (defined $mtime && $mtime <= time) {
# Set the mtime, this lets the build process get the right
# creation time on record for the new page.
- $IkiWiki::pagectime{$guid->{page}}=$mtime;
+ $IkiWiki::pagectime{$guid->{page}}=$mtime
+ unless exists $IkiWiki::pagectime{$guid->{page}};
+ }
+ else {
+ # Dummy value for expiry code.
+ $IkiWiki::pagectime{$guid->{page}}=time
+ unless exists $IkiWiki::pagectime{$guid->{page}};
# Convert links in html from relative to absolute.
# Note that this is a heuristic, which is not specified by the rss
# spec and may not be right for all feeds. Also, see Debian
# Convert links in html from relative to absolute.
# Note that this is a heuristic, which is not specified by the rss
# spec and may not be right for all feeds. Also, see Debian
# Take an exclusive lock to prevent multiple concurrent aggregators.
# Returns true if the lock was aquired.
if (! -d $config{wikistatedir}) {
# Take an exclusive lock to prevent multiple concurrent aggregators.
# Returns true if the lock was aquired.
if (! -d $config{wikistatedir}) {