X-Git-Url: http://git.vanrenterghem.biz/git.ikiwiki.info.git/blobdiff_plain/c23e8009bf7b4300b1bc32aea6528a1e7454ec1f..819b31d46c05c00d4183f824f74bfe3836471d78:/IkiWiki/Plugin/aggregate.pm?ds=inline diff --git a/IkiWiki/Plugin/aggregate.pm b/IkiWiki/Plugin/aggregate.pm index 99b26a546..c6ed6b209 100644 --- a/IkiWiki/Plugin/aggregate.pm +++ b/IkiWiki/Plugin/aggregate.pm @@ -5,6 +5,10 @@ package IkiWiki::Plugin::aggregate; use warnings; use strict; use IkiWiki; +use HTML::Entities; +use HTML::Parser; +use HTML::Tagset; +use URI; my %feeds; my %guids; @@ -70,11 +74,11 @@ sub preprocess (@) { #{{{ $feed->{name}=$name; $feed->{sourcepage}=$params{page}; $feed->{url}=$params{url}; - my $dir=exists $params{dir} ? $params{dir} : IkiWiki::titlepage($params{name}); + my $dir=exists $params{dir} ? $params{dir} : $params{page}."/".IkiWiki::titlepage($params{name}); $dir=~s/^\/+//; ($dir)=$dir=~/$IkiWiki::config{wiki_file_regexp}/; $feed->{dir}=$dir; - $feed->{feedurl}=defined $params{feedurl} ? $params{feedurl} : $params{url}; + $feed->{feedurl}=defined $params{feedurl} ? $params{feedurl} : ""; $feed->{updateinterval}=defined $params{updateinterval} ? $params{updateinterval} * 60 : 15 * 60; $feed->{expireage}=defined $params{expireage} ? $params{expireage} : 0; $feed->{expirecount}=defined $params{expirecount} ? $params{expirecount} : 0; @@ -83,6 +87,7 @@ sub preprocess (@) { #{{{ $feed->{numposts}=0 unless defined $feed->{numposts}; $feed->{newposts}=0 unless defined $feed->{newposts}; $feed->{message}="new feed" unless defined $feed->{message}; + $feed->{tags}=[]; while (@_) { my $key=shift; my $value=shift; @@ -93,7 +98,7 @@ sub preprocess (@) { #{{{ return "{url}."\">".$feed->{name}.": ". "".$feed->{message}." (".$feed->{numposts}. - " stored posts; ".$feed->{newposts}." new)"; + " stored posts; ".$feed->{newposts}." new)
"; } # }}} sub delete (@) { #{{{ @@ -116,8 +121,9 @@ sub loadstate () { #{{{ my $data={}; foreach my $i (split(/ /, $_)) { my ($field, $val)=split(/=/, $i, 2); - if ($field eq "name" || $field eq "message") { - $data->{$field}=IkiWiki::pagetitle($val); + if ($field eq "name" || $field eq "feed" || + $field eq "guid" || $field eq "message") { + $data->{$field}=decode_entities($val, " \t\n"); } elsif ($field eq "tag") { push @{$data->{tags}}, $val; @@ -140,6 +146,8 @@ sub loadstate () { #{{{ } #}}} sub savestate () { #{{{ + eval q{use HTML::Entities}; + die $@ if $@; open (OUT, ">$IkiWiki::config{wikistatedir}/aggregate" || die "$IkiWiki::config{wikistatedir}/aggregate: $!"); foreach my $data (values %feeds, values %guids) { @@ -159,8 +167,9 @@ sub savestate () { #{{{ my @line; foreach my $field (keys %$data) { - if ($field eq "name" || $field eq "message") { - push @line, "$field=".IkiWiki::titlepage($data->{$field}); + if ($field eq "name" || $field eq "feed" || + $field eq "guid" || $field eq "message") { + push @line, "$field=".encode_entities($data->{$field}, " \t\n"); } elsif ($field eq "tags") { push @line, "tag=$_" foreach @{$data->{tags}}; @@ -180,38 +189,47 @@ sub aggregate () { #{{{ eval q{use HTML::Entities}; die $@ if $@; -FEED: foreach my $feed (values %feeds) { - next unless time - $feed->{lastupdate} >= $feed->{updateinterval}; + foreach my $feed (values %feeds) { + next unless $IkiWiki::config{rebuild} || + time - $feed->{lastupdate} >= $feed->{updateinterval}; $feed->{lastupdate}=time; $feed->{newposts}=0; $IkiWiki::forcerebuild{$feed->{sourcepage}}=1; IkiWiki::debug("checking feed ".$feed->{name}." ..."); - my @urls=XML::Feed->find_feeds($feed->{feedurl}); - if (! @urls) { - $feed->{message}="could not find feed at ".$feed->{feedurl}; - IkiWiki::debug($feed->{message}); - } - foreach my $url (@urls) { - my $f=XML::Feed->parse(URI->new($url)); - if (! $f) { - $feed->{message}=XML::Feed->errstr; + if (! length $feed->{feedurl}) { + my @urls=XML::Feed->find_feeds($feed->{url}); + if (! @urls) { + $feed->{message}="could not find feed at ".$feed->{feedurl}; IkiWiki::debug($feed->{message}); - next FEED; + next; } + $feed->{feedurl}=pop @urls; + } + my $f=eval{XML::Feed->parse(URI->new($feed->{feedurl}))}; + if ($@) { + $feed->{message}="feed crashed XML::Feed! $@"; + IkiWiki::debug($feed->{message}); + next; + } + if (! $f) { + $feed->{message}=XML::Feed->errstr; + IkiWiki::debug($feed->{message}); + next; + } - foreach my $entry ($f->entries) { - add_page( - feed => $feed, - title => defined $entry->title ? decode_entities($entry->title) : "untitled", - link => $entry->link, - content => $entry->content->body, - guid => defined $entry->id ? $entry->id : time."_".$feed->name, - ctime => $entry->issued ? ($entry->issued->epoch || time) : time, - ); - } + foreach my $entry ($f->entries) { + add_page( + feed => $feed, + title => defined $entry->title ? decode_entities($entry->title) : "untitled", + link => $entry->link, + content => $entry->content->body, + guid => defined $entry->id ? $entry->id : time."_".$feed->name, + ctime => $entry->issued ? ($entry->issued->epoch || time) : time, + ); } + $feed->{message}="processed ok"; } @@ -237,12 +255,16 @@ sub add_page (@) { #{{{ $feed->{newposts}++; # assign it an unused page - my $page=$feed->{dir}."/".IkiWiki::titlepage($params{title}); + my $page=IkiWiki::titlepage($params{title}); + # escape slashes and periods in title so it doesn't specify + # directory name or trigger ".." disallowing code. + $page=~s!([/.])!"__".ord($1)."__"!eg; + $page=$feed->{dir}."/".$page; + $page=lc($page); ($page)=$page=~/$IkiWiki::config{wiki_file_regexp}/; if (! defined $page || ! length $page) { $page=$feed->{dir}."/item"; } - $page=~s/\.\.//g; # avoid ".." directory tricks my $c=""; while (exists $IkiWiki::pagesources{$page.$c} || -e pagefile($page.$c)) { @@ -259,18 +281,17 @@ sub add_page (@) { #{{{ eval q{use Digest::MD5 'md5_hex'}; require Encode; my $digest=md5_hex(Encode::encode_utf8($params{content})); - return unless ! exists $guid->{md5} || $guid->{md5} ne $digest; + return unless ! exists $guid->{md5} || $guid->{md5} ne $digest || $IkiWiki::config{rebuild}; $guid->{md5}=$digest; # Create the page. my $template=IkiWiki::template("aggregatepost.tmpl", blind_cache => 1); - my $content=$params{content}; - $params{content}=~s/(?param(content => $params{content}); - $template->param(url => $feed->{url}); + $template->param(title => $params{title}) + if defined $params{title} && length($params{title}); + $template->param(content => htmlescape(htmlabs($params{content}, $feed->{feedurl}))); $template->param(name => $feed->{name}); - $template->param(link => $params{link}) if defined $params{link}; + $template->param(permalink => urlabs($params{link}, $feed->{feedurl})) + if defined $params{link}; if (ref $feed->{tags}) { $template->param(tags => [map { tag => $_ }, @{$feed->{tags}}]); } @@ -282,6 +303,58 @@ sub add_page (@) { #{{{ utime $mtime, $mtime, pagefile($guid->{page}) if defined $mtime; } #}}} +sub htmlescape ($) { #{{{ + # escape accidental wikilinks and preprocessor stuff + my $html=shift; + $html=~s/(?new_abs($url, $urlbase)->as_string; +} #}}} + +sub htmlabs ($$) { #{{{ + # Convert links in html from relative to absolute. + # Note that this is a heuristic, which is not specified by the rss + # spec and may not be right for all feeds. Also, see Debian + # bug #XXXX TODO: get bug. + my $html=shift; + my $urlbase=shift; + + my $ret=""; + my $p = HTML::Parser->new(api_version => 3); + $p->handler(default => sub { $ret.=join("", @_) }, "text"); + $p->handler(start => sub { + my ($tagname, $pos, $text) = @_; + if (ref $HTML::Tagset::linkElements{$tagname}) { + while (4 <= @$pos) { + # use attribute sets from right to left + # to avoid invalidating the offsets + # when replacing the values + my($k_offset, $k_len, $v_offset, $v_len) = + splice(@$pos, -4); + my $attrname = lc(substr($text, $k_offset, $k_len)); + next unless grep { $_ eq $attrname } @{$HTML::Tagset::linkElements{$tagname}}; + next unless $v_offset; # 0 v_offset means no value + my $v = substr($text, $v_offset, $v_len); + $v =~ s/^([\'\"])(.*)\1$/$2/; + my $new_v=urlabs($v, $urlbase); + $new_v =~ s/\"/"/g; # since we quote with "" + substr($text, $v_offset, $v_len) = qq("$new_v"); + } + } + $ret.=$text; + }, "tagname, tokenpos, text"); + $p->parse($html); + $p->eof; + + return $ret; +} #}}} + sub remove_feeds () { #{{{ my $page=shift;