X-Git-Url: http://git.vanrenterghem.biz/git.ikiwiki.info.git/blobdiff_plain/bc2671082bd69fd10091c4598f2002a672a4296d..edb69335f2ca5d42c62f6a4cce0b044da5ef88e8:/IkiWiki/Plugin/aggregate.pm diff --git a/IkiWiki/Plugin/aggregate.pm b/IkiWiki/Plugin/aggregate.pm index 3ec067cc1..adaa619ab 100644 --- a/IkiWiki/Plugin/aggregate.pm +++ b/IkiWiki/Plugin/aggregate.pm @@ -1,5 +1,5 @@ #!/usr/bin/perl -# Blog aggregation plugin. +# Feed aggregation plugin. package IkiWiki::Plugin::aggregate; use warnings; @@ -7,6 +7,7 @@ use strict; use IkiWiki 2.00; use HTML::Parser; use HTML::Tagset; +use HTML::Entities; use URI; use open qw{:utf8 :std}; @@ -15,65 +16,180 @@ my %guids; sub import { #{{{ hook(type => "getopt", id => "aggregate", call => \&getopt); + hook(type => "getsetup", id => "aggregate", call => \&getsetup); hook(type => "checkconfig", id => "aggregate", call => \&checkconfig); hook(type => "needsbuild", id => "aggregate", call => \&needsbuild); hook(type => "preprocess", id => "aggregate", call => \&preprocess); hook(type => "delete", id => "aggregate", call => \&delete); hook(type => "savestate", id => "aggregate", call => \&savestate); + hook(type => "htmlize", id => "_aggregated", call => \&htmlize); + if (exists $config{aggregate_webtrigger} && $config{aggregate_webtrigger}) { + hook(type => "cgi", id => "aggregate", call => \&cgi); + } } # }}} sub getopt () { #{{{ eval q{use Getopt::Long}; error($@) if $@; Getopt::Long::Configure('pass_through'); - GetOptions("aggregate" => \$config{aggregate}); + GetOptions( + "aggregate" => \$config{aggregate}, + "aggregateinternal!" => \$config{aggregateinternal}, + ); +} #}}} + +sub getsetup () { #{{{ + return + plugin => { + safe => 1, + rebuild => undef, + }, + aggregateinternal => { + type => "boolean", + example => 0, + description => "enable aggregation to internal pages?", + safe => 0, # enabling needs manual transition + rebuild => 0, + }, + aggregate_webtrigger => { + type => "boolean", + example => 0, + description => "allow aggregation to be triggered via the web?", + safe => 1, + rebuild => 0, + }, } #}}} sub checkconfig () { #{{{ if ($config{aggregate} && ! ($config{post_commit} && IkiWiki::commit_hook_enabled())) { - # See if any feeds need aggregation. - loadstate(); - my @feeds=needsaggregate(); - return unless @feeds; - if (! lockaggregate()) { - debug("an aggregation process is already running"); - return; - } - # force a later rebuild of source pages - $IkiWiki::forcerebuild{$_->{sourcepage}}=1 - foreach @feeds; - - # Fork a child process to handle the aggregation. - # The parent process will then handle building the - # result. This avoids messy code to clear state - # accumulated while aggregating. - defined(my $pid = fork) or error("Can't fork: $!"); - if (! $pid) { - IkiWiki::loadindex(); + launchaggregation(); + } +} #}}} - # Aggregation happens without the main wiki lock - # being held. This allows editing pages etc while - # aggregation is running. - aggregate(@feeds); - - IkiWiki::lockwiki; - # Merge changes, since aggregation state may have - # changed on disk while the aggregation was happening. - mergestate(); - expire(); - savestate(); - IkiWiki::unlockwiki; - exit 0; +sub cgi ($) { #{{{ + my $cgi=shift; + + if (defined $cgi->param('do') && + $cgi->param("do") eq "aggregate_webtrigger") { + $|=1; + print "Content-Type: text/plain\n\n"; + $config{cgi}=0; + $config{verbose}=1; + $config{syslog}=0; + print gettext("Aggregation triggered via web.")."\n\n"; + if (launchaggregation()) { + IkiWiki::lockwiki(); + IkiWiki::loadindex(); + require IkiWiki::Render; + IkiWiki::refresh(); + IkiWiki::saveindex(); } - waitpid($pid,0); - if ($?) { - error "aggregation failed with code $?"; + else { + print gettext("Nothing to do right now, all feeds are up-to-date!")."\n"; } + exit 0; + } +} #}}} - clearstate(); - unlockaggregate(); +sub launchaggregation () { #{{{ + # See if any feeds need aggregation. + loadstate(); + my @feeds=needsaggregate(); + return unless @feeds; + if (! lockaggregate()) { + debug("an aggregation process is already running"); + return; + } + # force a later rebuild of source pages + $IkiWiki::forcerebuild{$_->{sourcepage}}=1 + foreach @feeds; + + # Fork a child process to handle the aggregation. + # The parent process will then handle building the + # result. This avoids messy code to clear state + # accumulated while aggregating. + defined(my $pid = fork) or error("Can't fork: $!"); + if (! $pid) { + IkiWiki::loadindex(); + # Aggregation happens without the main wiki lock + # being held. This allows editing pages etc while + # aggregation is running. + aggregate(@feeds); + + IkiWiki::lockwiki; + # Merge changes, since aggregation state may have + # changed on disk while the aggregation was happening. + mergestate(); + expire(); + savestate(); + IkiWiki::unlockwiki; + exit 0; } + waitpid($pid,0); + if ($?) { + error "aggregation failed with code $?"; + } + + clearstate(); + unlockaggregate(); + + return 1; +} #}}} + +# Pages with extension _aggregated have plain html markup, pass through. +sub htmlize (@) { #{{{ + my %params=@_; + return $params{content}; +} #}}} + +# Used by ikiwiki-transition aggregateinternal. +sub migrate_to_internal { #{{{ + if (! lockaggregate()) { + error("an aggregation process is currently running"); + } + + IkiWiki::lockwiki(); + loadstate(); + $config{verbose}=1; + + foreach my $data (values %guids) { + next unless $data->{page}; + next if $data->{expired}; + + $config{aggregateinternal} = 0; + my $oldname = "$config{srcdir}/".htmlfn($data->{page}); + my $oldoutput = $config{destdir}."/".IkiWiki::htmlpage($data->{page}); + + $config{aggregateinternal} = 1; + my $newname = "$config{srcdir}/".htmlfn($data->{page}); + + debug "moving $oldname -> $newname"; + if (-e $newname) { + if (-e $oldname) { + error("$newname already exists"); + } + else { + debug("already renamed to $newname?"); + } + } + elsif (-e $oldname) { + rename($oldname, $newname) || error("$!"); + } + else { + debug("$oldname not found"); + } + if (-e $oldoutput) { + require IkiWiki::Render; + debug("removing output file $oldoutput"); + IkiWiki::prune($oldoutput); + } + } + + savestate(); + IkiWiki::unlockwiki; + + unlockaggregate(); } #}}} sub needsbuild (@) { #{{{ @@ -97,7 +213,7 @@ sub preprocess (@) { #{{{ foreach my $required (qw{name url}) { if (! exists $params{$required}) { - return "[[aggregate ".sprintf(gettext("missing %s parameter"), $required)."]]"; + error sprintf(gettext("missing %s parameter"), $required) } } @@ -112,7 +228,7 @@ sub preprocess (@) { #{{{ $feed->{name}=$name; $feed->{sourcepage}=$params{page}; $feed->{url}=$params{url}; - my $dir=exists $params{dir} ? $params{dir} : $params{page}."/".IkiWiki::titlepage($params{name}); + my $dir=exists $params{dir} ? $params{dir} : $params{page}."/".titlepage($params{name}); $dir=~s/^\/+//; ($dir)=$dir=~/$config{wiki_file_regexp}/; $feed->{dir}=$dir; @@ -120,6 +236,13 @@ sub preprocess (@) { #{{{ $feed->{updateinterval}=defined $params{updateinterval} ? $params{updateinterval} * 60 : 15 * 60; $feed->{expireage}=defined $params{expireage} ? $params{expireage} : 0; $feed->{expirecount}=defined $params{expirecount} ? $params{expirecount} : 0; + if (exists $params{template}) { + $params{template}=~s/[^-_a-zA-Z0-9]+//g; + } + else { + $params{template} = "aggregatepost" + } + $feed->{template}=$params{template} . ".tmpl"; delete $feed->{unseen}; $feed->{lastupdate}=0 unless defined $feed->{lastupdate}; $feed->{numposts}=0 unless defined $feed->{numposts}; @@ -205,8 +328,6 @@ sub loadstate () { #{{{ sub savestate () { #{{{ return unless $state_loaded; garbage_collect(); - eval q{use HTML::Entities}; - error($@) if $@; my $newfile="$config{wikistatedir}/aggregate.new"; my $cleanup = sub { unlink($newfile) }; open (OUT, ">$newfile") || error("open $newfile: $!", $cleanup); @@ -243,13 +364,13 @@ sub garbage_collect () { #{{{ foreach my $guid (values %guids) { # any guid whose feed is gone should be removed if (! exists $feeds{$guid->{feed}}) { - unlink pagefile($guid->{page}) + unlink "$config{srcdir}/".htmlfn($guid->{page}) if exists $guid->{page}; delete $guids{$guid->{guid}}; } # handle expired guids elsif ($guid->{expired} && exists $guid->{page}) { - unlink pagefile($guid->{page}); + unlink "$config{srcdir}/".htmlfn($guid->{page}); delete $guid->{page}; delete $guid->{md5}; } @@ -299,11 +420,11 @@ sub expire () { #{{{ next unless $feed->{expireage} || $feed->{expirecount}; my $count=0; my %seen; - foreach my $item (sort { $IkiWiki::pagectime{$b->{page}} <=> $IkiWiki::pagectime{$a->{page}} } - grep { exists $_->{page} && $_->{feed} eq $feed->{name} && $IkiWiki::pagectime{$_->{page}} } + foreach my $item (sort { ($IkiWiki::pagectime{$b->{page}} || 0) <=> ($IkiWiki::pagectime{$a->{page}} || 0) } + grep { exists $_->{page} && $_->{feed} eq $feed->{name} } values %guids) { if ($feed->{expireage}) { - my $days_old = (time - $IkiWiki::pagectime{$item->{page}}) / 60 / 60 / 24; + my $days_old = (time - ($IkiWiki::pagectime{$item->{page}} || 0)) / 60 / 60 / 24; if ($days_old > $feed->{expireage}) { debug(sprintf(gettext("expiring %s (%s days old)"), $item->{page}, int($days_old))); @@ -335,13 +456,11 @@ sub aggregate (@) { #{{{ error($@) if $@; eval q{use URI::Fetch}; error($@) if $@; - eval q{use HTML::Entities}; - error($@) if $@; foreach my $feed (@_) { $feed->{lastupdate}=time; $feed->{newposts}=0; - $feed->{message}=sprintf(gettext("processed ok at %s"), + $feed->{message}=sprintf(gettext("last checked %s"), displaytime($feed->{lastupdate})); $feed->{error}=0; @@ -377,15 +496,19 @@ sub aggregate (@) { #{{{ # that contains invalid UTF-8 sequences. Convert # feed to ascii to try to work around. $feed->{message}.=" ".sprintf(gettext("(invalid UTF-8 stripped from feed)")); - $content=Encode::decode_utf8($content); - $f=eval{XML::Feed->parse(\$content)}; + $f=eval { + $content=Encode::decode_utf8($content, 0); + XML::Feed->parse(\$content) + }; } if ($@) { # Another possibility is badly escaped entities. $feed->{message}.=" ".sprintf(gettext("(feed entities escaped)")); $content=~s/\&(?!amp)(\w+);/&$1;/g; - $content=Encode::decode_utf8($content); - $f=eval{XML::Feed->parse(\$content)}; + $f=eval { + $content=Encode::decode_utf8($content, 0); + XML::Feed->parse(\$content) + }; } if ($@) { $feed->{message}=gettext("feed crashed XML::Feed!")." ($@)"; @@ -401,14 +524,21 @@ sub aggregate (@) { #{{{ } foreach my $entry ($f->entries) { + my $c=$entry->content; + # atom feeds may have no content, only a summary + if (! defined $c && ref $entry->summary) { + $c=$entry->summary; + } + add_page( feed => $feed, copyright => $f->copyright, title => defined $entry->title ? decode_entities($entry->title) : "untitled", link => $entry->link, - content => defined $entry->content->body ? $entry->content->body : "", - guid => defined $entry->id ? $entry->id : time."_".$feed->name, + content => (defined $c && defined $c->body) ? $c->body : "", + guid => defined $entry->id ? $entry->id : time."_".$feed->{name}, ctime => $entry->issued ? ($entry->issued->epoch || time) : time, + base => (defined $c && $c->can("base")) ? $c->base : undef, ); } } @@ -434,7 +564,7 @@ sub add_page (@) { #{{{ $feed->{newposts}++; # assign it an unused page - my $page=IkiWiki::titlepage($params{title}); + my $page=titlepage($params{title}); # escape slashes and periods in title so it doesn't specify # directory name or trigger ".." disallowing code. $page=~s!([/.])!"__".ord($1)."__"!eg; @@ -445,7 +575,7 @@ sub add_page (@) { #{{{ } my $c=""; while (exists $IkiWiki::pagecase{lc $page.$c} || - -e pagefile($page.$c)) { + -e "$config{srcdir}/".htmlfn($page.$c)) { $c++ } @@ -456,7 +586,7 @@ sub add_page (@) { #{{{ $c=""; $page=$feed->{dir}."/item"; while (exists $IkiWiki::pagecase{lc $page.$c} || - -e pagefile($page.$c)) { + -e "$config{srcdir}/".htmlfn($page.$c)) { $c++ } } @@ -477,10 +607,11 @@ sub add_page (@) { #{{{ $guid->{md5}=$digest; # Create the page. - my $template=template("aggregatepost.tmpl", blind_cache => 1); + my $template=template($feed->{template}, blind_cache => 1); $template->param(title => $params{title}) if defined $params{title} && length($params{title}); - $template->param(content => htmlescape(htmlabs($params{content}, $feed->{feedurl}))); + $template->param(content => wikiescape(htmlabs($params{content}, + defined $params{base} ? $params{base} : $feed->{feedurl}))); $template->param(name => $feed->{name}); $template->param(url => $feed->{url}); $template->param(copyright => $params{copyright}) @@ -493,17 +624,22 @@ sub add_page (@) { #{{{ writefile(htmlfn($guid->{page}), $config{srcdir}, $template->output); - # Set the mtime, this lets the build process get the right creation - # time on record for the new page. - utime $mtime, $mtime, pagefile($guid->{page}) - if defined $mtime && $mtime <= time; + if (defined $mtime && $mtime <= time) { + # Set the mtime, this lets the build process get the right + # creation time on record for the new page. + utime $mtime, $mtime, "$config{srcdir}/".htmlfn($guid->{page}); + # Store it in pagectime for expiry code to use also. + $IkiWiki::pagectime{$guid->{page}}=$mtime; + } + else { + # Dummy value for expiry code. + $IkiWiki::pagectime{$guid->{page}}=time; + } } #}}} -sub htmlescape ($) { #{{{ +sub wikiescape ($) { #{{{ # escape accidental wikilinks and preprocessor stuff - my $html=shift; - $html=~s/(?