2 # xapian-omega search engine plugin
3 package IkiWiki::Plugin::search;
10 hook(type => "getsetup", id => "search", call => \&getsetup);
11 hook(type => "checkconfig", id => "search", call => \&checkconfig);
12 hook(type => "pagetemplate", id => "search", call => \&pagetemplate);
13 hook(type => "indexhtml", id => "search", call => \&indexhtml);
14 hook(type => "delete", id => "search", call => \&delete);
15 hook(type => "cgi", id => "search", call => \&cgi);
16 hook(type => "disable", id => "search", call => \&disable);
17 hook(type => "needsbuild", id => "search", call => \&needsbuild);
19 eval q{ use Search::Xapian }; # load early to work around #622591
31 example => "/usr/lib/cgi-bin/omega/omega",
32 description => "path to the omega cgi program",
33 safe => 0, # external program
39 foreach my $required (qw(url cgiurl)) {
40 if (! length $config{$required}) {
41 error(sprintf(gettext("Must specify %s when using the %s plugin"), $required, 'search'));
45 if (! defined $config{omega_cgi}) {
46 $config{omega_cgi}="/usr/lib/cgi-bin/omega/omega";
49 # This is a mass dependency, so if the search form template
50 # changes, every page is rebuilt.
51 add_depends("", "templates/searchform.tmpl");
55 sub pagetemplate (@) {
57 my $page=$params{page};
58 my $template=$params{template};
60 # Add search box to page header.
61 if ($template->query(name => "searchform")) {
62 if (! defined $form) {
63 my $searchform = template("searchform.tmpl", blind_cache => 1);
64 $searchform->param(searchaction => IkiWiki::cgiurl());
65 $searchform->param(html5 => $config{html5});
66 $form=$searchform->output;
69 $template->param(searchform => $form);
80 # A unique pageterm is used to identify the document for a page.
81 my $pageterm=pageterm($params{page});
82 return unless defined $pageterm;
85 my $doc=Search::Xapian::Document->new();
86 my $caption=pagetitle($params{page});
88 if (exists $pagestate{$params{page}}{meta} &&
89 exists $pagestate{$params{page}}{meta}{title}) {
90 $title=$pagestate{$params{page}}{meta}{title};
96 # Remove html from text to be indexed.
97 if (! defined $scrubber) {
98 eval q{use HTML::Scrubber};
100 $scrubber=HTML::Scrubber->new(allow => []);
103 my $toindex = defined $scrubber ? $scrubber->scrub($params{content}) : $params{content};
105 # Take 512 characters for a sample, then extend it out
106 # if it stopped in the middle of a word.
108 my ($sample)=substr($toindex, 0, $size);
109 if (length($sample) == $size) {
110 my $max=length($toindex);
112 while ($size < $max &&
113 ($next=substr($toindex, $size++, 1)) !~ /\s/) {
119 my $url=urlto($params{destpage}, "");
120 if (defined $pagestate{$params{page}}{meta}{permalink}) {
121 $url=$pagestate{$params{page}}{meta}{permalink}
125 # Decode html entities in it, since omega re-encodes them.
126 eval q{use HTML::Entities};
130 "sample=".decode_entities($sample)."\n".
131 "caption=".decode_entities($caption)."\n".
132 "modtime=$IkiWiki::pagemtime{$params{page}}\n".
133 "size=".length($params{content})."\n"
136 # Index document and add terms for other metadata.
137 my $tg = Search::Xapian::TermGenerator->new();
139 my $langcode=$ENV{LANG} || "en";
142 # This whitelist is here to work around a xapian bug (#486138)
143 my @whitelist=qw{da de en es fi fr hu it no pt ru ro sv tr};
145 if (grep { $_ eq $langcode } @whitelist) {
146 $stemmer=Search::Xapian::Stem->new($langcode);
149 $stemmer=Search::Xapian::Stem->new("english");
152 $tg->set_stemmer($stemmer);
153 $tg->set_document($doc);
154 $tg->index_text($params{page}, 2);
155 $tg->index_text($caption, 2);
156 $tg->index_text($title, 2) if $title ne $caption;
157 $tg->index_text($toindex);
158 $tg->index_text(lc($title), 1, "S"); # for title:foo
159 foreach my $link (@{$links{$params{page}}}) {
160 $tg->index_text(lc($link), 1, "XLINK"); # for link:bar
163 $doc->add_term($pageterm);
164 $db->replace_document_by_term($pageterm, $doc);
169 foreach my $page (@_) {
170 my $pageterm=pageterm(pagename($page));
171 $db->delete_document_by_term($pageterm) if defined $pageterm;
178 if (defined $cgi->param('P')) {
179 # only works for GET requests
180 chdir("$config{wikistatedir}/xapian") || error("chdir: $!");
181 $ENV{OMEGA_CONFIG_FILE}="./omega.conf";
182 $ENV{CGIURL}=IkiWiki::cgiurl();
183 IkiWiki::loadindex();
184 $ENV{HELPLINK}=htmllink("", "", "ikiwiki/searching",
185 noimageinline => 1, linktext => "Help");
186 exec($config{omega_cgi}) || error("$config{omega_cgi} failed: $!");
193 # 240 is the number used by omindex to decide when to hash an
194 # overlong term. This does not use a compatible hash method though.
196 if (length encode_utf8($page) > 240) {
197 eval q{use Digest::SHA};
199 debug("search: ".sprintf(gettext("need Digest::SHA to index %s"), $page)) if $@;
203 # Note no colon, therefore it's guaranteed to not overlap
204 # with a page with the same name as the hash..
205 return "U".lc(Digest::SHA::sha1_hex($page));
217 use Search::Xapian::WritableDatabase;
220 $db=Search::Xapian::WritableDatabase->new($config{wikistatedir}."/xapian/default",
221 Search::Xapian::DB_CREATE_OR_OPEN());
229 if (! $setup and (! -e $config{wikistatedir}."/xapian" || $config{rebuild})) {
230 writefile("omega.conf", $config{wikistatedir}."/xapian",
232 "template_dir ./templates\n");
242 $_ eq "templates/page.tmpl" ||
243 $_ eq "templates/searchquery.tmpl"
250 # Avoid omega interpreting anything in the cgitemplate
251 # as an omegascript command.
252 eval q{use IkiWiki::CGI};
253 my $template=IkiWiki::cgitemplate(undef, gettext("search"), "\0",
254 searchform => "", # avoid showing the small search form
256 eval q{use HTML::Entities};
258 $template=encode_entities($template, '\$');
260 my $querytemplate=readfile(IkiWiki::template_file("searchquery.tmpl"));
261 $template=~s/\0/$querytemplate/;
262 writefile("query", $config{wikistatedir}."/xapian/templates",
267 if (-d $config{wikistatedir}."/xapian") {
268 system("rm", "-rf", $config{wikistatedir}."/xapian");