binmode($in) if ($binary);
return \*$in if $wantfd;
my $ret=<$in>;
+ # check for invalid utf-8, and toss it back to avoid crashes
+ if (! utf8::valid($ret)) {
+ $ret=encode_utf8($ret);
+ }
close $in || error("failed to read $file: $!");
return $ret;
} #}}}
my $wikilock;
-sub lockwiki (;$) { #{{{
- my $wait=@_ ? shift : 1;
+sub lockwiki () { #{{{
# Take an exclusive lock on the wiki to prevent multiple concurrent
# run issues. The lock will be dropped on program exit.
if (! -d $config{wikistatedir}) {
}
open($wikilock, '>', "$config{wikistatedir}/lockfile") ||
error ("cannot write to $config{wikistatedir}/lockfile: $!");
- if (! flock($wikilock, 2 | 4)) { # LOCK_EX | LOCK_NB
- if ($wait) {
- debug("wiki seems to be locked, waiting for lock");
- my $wait=600; # arbitrary, but don't hang forever to
- # prevent process pileup
- for (1..$wait) {
- return if flock($wikilock, 2 | 4);
- sleep 1;
- }
- error("wiki is locked; waited $wait seconds without lock being freed (possible stuck process or stale lock?)");
- }
- else {
- return 0;
- }
+ if (! flock($wikilock, 2)) { # LOCK_EX
+ error("failed to get lock");
}
return 1;
} #}}}
sub unlockwiki () { #{{{
+ POSIX::close($ENV{IKIWIKI_CGILOCK_FD}) if exists $ENV{IKIWIKI_CGILOCK_FD};
return close($wikilock) if $wikilock;
return;
} #}}}
rev => $info->{"revno"},
user => $user,
committype => "bzr",
- when => time - str2time($info->{"timestamp"}),
+ when => str2time($info->{"timestamp"}),
message => [@message],
pages => [@pages],
};
}
my $check_commit_hook="";
+ my $pre_exec="";
if ($config{post_commit}) {
# Optimise checking !commit_hook_enabled() ,
# so that ikiwiki does not have to be started if the
# the benefit of this optimisation.
$check_commit_hook=<<"EOF";
{
- int fd=open("$config{wikistatedir}/commitlock", O_CREAT | O_RDWR);
+ int fd=open("$config{wikistatedir}/commitlock", O_CREAT | O_RDWR, 0666);
if (fd != -1) {
if (flock(fd, LOCK_SH | LOCK_NB) != 0)
exit(0);
}
EOF
}
+ elsif ($config{cgi}) {
+ # Avoid more than one ikiwiki cgi running at a time by
+ # taking a cgi lock. Since ikiwiki uses several MB of
+ # memory, a pile up of processes could cause thrashing
+ # otherwise. The fd of the lock is stored in
+ # IKIWIKI_CGILOCK_FD so unlockwiki can close it.
+ $pre_exec=<<"EOF";
+ {
+ int fd=open("$config{wikistatedir}/cgilock", O_CREAT | O_RDWR, 0666);
+ if (fd != -1 && flock(fd, LOCK_EX) == 0) {
+ char *fd_s;
+ asprintf(&fd_s, "%i", fd);
+ setenv("IKIWIKI_CGILOCK_FD", fd_s, 1);
+ }
+ }
+EOF
+ }
$Data::Dumper::Indent=0; # no newlines
my $configstring=Data::Dumper->Dump([\%config], ['*config']);
exit(1);
}
+$pre_exec
execl("$this", "$this", NULL);
perror("exec $this");
exit(1);
-ikiwiki (2.69) UNRELEASED; urgency=low
+ikiwiki (2.70) unstable; urgency=low
+ * Avoid crash on malformed utf-8 discovered by intrigeri.
+
+ -- Joey Hess <joeyh@debian.org> Wed, 12 Nov 2008 17:45:58 -0500
+
+ikiwiki (2.69) unstable; urgency=low
+
+ * Avoid multiple ikiwiki cgi processes piling up, eating all memory,
+ and thrashing, by making the cgi wrapper wait on a cgilock.
+ If you had to set apache's MaxClients low to avoid ikiwiki thrashing your
+ server, you can now turn it up to a high value.
+ * Stop busy-waiting in lockwiki, as this could delay ikiwiki from waking up
+ for up to one second. The bailout code is no longer needed after above
+ change.
+ * Remove support for unused optional wait parameter from lockwiki.
* aggregate: Try to query XML::Feed for the base url when derelevatising
links. Since this needs the just released XML::Feed 0.3, as well
as a not yet released XML::RSS, it will fall back to the old method
was earlier added to edit links.
* tag: Normalize tagbase so leading/trailing slashes in it don't break
things.
+ * bzr: Fix dates for recentchanges.
- -- Joey Hess <joeyh@debian.org> Thu, 06 Nov 2008 16:01:00 -0500
+ -- Joey Hess <joeyh@debian.org> Tue, 11 Nov 2008 20:35:55 -0500
ikiwiki (2.68) unstable; urgency=low
` when => str2time($info->{"timestamp"}),`
fixed this for me.
+
+> Weird, I wonder why it was written to return an absolute time like that
+> in the first place? Can't have ever been right. Fixed, thanks. --[[Joey]]
+> [[done]]
> - Using <code>inline</code> would avoid the redefinition + code duplication.
> - A few plugins would need to be upgraded.
> - It may be necessary to adapt the testsuite in `t/pagetitle.t`, as well.
-
+>
> --[[intrigeri]]
+>
+>> It was actually more complicated than expected. A working prototype is
+>> now in my `meta` branch, see my userpage for the up-to-date url.
+>> Thus tagging [[patch]]. --[[intrigeri]]
+++ /dev/null
-ikiwiki 2.64 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
- * Avoid uninitialised value when --dumpsetup is used and no srcdir/destdir
- specified.
- * ddate: Stop clobbering timeformat when not enabled.
- * progress: New plugin to generate progress bars (willu)
- * Add allow\_symlinks\_before\_srcdir to config so websetup doesn't eat it.
- * img: Support sizes like 200x. Closes: #[475149](http://bugs.debian.org/475149)
- * goodstuff: Remove otl plugin from the bundle since it needs a significant
- external dependency and is not commonly used. If you use otl, make sure
- you explicitly enable it now.
- * goodstuff: Add more, progress, and table plugins to the bundle.
- * Improve error message if external plugin fails to load. Closes: #[498458](http://bugs.debian.org/498458)
- * Directive documentation broken out of the plugin documentation and into
- pages suitable to be used as an underlay. Thanks to Willu for doing most
- of the tedious work.
- * Move the directive documentation into its own underlay, separate from
- basewiki, since it's sorta large compared to the rest of basewiki.
- * listdirectives: Enable use of the directives underlay.
- * Removed the obsolete blog page from the basewiki. ikiwiki/blog still
- remains, but is now deprecated too.
- * Removed old redirecton pages from basewiki (helponformatting,
- markdown, openid, pagespec, preprocessordirective, subpage, wikilink).
- * inline: Treat rootpage as a link, so that it can refer to a subpage
- without hardcoding the path."""]]
\ No newline at end of file
+++ /dev/null
-ikiwiki 2.65 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
- * aggregate: Expire excess or old items on the same pass that adds them,
- not only on subsequent passes.
- * editdiff: Broken since 2.62 due to wrong syntax, now fixed.
- * aggregate: Support atom feeds with only a summary element, and no content
- elements.
- * progress: Display an error if the progress cannot be parsed, and allow
- the percent parameter to only optionally end with "%".
- * Fix reversion in use of ikiwiki -verbose -setup with a setup file that
- enables syslog. Setup output is once again output to stdout in this
- case.
- * edittemplate: Default new page file type to the same type as the template.
- (willu)
- * edittemplate: Add "silent" parameter. (Willu)
- * edittemplate: Link to template, to allow creating it. (Willu)
- * editpage: Add a missing check that the page name contains only legal
- characters, in addition to the existing check for pruned filenames.
- * Print a debug message if a page has multiple source files.
- * Add keepextension parameter to htmlize hook. (Willu)
- * rename, remove: Don't rely on a form parameter to tell whether the page
- should be treated as an attachment.
- * rename: Add support for moving SubPages of a page when renaming it.
- (Sponsored by The TOVA Company.)
- * rename: Hide type field from rename form when renaming attachments."""]]
\ No newline at end of file
--- /dev/null
+ikiwiki 2.69 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+ * Avoid multiple ikiwiki cgi processes piling up, eating all memory,
+ and thrashing, by making the cgi wrapper wait on a cgilock.
+ If you had to set apache's MaxClients low to avoid ikiwiki thrashing your
+ server, you can now turn it up to a high value.
+ * Stop busy-waiting in lockwiki, as this could delay ikiwiki from waking up
+ for up to one second. The bailout code is no longer needed after above
+ change.
+ * Remove support for unused optional wait parameter from lockwiki.
+ * aggregate: Try to query XML::Feed for the base url when derelevatising
+ links. Since this needs the just released XML::Feed 0.3, as well
+ as a not yet released XML::RSS, it will fall back to the old method
+ if no xml:base info is available.
+ * meta: Plugin is now enabled by default since the basewiki uses it.
+ * txt: Do not encode quotes when filtering the txt, as that broke
+ later parsing of any directives on the page.
+ * Fix the link() pagespec to match links that are internally recorded as
+ absolute.
+ * Add rel=nofollow to recentchanges\_links for the same (weak) reasons it
+ was earlier added to edit links.
+ * tag: Normalize tagbase so leading/trailing slashes in it don't break
+ things.
+ * bzr: Fix dates for recentchanges."""]]
\ No newline at end of file
--- /dev/null
+ikiwiki 2.70 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+ * Avoid crash on malformed utf-8 discovered by intrigeri."""]]
\ No newline at end of file
>>> When a new language is added to `po_slave_languages`, a rebuild is
>>> triggered, and all missing PO files are created and checked into
>>> VCS. An unpriviledged wiki user can not add a new language to
->>> `po_slave_languages`, though. One could thing of adding the needed
+>>> `po_slave_languages`, though. One could think of adding the needed
>>> interface to translate a page into a yet-unsupported slave
>>> language, and this would automagically add this new language to
>>> `po_slave_languages`. It would probably be useful in some
>>>> I guess that if the template modification is made, it will list those
>>>> languages on the page, and if a translation to a language is missing,
>>>> the link will allow creating it?
+>>>>
+>>>>> Any translation page always exist for every supported slave
+>>>>> language, even if no string at all have been translated yet.
+>>>>> This implies the po plugin is especially friendly to people who
+>>>>> prefer reading in their native language if available, but don't
+>>>>> mind reading in English else.
+>>>>>
+>>>>> While I'm at it, there is a remaining issue that needs to be
+>>>>> sorted out: how painful it could be for non-English speakers
+>>>>> (assuming the master language is English) to be perfectly able
+>>>>> to navigate between translation pages supposed to be written in
+>>>>> their own language, when their translation level is most
+>>>>> often low.
+>>>>>
+>>>>> (It is currently easy to display this status on the translation
+>>>>> page itself, but then it's too late, and how frustrating to load
+>>>>> a page just to realize it's actually not translated enough for
+>>>>> you. The "other languages" loop also allows displaying this
+>>>>> information, but it is generally not the primary
+>>>>> navigation tool.)
+>>>>>
+>>>>> IMHO, this is actually a social problem (i.e. it's no use adding
+>>>>> a language to the supported slave ones if you don't have the
+>>>>> manpower to actually do the translations), that can't be fully
+>>>>> solved by technical solutions, but I can think of some hacks
+>>>>> that would limit the negative impact: a given translation's
+>>>>> status (currently = percent translated) could be displayed next
+>>>>> to the link that leads to it; a color code could as well be used
+>>>>> ("just" a matter of adding a CSS id or class to the links,
+>>>>> depending on this variable). As there is already work to be done
+>>>>> to have the links text generation more customizable through
+>>>>> plugins, I could do both at the same time if we consider this
+>>>>> matter to be important enough. --[[intrigeri]]
>> FWIW, I'm tracking your po branch in ikiwiki master git in the po
>> branch. One thing I'd like to try in there is setting up a translated
>>>>> Joey, please have a look at my branch, your help would be really
>>>>> welcome for the security research, as I'm almost done with what
>>>>> I am able to do myself in this area. --[[intrigeri]]
+>>>>>>
+>>>>>> I came up with a patch for the WrapI18N issue --[[Joey]]
I recommend upgrading to 2.48 immediatly if your wiki allows both password
and openid logins.
+
+## Malformed UTF-8 DOS
+
+Feeding ikiwiki page sources containing certian forms of malformed UTF-8
+can cause it to crash. This can potentially be used for a denial of service
+attack.
+
+intrigeri discovered this problem on 12 Nov 2008 and a patch put in place
+later that day, in version 2.70. The fix was backported to testing as version
+2.53.2, and to stable as version 1.33.7.
I've not written actual utilities to do this yet because I've only needed
to do it rarely, and the data I've wanted has been different each time.
--[[Joey]]
+
+## the session database
+
+`.ikiwiki/sessions.db` is the session database. See the [[cpan CGI::Session]]
+documentation for more details.
+
+## lockfiles
+
+In case you're curious, here's what the various lock files do.
+
+* `.ikiwiki/lockfile` is the master ikiwiki lock file. Ikiwiki takes this
+ lock before reading/writing state.
+* `.ikiwiki/commitlock` is locked as a semophore, to disable the commit hook
+ from doing anything.
+* `.ikiwiki/cgilock` is locked by the cgi wrapper, to ensure that only
+ one ikiwiki process is run at a time to handle cgi requests.
+
+## plugin state files
+
+Some plugins create other files to store their state.
+
+* `.ikiwiki/aggregate` is a plain text database used by the aggregate plugin
+ to record feeds and known posts.
+* `.ikiwiki/xapian/` is created by the search plugin, and contains xapian-omega
+ configuration and the xapian database.
--- /dev/null
+Problem: Suppose a server has 256 mb ram. Each ikiwiki process needs about
+15 mb, before it's loaded the index. (And maybe 25 after, but only one such
+process runs at any time). That allows for about 16 ikiwiki processes to
+run concurrently on a server, before it starts to swap. Of course, anything
+else that runs on the server and eats memory will affect this.
+
+One could just set `MaxClients 16` in the apache config, but then it's also
+limited to 16 clients serving static pages, which is silly. Also, 16 is
+optimistic -- 8 might be a saner choice. And then, what if something on the
+server decides to eat a lot of memory? Ikiwiki can again overflow memory
+and thrash.
+
+It occurred to me that the ikiwiki cgi wrapper could instead do locking of
+its own (say of `.ikiwiki/cgilock`). The wrapper only needs a few kb to
+run, and it starts *fast*. So hundreds could be running waiting for a lock
+with no ill effects. Crank `MaxClients` up to 256? No problem..
+
+And there's no real reason to allow more than one ikiwiki cgi to run at a
+time. Since almost all uses of the CGI lock the index, only one can really
+be doing anything at a time. --[[Joey]]
+
+[[done]]
--- /dev/null
+As documented in [[plugins/write]], the current `renamepage` hook is
+heavily oriented towards updating links in pages' content: it is run
+once per page linking to the renamed page.
+
+That's fine, but it can't be used to trigger more general actions on
+page rename. E.g. it won't be run at all if the page being renamed is
+an orphan one.
+
+This is a real issue for the [[plugins/contrib/po]] development: what
+I'm about to achieve is:
+
+- when a master page is renamed, the plugin takes notice of it (using
+ the `rename` hook), and later renames the translation pages
+ accordingly (in the `change` hook)
+- when a master page is deleted, the plugin deletes its translations
+ (using the `delete` hook)
+
+With the current `renamepage` hook behavior, combining these two goals
+has an annoying drawback: a plugin can't notice an orphan master page
+has been renamed, so instead of renaming (and preserving) its
+translations, it considers the oldpage as deleted, and deletes its
+translations. Game over.
+
+It may seem like a corner case, but I want to be very careful when
+deleting files automatically in `srcdir`, which is not always under
+version control.
+
+As an sad workaround, I can still disable any deletion in `srcdir`
+when it is not under version control. But I think ikiwiki deserves
+a global `renamepage` hook that would be run once per rename
+operation.
+
+My proposal is thus:
+
+- keep the documented `renamepage` hook as it is
+- use something inspired by the trick `preprocess` uses: when `hook`
+ is passed an optional "global" parameter, set to a true value, the
+ declared `renamepage` hook is run once per rename operation, and is
+ passed named parameters: `src`, `srcfile`, `dest` and `destfile`.
+
+I'm of course volunteering to implement this, or anything related that
+would solve my problem. Hmmm? --[[intrigeri]]
msgstr ""
"Project-Id-Version: PACKAGE VERSION\n"
"Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2008-11-07 12:23-0500\n"
+"POT-Creation-Date: 2008-11-11 20:48-0500\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language-Team: LANGUAGE <LL@li.org>\n"
#. translators: The first parameter is a filename, and the second is
#. translators: a (probably not translated) error message.
-#: ../IkiWiki/Wrapper.pm:79
+#: ../IkiWiki/Wrapper.pm:97
#, perl-format
msgid "failed to write %s: %s"
msgstr ""
#. translators: The parameter is a C filename.
-#: ../IkiWiki/Wrapper.pm:135
+#: ../IkiWiki/Wrapper.pm:154
#, perl-format
msgid "failed to compile %s"
msgstr ""
#. translators: The parameter is a filename.
-#: ../IkiWiki/Wrapper.pm:155
+#: ../IkiWiki/Wrapper.pm:174
#, perl-format
msgid "successfully generated %s"
msgstr ""
msgid "preprocessing loop detected on %s at depth %i"
msgstr ""
-#: ../IkiWiki.pm:1685
+#: ../IkiWiki.pm:1673
msgid "yes"
msgstr ""