From: smcv@ <smcv@>
Date: Thu, 18 Jun 2009 14:54:53 +0000 (+0100)
Subject: Optimize the dependencies list
X-Git-Url: http://git.vanrenterghem.biz/git.ikiwiki.info.git/commitdiff_plain/b72a83e973e7cbb15d57faaaf14f18b11d61102e?ds=sidebyside

Optimize the dependencies list

On a large wiki you can spend a lot of time reading through large lists
of dependencies to see whether files need to be rebuilt (album, with its
one-page-per-photo arrangement, suffers particularly badly from this).

The dependency list is currently a single pagespec, but it's not used like
a normal pagespec - in practice, it's a list of pagespecs joined with the
"or" operator.

Accordingly, change it to be stored as a list of pagespecs. On a wiki
with many tagged photo albums, this reduces the time to refresh after
`touch tags/*.mdwn` from about 31 to 25 seconds.

Getting the benefit of this change on an existing wiki requires a rebuild.
---

diff --git a/IkiWiki.pm b/IkiWiki.pm
index a11b330f2..e88613307 100644
--- a/IkiWiki.pm
+++ b/IkiWiki.pm
@@ -1499,8 +1499,11 @@ sub loadindex () {
 				$links{$page}=$d->{links};
 				$oldlinks{$page}=[@{$d->{links}}];
 			}
-			if (exists $d->{depends}) {
-				$depends{$page}=$d->{depends};
+			if (exists $d->{dependslist}) {
+				$depends{$page}=$d->{dependslist};
+			}
+			elsif (exists $d->{depends}) {
+				$depends{$page}=[$d->{depends}];
 			}
 			if (exists $d->{state}) {
 				$pagestate{$page}=$d->{state};
@@ -1546,7 +1549,8 @@ sub saveindex () {
 		};
 
 		if (exists $depends{$page}) {
-			$index{page}{$src}{depends} = $depends{$page};
+			$index{page}{$src}{depends} = join(" or ", @{$depends{$page}});
+			$index{page}{$src}{dependslist} = $depends{$page};
 		}
 
 		if (exists $pagestate{$page}) {
@@ -1716,14 +1720,17 @@ sub rcs_receive () {
 sub add_depends ($$) {
 	my $page=shift;
 	my $pagespec=shift;
-	
+
 	return unless pagespec_valid($pagespec);
 
 	if (! exists $depends{$page}) {
-		$depends{$page}=$pagespec;
+		$depends{$page}=[$pagespec];
 	}
 	else {
-		$depends{$page}=pagespec_merge($depends{$page}, $pagespec);
+		foreach my $p (@{$depends{$page}}) {
+			return 1 if $p eq $pagespec;
+		}
+		push @{$depends{$page}}, $pagespec;
 	}
 
 	return 1;
diff --git a/IkiWiki/Render.pm b/IkiWiki/Render.pm
index bed9cb777..1247e892b 100644
--- a/IkiWiki/Render.pm
+++ b/IkiWiki/Render.pm
@@ -444,20 +444,22 @@ sub refresh () {
 		my @changed=(keys %rendered, @del);
 
 		# rebuild dependant pages
-		foreach my $f (@$files) {
+		F: foreach my $f (@$files) {
 			next if $rendered{$f};
 			my $p=pagename($f);
 			if (exists $depends{$p}) {
-				# only consider internal files
-				# if the page explicitly depends on such files
-				foreach my $file (@changed, $depends{$p}=~/internal\(/ ? @internal : ()) {
-					next if $f eq $file;
-					my $page=pagename($file);
-					if (pagespec_match($page, $depends{$p}, location => $p)) {
-						debug(sprintf(gettext("building %s, which depends on %s"), $f, $page));
-						render($f);
-						$rendered{$f}=1;
-						last;
+				foreach my $d (@{$depends{$p}}) {
+					# only consider internal files
+					# if the page explicitly depends on such files
+					foreach my $file (@changed, $d=~/internal\(/ ? @internal : ()) {
+						next if $f eq $file;
+						my $page=pagename($file);
+						if (pagespec_match($page, $d, location => $p)) {
+							debug(sprintf(gettext("building %s, which depends on %s"), $f, $page));
+							render($f);
+							$rendered{$f}=1;
+							next F;
+						}
 					}
 				}
 			}
diff --git a/ikiwiki-transition b/ikiwiki-transition
index 398b1a3c8..60cea3d54 100755
--- a/ikiwiki-transition
+++ b/ikiwiki-transition
@@ -299,7 +299,7 @@ sub oldloadindex {
 			$pagemtime{$page}=$items{mtime}[0];
 			$oldlinks{$page}=[@{$items{link}}];
 			$links{$page}=[@{$items{link}}];
-			$depends{$page}=$items{depends}[0] if exists $items{depends};
+			$depends{$page}=[$items{depends}[0]] if exists $items{depends};
 			$destsources{$_}=$page foreach @{$items{dest}};
 			$renderedfiles{$page}=[@{$items{dest}}];
 			$pagecase{lc $page}=$page;
diff --git a/t/index.t b/t/index.t
index e79609902..107dac9d0 100755
--- a/t/index.t
+++ b/t/index.t
@@ -32,9 +32,9 @@ $renderedfiles{"bar.png"}=["bar.png"];
 $links{"Foo"}=["bar.png"];
 $links{"bar"}=["Foo", "new-page"];
 $links{"bar.png"}=[];
-$depends{"Foo"}="";
-$depends{"bar"}="foo*";
-$depends{"bar.png"}="";
+$depends{"Foo"}=[];
+$depends{"bar"}=["foo*"];
+$depends{"bar.png"}=[];
 $pagestate{"bar"}{meta}{title}="a page about bar";
 $pagestate{"bar"}{meta}{moo}="mooooo";
 # only loaded plugins save state, so this should not be saved out
@@ -80,9 +80,9 @@ is_deeply(\%links, {
 	"bar.png" => [],
 }, "%links loaded correctly");
 is_deeply(\%depends, {
-	Foo => "",
-	bar => "foo*",
-	"bar.png" => "",
+	Foo => [],
+	bar => ["foo*"],
+	"bar.png" => [],
 }, "%depends loaded correctly");
 is_deeply(\%pagestate, {
 	bar => {