git: Add test coverage for reverting attachments

[git.ikiwiki.info.git] / IkiWiki / Plugin / amazon_s3.pm

diff --git a/IkiWiki/Plugin/amazon_s3.pm b/IkiWiki/Plugin/amazon_s3.pm
index cc15fbdb1bd2a967ed7b2aca4e4979b9fd81ffdb..a9da6bf12b69dc62f2ee05f22aaaef2c88fe7dde 100644 (file)
--- a/IkiWiki/Plugin/amazon_s3.pm
+++ b/IkiWiki/Plugin/amazon_s3.pm
@@ -4,7 +4,7 @@ package IkiWiki::Plugin::amazon_s3;
 use warnings;
 no warnings 'redefine';
 use strict;
 use warnings;
 no warnings 'redefine';
 use strict;

-use IkiWiki 2.00;
+use IkiWiki 3.00;

 use IkiWiki::Render;
 use Net::Amazon::S3;
 
 use IkiWiki::Render;
 use Net::Amazon::S3;
 


@@ -16,11 +16,81 @@ BEGIN {
        }
 };
 
        }
 };
 

-sub import { #{{{
+sub import {
+       hook(type => "getopt", id => "amazon_s3", call => \&getopt);
+       hook(type => "getsetup", id => "amazon_s3", call => \&getsetup);

        hook(type => "checkconfig", id => "amazon_s3", call => \&checkconfig);
        hook(type => "checkconfig", id => "amazon_s3", call => \&checkconfig);

-} # }}}
+}
+
+sub getopt () {
+        eval q{use Getopt::Long};
+        error($@) if $@;
+        Getopt::Long::Configure('pass_through');
+        GetOptions("delete-bucket" => sub {
+               my $bucket=getbucket();
+               debug(gettext("deleting bucket.."));
+               my $resp = $bucket->list_all or die $bucket->err . ": " . $bucket->errstr;
+               foreach my $key (@{$resp->{keys}}) {
+                       debug("\t".$key->{key});
+                       $bucket->delete_key($key->{key}) or die $bucket->err . ": " . $bucket->errstr;
+               }
+               $bucket->delete_bucket or die $bucket->err . ": " . $bucket->errstr;
+               debug(gettext("done"));
+               exit(0);
+       });
+}

 
 

-sub checkconfig { #{{{
+sub getsetup () {
+       return
+               plugin => {
+                       safe => 0,
+                       rebuild => 0,
+               },
+               amazon_s3_key_id => {
+                       type => "string",
+                       example => "XXXXXXXXXXXXXXXXXXXX",
+                       description => "public access key id",
+                       safe => 1,
+                       rebuild => 0,
+               },
+               amazon_s3_key_id => {
+                       type => "string",
+                       example => "$ENV{HOME}/.s3_key",
+                       description => "file holding secret key (must not be readable by others!)",
+                       safe => 0, # ikiwiki reads this file
+                       rebuild => 0,
+               },
+               amazon_s3_bucket => {
+                       type => "string",
+                       example => "mywiki",
+                       description => "globally unique name of bucket to store wiki in",
+                       safe => 1,
+                       rebuild => 1,
+               },
+               amazon_s3_prefix => {
+                       type => "string",
+                       example => "wiki/",
+                       description => "a prefix to prepend to each page name",
+                       safe => 1,
+                       rebuild => 1,
+               },
+               amazon_s3_location => {
+                       type => "string",
+                       example => "EU",
+                       description => "which S3 datacenter to use (leave blank for default)",
+                       safe => 1,
+                       rebuild => 1,
+               },
+               amazon_s3_dupindex => {
+                       type => "boolean",
+                       example => 0,
+                       description => "store each index file twice? (allows urls ending in \"/index.html\" and \"/\")",
+                       safe => 1,
+                       rebuild => 1,
+               },
+}
+
+sub checkconfig {

        foreach my $field (qw{amazon_s3_key_id amazon_s3_key_file
                              amazon_s3_bucket}) {
                if (! exists $config{$field} || ! defined $config{$field}) {
        foreach my $field (qw{amazon_s3_key_id amazon_s3_key_file
                              amazon_s3_bucket}) {
                if (! exists $config{$field} || ! defined $config{$field}) {


@@ -31,11 +101,11 @@ sub checkconfig { #{{{
            ! defined $config{amazon_s3_prefix}) {
            $config{amazon_s3_prefix}="wiki/";
        }
            ! defined $config{amazon_s3_prefix}) {
            $config{amazon_s3_prefix}="wiki/";
        }

-} #}}}
+}

 
 {
 my $bucket;
 
 {
 my $bucket;

-sub getbucket { #{{{
+sub getbucket {

        return $bucket if defined $bucket;
        
        open(IN, "<", $config{amazon_s3_key_file}) || error($config{amazon_s3_key_file}.": ".$!);
        return $bucket if defined $bucket;
        
        open(IN, "<", $config{amazon_s3_key_file}) || error($config{amazon_s3_key_file}.": ".$!);


@@ -63,12 +133,39 @@ sub getbucket { #{{{
        }
 
        if (! $bucket) {
        }
 
        if (! $bucket) {

-               error(gettext("Failed to create bucket in S3: ").
+               # Try to use existing bucket.
+               $bucket=$s3->bucket($config{amazon_s3_bucket});
+       }
+       if (! $bucket) {
+               error(gettext("Failed to create S3 bucket: ").

                        $s3->err.": ".$s3->errstr."\n");
        }
 
        return $bucket;
                        $s3->err.": ".$s3->errstr."\n");
        }
 
        return $bucket;

-} #}}}
+}
+}
+
+# Given a file, return any S3 keys associated with it.
+sub file2keys ($) {
+       my $file=shift;
+
+       my @keys;
+       if ($file =~ /^\Q$config{destdir}\/\E(.*)/) {
+               push @keys, $config{amazon_s3_prefix}.$1;
+
+               # Munge foo/index.html to foo/
+               if ($keys[0]=~/(^|.*\/)index.$config{htmlext}$/) {
+                       # A duplicate might need to be stored under the
+                       # unmunged name too.
+                       if (!$config{usedirs} || $config{amazon_s3_dupindex}) {
+                               push @keys, $1;
+                       }
+                       else {
+                               @keys=($1);
+                       }
+               }
+       }
+       return @keys;

 }
 
 package IkiWiki;
 }
 
 package IkiWiki;


@@ -76,7 +173,7 @@ use File::MimeInfo;
 use Encode;
 
 # This is a wrapper around the real writefile.
 use Encode;
 
 # This is a wrapper around the real writefile.

-sub writefile ($$$;$$) { #{{{
+sub writefile ($$$;$$) {

         my $file=shift;
         my $destdir=shift;
         my $content=shift;
         my $file=shift;
         my $destdir=shift;
         my $content=shift;


@@ -86,20 +183,10 @@ sub writefile ($$$;$$) { #{{{
        # First, write the file to disk.
        my $ret=$IkiWiki::Plugin::amazon_s3::subs{'IkiWiki::writefile'}->($file, $destdir, $content, $binary, $writer);
 
        # First, write the file to disk.
        my $ret=$IkiWiki::Plugin::amazon_s3::subs{'IkiWiki::writefile'}->($file, $destdir, $content, $binary, $writer);
 

-       # Now, determine if the file was written to the destdir.
-       # writefile might be used for writing files elsewhere.
-       # Also, $destdir might be set to a subdirectory of the destdir.
-       my $key;
-       if ($destdir eq $config{destdir}) {
-               $key=$file;
-       }
-       elsif ("$destdir/$file" =~ /^\Q$config{destdir}\/\E(.*)/) {
-               $key=$1;
-       }
+       my @keys=IkiWiki::Plugin::amazon_s3::file2keys("$destdir/$file");

 
        # Store the data in S3.
 
        # Store the data in S3.

-       if (defined $key) {
-               $key=$config{amazon_s3_prefix}.$key;
+       if (@keys) {

                my $bucket=IkiWiki::Plugin::amazon_s3::getbucket();
 
                # The http layer tries to downgrade utf-8
                my $bucket=IkiWiki::Plugin::amazon_s3::getbucket();
 
                # The http layer tries to downgrade utf-8


@@ -108,69 +195,63 @@ sub writefile ($$$;$$) { #{{{
                # so force convert it to bytes.
                $content=encode_utf8($content) if defined $content;
 
                # so force convert it to bytes.
                $content=encode_utf8($content) if defined $content;
 

-               if (defined $content && ! length $content) {
-                       # S3 doesn't allow storing empty files!
-                       $content=" ";
-               }
-               

                my %opts=(
                        acl_short => 'public-read',
                        content_type => mimetype("$destdir/$file"),
                );
                my %opts=(
                        acl_short => 'public-read',
                        content_type => mimetype("$destdir/$file"),
                );

-               my $res;
-               if (! $writer) {
-                       $res=$bucket->add_key($key, $content, \%opts);
-               }
-               else {
-                       # read back in the file that the writer emitted
-                       $res=$bucket->add_key_filename($key, "$destdir/$file", \%opts);
-               }
-               if ($res && $key=~/(^|.*\/)index.$config{htmlext}$/) {
-                       # index.html files are a special case. Since S3 is
-                       # not a normal web server, it won't serve up
-                       # foo/index.html when foo/ is requested. So the
-                       # file has to be stored twice. (This is bad news
-                       # when usedirs is enabled!)
-                       # TODO: invesitgate using the new copy operation.
-                       #       (It may not be robust enough.)
-                       my $base=$1;
+
+               # If there are multiple keys to write, data is sent
+               # multiple times.
+               # TODO: investigate using the new copy operation.
+               #       (It may not be robust enough.)
+               foreach my $key (@keys) {
+                       my $res;

                        if (! $writer) {
                        if (! $writer) {

-                               $res=$bucket->add_key($base, $content, \%opts);
+                               $res=$bucket->add_key($key, $content, \%opts);

                        }
                        else {
                        }
                        else {

-                               $res=$bucket->add_key_filename($base, "$destdir/$file", \%opts);
+                               # This test for empty files is a workaround
+                               # for this bug:
+                               # http://rt.cpan.org//Ticket/Display.html?id=35731
+                               if (-z "$destdir/$file") {
+                                       $res=$bucket->add_key($key, "", \%opts);
+                               }
+                               else {
+                                       # read back in the file that the writer emitted
+                                       $res=$bucket->add_key_filename($key, "$destdir/$file", \%opts);
+                               }
+                       }
+                       if (! $res) {
+                               error(gettext("Failed to save file to S3: ").
+                                       $bucket->err.": ".$bucket->errstr."\n");

                        }
                        }

-               }
-               if (! $res) {
-                       error(gettext("Failed to save file to S3: ").
-                               $bucket->err.": ".$bucket->errstr."\n");

                }
        }
 
        return $ret;
                }
        }
 
        return $ret;

-} #}}}
+}

 
 # This is a wrapper around the real prune.
 
 # This is a wrapper around the real prune.

-sub prune ($) { #{{{
+sub prune ($;$) {

        my $file=shift;
        my $file=shift;

+       my $up_to=shift;

 
 

-       # If a file in the destdir is being pruned, need to delete it out
-       # of S3 as well.
-       if ($file =~ /^\Q$config{destdir}\/\E(.*)/) {
-               my $key=$config{amazon_s3_prefix}.$1;
+       my @keys=IkiWiki::Plugin::amazon_s3::file2keys($file);
+
+       # Prune files out of S3 too.
+       if (@keys) {

                my $bucket=IkiWiki::Plugin::amazon_s3::getbucket();
                my $bucket=IkiWiki::Plugin::amazon_s3::getbucket();

-               my $res=$bucket->delete_key($key);
-               if ($res && $key=~/(^|.*\/)index.$config{htmlext}$/) {
-                       # index.html special case: Delete other file too
-                       $res=$bucket->delete_key($1);
-               }
-               if (! $res) {
-                       error(gettext("Failed to delete file from S3: ").
-                               $bucket->err.": ".$bucket->errstr."\n");
+
+               foreach my $key (@keys) {
+                       my $res=$bucket->delete_key($key);
+                       if (! $res) {
+                               error(gettext("Failed to delete file from S3: ").
+                                       $bucket->err.": ".$bucket->errstr."\n");
+                       }

                }
        }
 
                }
        }
 

-       return $IkiWiki::Plugin::amazon_s3::subs{'IkiWiki::prune'}->($file);
-} #}}}
+       return $IkiWiki::Plugin::amazon_s3::subs{'IkiWiki::prune'}->($file, $up_to);
+}

 
 1
 
 1