IkiWiki/Plugin/amazon_s3.pm

   1 #!/usr/bin/perl
   2 package IkiWiki::Plugin::amazon_s3;
   3
   4 use warnings;
   5 no warnings 'redefine';
   6 use strict;
   7 use IkiWiki 3.00;
   8 use IkiWiki::Render;
   9 use Net::Amazon::S3;
  10
  11 # Store references to real subs before overriding them.
  12 our %subs;
  13 BEGIN {
  14         foreach my $sub (qw{IkiWiki::writefile IkiWiki::prune}) {
  15                 $subs{$sub}=\&$sub;
  16         }
  17 };
  18
  19 sub import {
  20         hook(type => "getopt", id => "amazon_s3", call => \&getopt);
  21         hook(type => "getsetup", id => "amazon_s3", call => \&getsetup);
  22         hook(type => "checkconfig", id => "amazon_s3", call => \&checkconfig);
  23 }
  24
  25 sub getopt () {
  26         eval q{use Getopt::Long};
  27         error($@) if $@;
  28         Getopt::Long::Configure('pass_through');
  29         GetOptions("delete-bucket" => sub {
  30                 my $bucket=getbucket();
  31                 debug(gettext("deleting bucket.."));
  32                 my $resp = $bucket->list_all or die $bucket->err . ": " . $bucket->errstr;
  33                 foreach my $key (@{$resp->{keys}}) {
  34                         debug("\t".$key->{key});
  35                         $bucket->delete_key($key->{key}) or die $bucket->err . ": " . $bucket->errstr;
  36                 }
  37                 $bucket->delete_bucket or die $bucket->err . ": " . $bucket->errstr;
  38                 debug(gettext("done"));
  39                 exit(0);
  40         });
  41 }
  42
  43 sub getsetup () {
  44         return
  45                 plugin => {
  46                         safe => 0,
  47                         rebuild => 0,
  48                 },
  49                 amazon_s3_key_id => {
  50                         type => "string",
  51                         example => "XXXXXXXXXXXXXXXXXXXX",
  52                         description => "public access key id",
  53                         safe => 1,
  54                         rebuild => 0,
  55                 },
  56                 amazon_s3_key_id => {
  57                         type => "string",
  58                         example => "$ENV{HOME}/.s3_key",
  59                         description => "file holding secret key (must not be readable by others!)",
  60                         safe => 0, # ikiwiki reads this file
  61                         rebuild => 0,
  62                 },
  63                 amazon_s3_bucket => {
  64                         type => "string",
  65                         example => "mywiki",
  66                         description => "globally unique name of bucket to store wiki in",
  67                         safe => 1,
  68                         rebuild => 1,
  69                 },
  70                 amazon_s3_prefix => {
  71                         type => "string",
  72                         example => "wiki/",
  73                         description => "a prefix to prepend to each page name",
  74                         safe => 1,
  75                         rebuild => 1,
  76                 },
  77                 amazon_s3_location => {
  78                         type => "string",
  79                         example => "EU",
  80                         description => "which S3 datacenter to use (leave blank for default)",
  81                         safe => 1,
  82                         rebuild => 1,
  83                 },
  84                 amazon_s3_dupindex => {
  85                         type => "boolean",
  86                         example => 0,
  87                         description => "store each index file twice? (allows urls ending in \"/index.html\" and \"/\")",
  88                         safe => 1,
  89                         rebuild => 1,
  90                 },
  91 }
  92
  93 sub checkconfig {
  94         foreach my $field (qw{amazon_s3_key_id amazon_s3_key_file
  95                               amazon_s3_bucket}) {
  96                 if (! exists $config{$field} || ! defined $config{$field}) {
  97                         error(sprintf(gettext("Must specify %s"), $field));
  98                 }
  99         }
 100         if (! exists $config{amazon_s3_prefix} ||
 101             ! defined $config{amazon_s3_prefix}) {
 102             $config{amazon_s3_prefix}="wiki/";
 103         }
 104 }
 105
 106 {
 107 my $bucket;
 108 sub getbucket {
 109         return $bucket if defined $bucket;
 110
 111         open(IN, "<", $config{amazon_s3_key_file}) || error($config{amazon_s3_key_file}.": ".$!);
 112         my $key=<IN>;
 113         chomp $key;
 114         close IN;
 115
 116         my $s3=Net::Amazon::S3->new({
 117                 aws_access_key_id => $config{amazon_s3_key_id},
 118                 aws_secret_access_key => $key,
 119                 retry => 1,
 120         });
 121
 122         # make sure the bucket exists
 123         if (exists $config{amazon_s3_location}) {
 124                 $bucket=$s3->add_bucket({
 125                         bucket => $config{amazon_s3_bucket},
 126                         location_constraint => $config{amazon_s3_location},
 127                 });
 128         }
 129         else {
 130                 $bucket=$s3->add_bucket({
 131                         bucket => $config{amazon_s3_bucket},
 132                 });
 133         }
 134
 135         if (! $bucket) {
 136                 # Try to use existing bucket.
 137                 $bucket=$s3->bucket($config{amazon_s3_bucket});
 138         }
 139         if (! $bucket) {
 140                 error(gettext("Failed to create S3 bucket: ").
 141                         $s3->err.": ".$s3->errstr."\n");
 142         }
 143
 144         return $bucket;
 145 }
 146 }
 147
 148 # Given a file, return any S3 keys associated with it.
 149 sub file2keys ($) {
 150         my $file=shift;
 151
 152         my @keys;
 153         if ($file =~ /^\Q$config{destdir}\/\E(.*)/) {
 154                 push @keys, $config{amazon_s3_prefix}.$1;
 155
 156                 # Munge foo/index.html to foo/
 157                 if ($keys[0]=~/(^|.*\/)index.$config{htmlext}$/) {
 158                         # A duplicate might need to be stored under the
 159                         # unmunged name too.
 160                         if (!$config{usedirs} || $config{amazon_s3_dupindex}) {
 161                                 push @keys, $1;
 162                         }
 163                         else {
 164                                 @keys=($1);
 165                         }
 166                 }
 167         }
 168         return @keys;
 169 }
 170
 171 package IkiWiki;
 172 use File::MimeInfo;
 173 use Encode;
 174
 175 # This is a wrapper around the real writefile.
 176 sub writefile ($$$;$$) {
 177         my $file=shift;
 178         my $destdir=shift;
 179         my $content=shift;
 180         my $binary=shift;
 181         my $writer=shift;
 182
 183         # First, write the file to disk.
 184         my $ret=$IkiWiki::Plugin::amazon_s3::subs{'IkiWiki::writefile'}->($file, $destdir, $content, $binary, $writer);
 185
 186         my @keys=IkiWiki::Plugin::amazon_s3::file2keys("$destdir/$file");
 187
 188         # Store the data in S3.
 189         if (@keys) {
 190                 my $bucket=IkiWiki::Plugin::amazon_s3::getbucket();
 191
 192                 # The http layer tries to downgrade utf-8
 193                 # content, but that can fail (see
 194                 # http://rt.cpan.org/Ticket/Display.html?id=35710),
 195                 # so force convert it to bytes.
 196                 $content=encode_utf8($content) if defined $content;
 197
 198                 my %opts=(
 199                         acl_short => 'public-read',
 200                         content_type => mimetype("$destdir/$file"),
 201                 );
 202
 203                 # If there are multiple keys to write, data is sent
 204                 # multiple times.
 205                 # TODO: investigate using the new copy operation.
 206                 #       (It may not be robust enough.)
 207                 foreach my $key (@keys) {
 208                         my $res;
 209                         if (! $writer) {
 210                                 $res=$bucket->add_key($key, $content, \%opts);
 211                         }
 212                         else {
 213                                 # This test for empty files is a workaround
 214                                 # for this bug:
 215                                 # http://rt.cpan.org//Ticket/Display.html?id=35731
 216                                 if (-z "$destdir/$file") {
 217                                         $res=$bucket->add_key($key, "", \%opts);
 218                                 }
 219                                 else {
 220                                         # read back in the file that the writer emitted
 221                                         $res=$bucket->add_key_filename($key, "$destdir/$file", \%opts);
 222                                 }
 223                         }
 224                         if (! $res) {
 225                                 error(gettext("Failed to save file to S3: ").
 226                                         $bucket->err.": ".$bucket->errstr."\n");
 227                         }
 228                 }
 229         }
 230
 231         return $ret;
 232 }
 233
 234 # This is a wrapper around the real prune.
 235 sub prune ($;$) {
 236         my $file=shift;
 237         my $up_to=shift;
 238
 239         my @keys=IkiWiki::Plugin::amazon_s3::file2keys($file);
 240
 241         # Prune files out of S3 too.
 242         if (@keys) {
 243                 my $bucket=IkiWiki::Plugin::amazon_s3::getbucket();
 244
 245                 foreach my $key (@keys) {
 246                         my $res=$bucket->delete_key($key);
 247                         if (! $res) {
 248                                 error(gettext("Failed to delete file from S3: ").
 249                                         $bucket->err.": ".$bucket->errstr."\n");
 250                         }
 251                 }
 252         }
 253
 254         return $IkiWiki::Plugin::amazon_s3::subs{'IkiWiki::prune'}->($file, $up_to);
 255 }
 256
 257 1