From b1ecbd2ff1b736158c149e8cc059a42414d4dcba Mon Sep 17 00:00:00 2001 From: Jonas Kaufmann Date: Sun, 28 Sep 2014 17:12:53 +0200 Subject: [PATCH 1/2] Added plugin to upload generated gallery to Amazon S3 --- AUTHORS | 1 + sigal/plugins/upload_s3.py | 63 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 sigal/plugins/upload_s3.py diff --git a/AUTHORS b/AUTHORS index af81d48..3d4af53 100644 --- a/AUTHORS +++ b/AUTHORS @@ -2,6 +2,7 @@ Sigal is written and maintained by Simon Conseil and contributors (in alphabetical order): - Christophe-Marie Duquesne +- Jonas Kaufmann - Antoine Pitrou - Matthias Vogelgesang - Vikram Shirgur diff --git a/sigal/plugins/upload_s3.py b/sigal/plugins/upload_s3.py new file mode 100644 index 0000000..473573f --- /dev/null +++ b/sigal/plugins/upload_s3.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- + +"""Plugin to upload generated files to Amazon S3. + +This plugin requires boto_. All generated files are uploaded to a specified S3 bucket. +When using this plugin you have to make sure that the bucket already exists and the +you have access to the S3 bucket. The access credentials are managed by boto_ and +can be given as environment variables, configuration files etc. More information +can be found on the boto_ documentation. + +.. _boto: https://pypi.python.org/pypi/boto + +Settings (all settings are wrapped in ``upload_s3_options`` dict): + +- ``bucket``: The to-be-used bucket for uploading. + +""" + +import logging +import os +from sigal import signals +import boto +from boto.s3.key import Key + +logger = logging.getLogger(__name__) + + +def upload_s3(gallery, settings=None): + upload_files = [] + + for album in gallery.albums.values(): + logger.debug("Processing album: %s" % (album)) + for media in album.medias: + upload_files.append(os.path.join(album.path, media.filename)) + upload_files.append(os.path.join(album.path, gallery.settings['thumb_dir'], media.filename)) + + # generated HTML files + html_file_upload_path = album.path if album.path != '.' else '' + upload_files.append(os.path.join(html_file_upload_path, album.output_file)) + + # static directory from template + static_path = os.path.join(gallery.settings['destination'], 'static') + static_files = [ os.path.join(root[len(gallery.settings['destination'])+1:], name) + for root, dirs, files in os.walk(static_path) + for name in files ] + upload_files += static_files + + + conn = boto.connect_s3() + bucket = conn.get_bucket(gallery.settings['upload_s3_options']['bucket']) + for f in upload_files: + logger.debug("Uploading file %s" % (f)) + key = Key(bucket) + key.key = f + key.set_contents_from_filename(os.path.join(gallery.settings['destination'], f)) + return None + + +def register(settings): + if settings.get('upload_s3_options'): + signals.gallery_build.connect(upload_s3) + else: + logger.warning('Upload to S3 is not configured.') From 8472126a078ba721c20c5e5f5ef8e397acd5a944 Mon Sep 17 00:00:00 2001 From: Jonas Kaufmann Date: Sun, 12 Oct 2014 14:03:04 +0200 Subject: [PATCH 2/2] Improvements for upload to s3 plugin - All files in the sigal build directory are now uploaded instead of manually building file list. This makes the plugin more robust for future changes. - Added progressbar to check upload progress - Added option to set a S3 permission policy (for example public-read or private) on all uploaded files - Added option to skip files that were already uploaded before - Added plugin to documentation - Added plugin settings to default configuration template --- docs/plugins.rst | 5 ++++ sigal/plugins/upload_s3.py | 56 +++++++++++++++++++++-------------- sigal/templates/sigal.conf.py | 10 ++++++- 3 files changed, 47 insertions(+), 24 deletions(-) diff --git a/docs/plugins.rst b/docs/plugins.rst index 06d6b2b..d33a08d 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -93,3 +93,8 @@ Copyright plugin ================ .. automodule:: sigal.plugins.copyright + +Upload to S3 plugin +=================== + +.. automodule:: sigal.plugins.upload_s3 diff --git a/sigal/plugins/upload_s3.py b/sigal/plugins/upload_s3.py index 473573f..71404be 100644 --- a/sigal/plugins/upload_s3.py +++ b/sigal/plugins/upload_s3.py @@ -13,6 +13,10 @@ can be found on the boto_ documentation. Settings (all settings are wrapped in ``upload_s3_options`` dict): - ``bucket``: The to-be-used bucket for uploading. +- ``policy``: Specifying access control to the uploaded files. Possible values: + private, public-read, public-read-write, authenticated-read +- ``overwrite``: Boolean indicating if all files should be uploaded and overwritten + or if already uploaded files should be skipped. """ @@ -21,6 +25,7 @@ import os from sigal import signals import boto from boto.s3.key import Key +from click import progressbar logger = logging.getLogger(__name__) @@ -28,33 +33,38 @@ logger = logging.getLogger(__name__) def upload_s3(gallery, settings=None): upload_files = [] - for album in gallery.albums.values(): - logger.debug("Processing album: %s" % (album)) - for media in album.medias: - upload_files.append(os.path.join(album.path, media.filename)) - upload_files.append(os.path.join(album.path, gallery.settings['thumb_dir'], media.filename)) - - # generated HTML files - html_file_upload_path = album.path if album.path != '.' else '' - upload_files.append(os.path.join(html_file_upload_path, album.output_file)) - - # static directory from template - static_path = os.path.join(gallery.settings['destination'], 'static') - static_files = [ os.path.join(root[len(gallery.settings['destination'])+1:], name) - for root, dirs, files in os.walk(static_path) - for name in files ] - upload_files += static_files - + # Get local files + for root, dirs, files in os.walk(gallery.settings['destination']): + for f in files: + path = os.path.join(root[len(gallery.settings['destination'])+1:], f) + size = os.path.getsize(os.path.join(root, f)) + upload_files += [ (path, size) ] + # Connect to specified bucket conn = boto.connect_s3() bucket = conn.get_bucket(gallery.settings['upload_s3_options']['bucket']) - for f in upload_files: - logger.debug("Uploading file %s" % (f)) - key = Key(bucket) - key.key = f - key.set_contents_from_filename(os.path.join(gallery.settings['destination'], f)) - return None + # Upload the files + with progressbar(upload_files, label="Uploading files to S3") as progress_upload: + for (f, size) in progress_upload: + if gallery.settings['upload_s3_options']['overwrite'] == False: + # Check if file was uploaded before + key = bucket.get_key(f) + if key != None and key.size == size: + logger.debug("Skipping file %s" % (f)) + else: + upload_file(gallery, bucket, f) + else: + # File is not available on S3 yet + upload_file(gallery, bucket, f) + +def upload_file(gallery, bucket, f): + logger.debug("Uploading file %s" % (f)) + key = Key(bucket) + key.key = f + key.set_contents_from_filename( + os.path.join(gallery.settings['destination'], f), + policy = gallery.settings['upload_s3_options']['policy']) def register(settings): if settings.get('upload_s3_options'): diff --git a/sigal/templates/sigal.conf.py b/sigal/templates/sigal.conf.py index 870839c..ca58e42 100644 --- a/sigal/templates/sigal.conf.py +++ b/sigal/templates/sigal.conf.py @@ -171,7 +171,8 @@ ignore_files = [] # Another option is to import the plugin and put the module in the list, but # this will break with the multiprocessing feature (the settings dict obtained # from this file must be serializable). -# plugins = ['sigal.plugins.adjust', 'sigal.plugins.copyright'] +# plugins = ['sigal.plugins.adjust', 'sigal.plugins.copyright', +# 'sigal.plugins.upload_s3'] # Add a copyright text on the image (default: '') # copyright = "© An example copyright message" @@ -182,3 +183,10 @@ ignore_files = [] # 'brightness': 1.0, # 'contrast': 1.0, # 'sharpness': 1.0} + +# Settings for upload to s3 plugin +# upload_s3_options = { +# 'bucket': 'my-bucket', +# 'policy': 'public-read', +# 'overwrite': False +# } \ No newline at end of file