Merge pull request #236 from t-animal/extended_caching_plugin

plugin to cache the exif data of images

sigal/plugins/extended_caching.py

@@ -0,0 +1,95 @@
+# encoding: utf-8
+
+# Copyright 2017 - Tilman 't.animal' Adler
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+""" Decreases the time needed to build large galleries (e.g.: 25k images in 2.5s instead of 30s)
+
+    This plugin allows extended caching, which is useful for large galleries. Once a gallery has
+    been built it caches the exif-data of the contained images in the gallery target folder.
+    Before the next run it restores them so that the image does not have to be parsed again. For
+    large galleries this can speed up the creation of index files dramatically.
+"""
+
+import pickle
+import io
+import logging
+import os
+from sigal import signals
+
+logger = logging.getLogger(__name__)
+
+def load_exif(album):
+    """Loads the exif data of all images in an album from cache"""
+    if not hasattr(album.gallery, "exifCache"):
+        _restore_cache(album.gallery)
+    cache = album.gallery.exifCache
+
+    for media in album.medias:
+        if media.type == "image":
+            key = os.path.join(media.path, media.filename)
+            if key in cache:
+                media.exif = cache[key]
+
+def _restore_cache(gallery):
+    """Restores the exif data cache from the cache file"""
+    cachePath = os.path.join(gallery.settings["destination"], ".exif_cache")
+    try:
+        if os.path.exists(cachePath):
+            with open(cachePath, "rb") as cacheFile:
+                gallery.exifCache = pickle.load(cacheFile)
+                logger.debug("Loaded cache with %d entries", len(gallery.exifCache))
+        else:
+            gallery.exifCache = {}
+    except Exception as e:
+        logger.warn("Could not load cache: %s", e)
+        gallery.exifCache = {}
+
+def save_cache(gallery):
+    """Stores the exif data of all images in the gallery"""
+
+    if hasattr(gallery, "exifCache"):
+        cache = gallery.exifCache
+    else:
+        cache = gallery.exifCache = {}
+
+    for album in gallery.albums.values():
+        for image in album.images:
+            cache[os.path.join(image.path, image.filename)] = image.exif
+
+    cachePath = os.path.join(gallery.settings["destination"], ".exif_cache")
+
+    if len(cache) == 0:
+        if os.path.exists(cachePath):
+            os.remove(cachePath)
+        return
+
+    try:
+        with open(cachePath, "wb") as cacheFile:
+            pickle.dump(cache, cacheFile)
+            logger.debug("Stored cache with %d entries", len(gallery.exifCache))
+    except Exception as e:
+        logger.warn("Could not store cache: %s", e)
+        os.remove(cachePath)
+
+
+def register(settings):
+    signals.gallery_build.connect(save_cache)
+    signals.album_initialized.connect(load_exif)

tests/sample/sigal.conf.py

@@ -11,7 +11,7 @@ links = [('Example link', 'http://example.org'),
 
 plugins = ['sigal.plugins.adjust', 'sigal.plugins.copyright',
            'sigal.plugins.watermark', 'sigal.plugins.feeds',
-           'sigal.plugins.nomedia']
+           'sigal.plugins.nomedia', 'sigal.plugins.extended_caching']
 copyright = u"© An example copyright message"
 adjust_options = {'color': 0.0, 'brightness': 1.0,
                   'contrast': 1.0, 'sharpness': 0.0}

tests/test_extended_caching.py

@@ -0,0 +1,53 @@
+# -*- coding:utf-8 -*-
+
+import datetime
+import os
+import pickle
+import pytest
+
+from sigal.gallery import Gallery
+from sigal.plugins import extended_caching
+
+CURRENT_DIR = os.path.dirname(__file__)
+
+def test_save_cache(settings):
+    gal = Gallery(settings, ncpu=1)
+    extended_caching.save_cache(gal)
+
+    cachePath = os.path.join(settings['destination'], ".exif_cache")
+
+    assert os.path.isfile(cachePath)
+
+    with open(cachePath, "rb") as cacheFile:
+        cache = pickle.load(cacheFile)
+
+    assert cache["exifTest/21.jpg"] == gal.albums["exifTest"].medias[0].exif
+    assert cache["exifTest/22.jpg"] == gal.albums["exifTest"].medias[1].exif
+    assert cache["exifTest/noexif.png"] == gal.albums["exifTest"].medias[2].exif
+
+def test__restore_cache(settings):
+    gal1 = Gallery(settings, ncpu=1)
+    gal2 = Gallery(settings, ncpu=1)
+    extended_caching.save_cache(gal1)
+
+    cachePath = os.path.join(settings['destination'], ".exif_cache")
+    extended_caching._restore_cache(gal2)
+
+    assert gal1.exifCache == gal2.exifCache
+
+def test_load_exif(settings):
+    gal1 = Gallery(settings, ncpu=1)
+    gal1.albums["exifTest"].medias[2].exif = "blafoo"
+    gal1.exifCache = {"exifTest/21.jpg": "Foo",
+                     "exifTest/22.jpg": "Bar"}
+
+    extended_caching.load_exif(gal1.albums["exifTest"])
+
+    assert gal1.albums["exifTest"].medias[0].exif == "Foo"
+    assert gal1.albums["exifTest"].medias[1].exif == "Bar"
+    assert gal1.albums["exifTest"].medias[2].exif == "blafoo"
+
+    #check if setting gallery.exifCache works
+    gal2 = Gallery(settings, ncpu=1)
+    extended_caching.save_cache(gal1)
+    extended_caching.load_exif(gal2.albums["exifTest"])