Remove files that can't be processed for some reason (fix #112).

sigal/gallery.py

@@ -594,10 +594,14 @@ class Gallery(object):
         bar_opt = {'label': "Processing files",
                    'show_pos': True,
                    'file': self.progressbar_target}
+
         if self.pool:
+            failed_files = []
             try:
                 with progressbar(length=len(media_list), **bar_opt) as bar:
-                    for _ in self.pool.imap_unordered(worker, media_list):
+                    for res in self.pool.imap_unordered(worker, media_list):
+                        if res:
+                            failed_files.append(res)
                         next(bar)
                 self.pool.close()
                 self.pool.join()
@@ -612,6 +616,8 @@ class Gallery(object):
                     exc_info=True)
                 sys.exit('Abort')
 
+            if failed_files:
+                self.remove_files(failed_files)
             print('')
         else:
             with progressbar(media_list, **bar_opt) as medias:
@@ -625,6 +631,18 @@ class Gallery(object):
 
         signals.gallery_build.send(self)
 
+    def remove_files(self, files):
+        self.logger.error('Some files have failed to be processed:')
+        for path, filename in files:
+            self.logger.error('  - %s/%s', path, filename)
+            album = self.albums[path]
+            for f in album.medias:
+                if f.filename == filename:
+                    album.medias.remove(f)
+                    break
+        self.logger.error('You can run sigal in verbose (--verbose) or debug '
+                          '(--debug) mode to get more details.')
+
     def process_dir(self, album, force=False):
         """Process a list of images in a directory."""
         for f in album:
@@ -633,22 +651,21 @@ class Gallery(object):
                 self.stats[f.type + '_skipped'] += 1
             else:
                 self.stats[f.type] += 1
-                yield f.type, f.src_path, album.dst_path, self.settings
+                yield (f.type, f.path, f.filename, f.src_path, album.dst_path,
+                       self.settings)
 
 
 def process_file(args):
-    ftype, src_path, dst_path, settings = args
-    logger = logging.getLogger(__name__)
-    logger.info('Processing %s', src_path)
-
-    if ftype == 'image':
-        return process_image(src_path, dst_path, settings)
-    elif ftype == 'video':
-        return process_video(src_path, dst_path, settings)
+    # args => ftype, path, filename, src_path, dst_path, settings
+    processor = process_image if args[0] == 'image' else process_video
+    ret = processor(*args[3:])
+    # If the processor return an error (ret != 0), then we return the path and
+    # filename of the failed file to the parent process.
+    return args[1:3] if ret else None
 
 
 def worker(args):
     try:
-        process_file(args)
+        return process_file(args)
     except KeyboardInterrupt:
-        return 'KeyboardException'
+        pass

sigal/image.py

@@ -43,7 +43,7 @@ from pilkit.processors import Transpose
 from pilkit.utils import save_image
 
 from . import compat, signals
-from .settings import get_thumb
+from .settings import get_thumb, Status
 
 
 def _has_exif_tags(img):
@@ -128,6 +128,7 @@ def process_image(filepath, outpath, settings):
     """Process one image: resize, create thumbnail."""
 
     logger = logging.getLogger(__name__)
+    logger.info('Processing %s', filepath)
     filename = os.path.split(filepath)[1]
     outname = os.path.join(outpath, filename)
     ext = os.path.splitext(filename)[1]
@@ -141,15 +142,16 @@ def process_image(filepath, outpath, settings):
 
     try:
         generate_image(filepath, outname, settings, options=options)
-    except Exception as e:
-        logger.error('Failed to process image: %s', e)
-        return
+    except Exception:
+        return Status.FAILURE
 
     if settings['make_thumbs']:
         thumb_name = os.path.join(outpath, get_thumb(settings, filename))
         generate_thumbnail(outname, thumb_name, settings['thumb_size'],
                            fit=settings['thumb_fit'], options=options)
 
+    return Status.SUCCESS
+
 
 def _get_exif_data(filename):
     """Return a dict with EXIF data."""

sigal/settings.py

@@ -71,6 +71,11 @@ _DEFAULT_CONFIG = {
 }
 
 
+class Status(object):
+    SUCCESS = 0
+    FAILURE = 1
+
+
 def get_thumb(settings, filename):
     """Return the path to the thumb.
 

sigal/video.py

@@ -30,7 +30,7 @@ import shutil
 from os.path import splitext
 
 from . import image
-from .settings import get_thumb
+from .settings import get_thumb, Status
 from .utils import call_subprocess
 
 
@@ -53,7 +53,6 @@ def check_subprocess(cmd, source, outname):
         raise
 
     if returncode:
-        logger.error('Failed to process ' + source)
         logger.debug('STDOUT:\n %s', stdout)
         logger.debug('STDERR:\n %s', stderr)
         if os.path.isfile(outname):
@@ -121,10 +120,7 @@ def generate_video(source, outname, size, options=None):
     cmd += resize_opt + [outname]
 
     logger.debug('Processing video: %s', ' '.join(cmd))
-    try:
-        check_subprocess(cmd, source, outname)
-    except Exception:
-        pass
+    check_subprocess(cmd, source, outname)
 
 
 def generate_thumbnail(source, outname, box, fit=True, options=None):
@@ -156,11 +152,19 @@ def process_video(filepath, outpath, settings):
     basename = splitext(filename)[0]
     outname = os.path.join(outpath, basename + '.webm')
 
-    generate_video(filepath, outname, settings['video_size'],
-                   options=settings['webm_options'])
+    try:
+        generate_video(filepath, outname, settings['video_size'],
+                       options=settings['webm_options'])
+    except Exception:
+        return Status.FAILURE
 
     if settings['make_thumbs']:
         thumb_name = os.path.join(outpath, get_thumb(settings, filename))
-        generate_thumbnail(
-            outname, thumb_name, settings['thumb_size'],
-            fit=settings['thumb_fit'], options=settings['jpg_options'])
+        try:
+            generate_thumbnail(
+                outname, thumb_name, settings['thumb_size'],
+                fit=settings['thumb_fit'], options=settings['jpg_options'])
+        except Exception:
+            return Status.FAILURE
+
+    return Status.SUCCESS