From fa07bdca40bf7c34f6d1f0704871cda3c94d4c55 Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 31 Oct 2012 23:23:16 +0100 Subject: [PATCH] reorganize the parsing of the directory structure Avoid parsing too often the directory structure and files by doing it once at the beginning and storing the info in a dict --- sigal/__init__.py | 3 - sigal/gallery.py | 144 +++++++++++++++++++++++-------- sigal/generator.py | 208 ++++++++++++++------------------------------- 3 files changed, 170 insertions(+), 185 deletions(-) diff --git a/sigal/__init__.py b/sigal/__init__.py index 4d4b2f5..28b1c8e 100644 --- a/sigal/__init__.py +++ b/sigal/__init__.py @@ -105,6 +105,3 @@ def main(): gallery = Gallery(settings, args.input_dir, args.output_dir, force=args.force) gallery.build() - - r = Generator(settings, args.output_dir) - r.generate() diff --git a/sigal/gallery.py b/sigal/gallery.py index 7018443..7551293 100644 --- a/sigal/gallery.py +++ b/sigal/gallery.py @@ -21,13 +21,16 @@ # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. +import codecs import logging +import markdown import os +import PIL from clint.textui import progress -from shutil import copy2 from .image import Image, copy_exif +from .generator import Generator DESCRIPTION_FILE = "index.md" @@ -41,59 +44,88 @@ class Gallery: self.input_dir = os.path.abspath(input_dir) self.output_dir = os.path.abspath(output_dir) self.logger = logging.getLogger(__name__) + self.writer = Generator(settings, output_dir) - def filelist(self): - "get the list of directories with files of particular extensions" + def build_paths(self): + "get the list of directories with images" - for dirpath, dirnames, filenames in os.walk(self.input_dir): - # filelist = [os.path.normcase(f) for f in os.listdir(dir)] - imglist = [os.path.join(dirpath, f) for f in filenames - if os.path.splitext(f)[1] in self.settings['fileextlist']] - yield dirpath, dirnames, imglist + self.paths = {} + + for path, dirnames, filenames in os.walk(self.input_dir): + relpath = os.path.relpath(path, self.input_dir) + + # sort images and sub-albums by name + filenames.sort(key=str.lower) + dirnames.sort(key=str.lower) + + self.paths[relpath] = { + 'img': [ + f for f in filenames + if os.path.splitext(f)[1] in self.settings['fileextlist']], + 'subdir': dirnames + } + self.paths[relpath].update(get_metadata(path)) + + if relpath != '.': + alb_thumb = self.paths[relpath]['representative'] + if (not alb_thumb) or \ + (not os.path.isfile(os.path.join(path, alb_thumb))): + alb_thumb = self.find_representative(relpath) + self.paths[relpath]['representative'] = alb_thumb + + # import json + # print json.dumps(self.paths, indent=4) + + def find_representative(self, path): + "Find the representative image for a given path" + + for f in self.paths[path]['img']: + # find and return the first landscape image + im = PIL.Image.open(os.path.join(self.input_dir, path, f)) + if im.size[0] > im.size[1]: + return f + + # else simply return the 1st image + return self.paths[path]['img'][0] def build(self): - "create image gallery" + "Create image gallery" - if not os.path.isdir(self.output_dir): - self.logger.info("Create output directory %s", self.output_dir) - os.makedirs(self.output_dir) + self.logger.info("Generate gallery in %s ...", self.output_dir) + self.build_paths() + check_or_create_dir(self.output_dir) # loop on directories - for dirpath, dirnames, imglist in self.filelist(): - self.logger.warning("%s - %i images", - os.path.relpath(dirpath, self.input_dir), - len(imglist)) + for path in self.paths.keys(): + imglist = [os.path.join(self.input_dir, path, f) + for f in self.paths[path]['img']] - img_dir = dirpath.replace(self.input_dir, self.output_dir) + self.logger.warning("%s - %i images", path, len(imglist)) - if not os.path.isdir(img_dir): - os.mkdir(img_dir) - - descfile = os.path.join(dirpath, DESCRIPTION_FILE) - if os.path.isfile(descfile): - copy2(descfile, img_dir) + # output dir for the current path + img_out = os.path.join(self.output_dir, path) + check_or_create_dir(img_out) if len(imglist) != 0: - self.process_dir(imglist, img_dir) + self.process_dir(imglist, img_out) - def process_dir(self, imglist, img_dir): - "prepare images for a directory" + self.writer.generate(self.paths, path) - thumb_dir = os.path.join(img_dir, self.settings['thumb_dir']) - if not os.path.isdir(thumb_dir): - os.mkdir(thumb_dir) + def process_dir(self, imglist, img_out): + "Process images for a directory" + + thumb_dir = os.path.join(img_out, self.settings['thumb_dir']) + check_or_create_dir(thumb_dir) if self.settings['big_img']: - bigimg_dir = os.path.join(img_dir, - self.settings['bigimg_dir']) - if not os.path.isdir(bigimg_dir): - os.mkdir(bigimg_dir) + bigimg_dir = os.path.join(img_out, self.settings['bigimg_dir']) + check_or_create_dir(bigimg_dir) # loop on images for f in progress.bar(imglist): filename = os.path.split(f)[1] - im_name = os.path.join(img_dir, filename) + im_name = os.path.join(img_out, filename) if os.path.isfile(im_name) and not self.force: self.logger.info("%s exists - skipping", filename) @@ -114,12 +146,50 @@ class Gallery: img.save(im_name, quality=self.settings['jpg_quality']) if self.settings['make_thumbs']: - thumb_name = os.path.join(thumb_dir, - self.settings['thumb_prefix'] + - filename) + thumb_name = os.path.join( + thumb_dir, self.settings['thumb_prefix'] + filename) img.thumbnail(thumb_name, self.settings['thumb_size'], fit=self.settings['thumb_fit'], quality=self.settings['jpg_quality']) if self.settings['exif']: copy_exif(f, im_name) + + +def get_metadata(path): + """ Get album metadata from DESCRIPTION_FILE: + + - title + - representative image + - description + """ + + descfile = os.path.join(path, DESCRIPTION_FILE) + meta = {} + + if not os.path.isfile(descfile): + # default: get title from directory name + meta['title'] = os.path.basename(path).replace('_', ' ').\ + replace('-', ' ').capitalize() + else: + md = markdown.Markdown(extensions=['meta']) + + with codecs.open(descfile, "r", "utf-8") as f: + text = f.read() + + html = md.convert(text) + + meta = { + 'title': md.Meta.get('title', [''])[0], + 'description': html, + 'representative': md.Meta.get('representative', [''])[0] + } + + return meta + + +def check_or_create_dir(path): + "Create the directory if it does not exist" + + if not os.path.isdir(path): + os.mkdir(path) diff --git a/sigal/generator.py b/sigal/generator.py index 36506ee..e70ce24 100644 --- a/sigal/generator.py +++ b/sigal/generator.py @@ -25,23 +25,20 @@ Generate html pages for each directory of images """ +import copy import os -import codecs -import markdown -import PIL +from os.path import abspath from distutils.dir_util import copy_tree -from fnmatch import fnmatch from jinja2 import Environment, PackageLoader from sigal.image import Image DEFAULT_THEME = "default" INDEX_PAGE = "index.html" -DESCRIPTION_FILE = "index.md" SIGAL_LINK = "https://github.com/saimn/sigal" PATH_SEP = u" ยป " THEMES_PATH = os.path.normpath(os.path.join( - os.path.abspath(os.path.dirname(__file__)), 'themes')) + abspath(os.path.dirname(__file__)), 'themes')) def do_link(link, title): @@ -52,10 +49,9 @@ def do_link(link, title): class Generator(): """ Generate html pages for each directory of images """ - def __init__(self, settings, path, theme=DEFAULT_THEME, tpl=INDEX_PAGE): - self.data = {} + def __init__(self, settings, output_dir, theme=DEFAULT_THEME): self.settings = settings - self.path = os.path.normpath(path) + self.output_dir = os.path.abspath(output_dir) self.theme = settings['theme'] or theme # search the theme in sigal/theme if the given one does not exists @@ -67,158 +63,80 @@ class Generator(): theme_relpath = os.path.relpath(self.theme, os.path.dirname(__file__)) env = Environment(loader=PackageLoader('sigal', theme_relpath)) - self.template = env.get_template(tpl) + self.template = env.get_template(INDEX_PAGE) - self.ctx = {} - self.ctx['sigal_link'] = SIGAL_LINK + self.copy_assets() - def directory_list(self): - "get the list of directories with files of particular extensions" + self.ctx = { + 'sigal_link': SIGAL_LINK, + 'theme': {'name': os.path.basename(self.theme)}, + 'images': [], + 'albums': [], + } - ignored = ['theme', self.settings['bigimg_dir']] - if self.settings['thumb_dir']: - ignored.append(self.settings['thumb_dir']) + def copy_assets(self): + "copy the theme files in the output dir" - for dirpath, dirnames, filenames in os.walk(self.path): - dirpath = os.path.normpath(dirpath) - if os.path.split(dirpath)[1] not in ignored and \ - not fnmatch(dirpath, '*theme*'): - # sort images and sub-albums by name - filenames.sort(key=str.lower) - dirnames.sort(key=str.lower) + self.theme_path = os.path.join(self.output_dir, 'theme') + copy_tree(self.theme, self.theme_path) - self.data[dirpath] = {} - self.data[dirpath]['img'] = [f for f in filenames - if os.path.splitext(f)[1] in - self.settings['fileextlist']] - self.data[dirpath]['subdir'] = [d for d in dirnames - if d not in ignored] - - def find_representative(self, path): - """ - find the representative image for a given album/path - at the moment, this is the first image found. - """ - - files = [f for f in os.listdir(path) - if os.path.isfile(os.path.join(path, f)) - and os.path.splitext(f)[1] in self.settings['fileextlist']] - - for f in files: - # find and return the first landscape image - im = PIL.Image.open(os.path.join(path, f)) - if im.size[0] > im.size[1]: - return f - - # else simply return the 1st image - return files[0] - - def generate(self): + def generate(self, paths, relpath): """ Render the html page """ - # copy static files in the output dir - theme_outpath = os.path.join(os.path.abspath(self.path), 'theme') - copy_tree(self.theme, theme_outpath) - self.ctx['theme'] = {'name': os.path.basename(self.theme)} + path = os.path.join(self.output_dir, relpath) - self.directory_list() + ctx = copy.deepcopy(self.ctx) + ctx['theme']['path'] = os.path.relpath(self.theme_path, path) + ctx['home_path'] = os.path.join( + os.path.relpath(self.output_dir, path), INDEX_PAGE) - for dirpath in self.data.keys(): - self.data[dirpath].update(get_metadata(dirpath)) + # paths to upper directories (with titles and links) + tmp_path = relpath + ctx['paths'] = do_link(INDEX_PAGE, paths[tmp_path]['title']) - # loop on directories - for dirpath in self.data.keys(): - self.ctx['theme']['path'] = os.path.relpath(theme_outpath, dirpath) - dir_relpath = os.path.relpath(self.path, dirpath) - self.ctx['home_path'] = os.path.join(dir_relpath, INDEX_PAGE) + while tmp_path != '.': + tmp_path = os.path.normpath(os.path.join(tmp_path, '..')) + link = os.path.relpath(tmp_path, relpath) + "/" + INDEX_PAGE + ctx['paths'] = do_link(link, paths[tmp_path]['title']) + \ + PATH_SEP + ctx['paths'] - # paths to upper directories (with titles and links) - tmp_path = dirpath - self.ctx['paths'] = do_link(INDEX_PAGE, - self.data[tmp_path]['title']) + for i in paths[relpath]['img']: + image = { + 'file': i, + 'thumb': os.path.join(self.settings['thumb_dir'], + self.settings['thumb_prefix'] + i) + } + ctx['images'].append(image) - while tmp_path != self.path: - tmp_path = os.path.normpath(os.path.join(tmp_path, '..')) - link = os.path.relpath(tmp_path, dirpath) + "/" + INDEX_PAGE - self.ctx['paths'] = do_link(link, - self.data[tmp_path]['title']) + \ - PATH_SEP + self.ctx['paths'] + for d in paths[relpath]['subdir']: - self.ctx['images'] = [] - for i in self.data[dirpath]['img']: - image = { - 'file': i, - 'thumb': os.path.join(self.settings['thumb_dir'], - self.settings['thumb_prefix'] + i) - } - self.ctx['images'].append(image) + dpath = os.path.normpath(os.path.join(relpath, d)) + alb_thumb = paths[dpath]['representative'] + thumb_name = os.path.join(self.settings['thumb_dir'], + self.settings['thumb_prefix'] + + alb_thumb) + thumb_path = os.path.join(self.output_dir, dpath, thumb_name) - self.ctx['albums'] = [] - for d in self.data[dirpath]['subdir']: + # generate the thumbnail if it is missing (if + # settings['make_thumbs'] is False) + if not os.path.exists(thumb_path): + img = Image(os.path.join(self.output_dir, dpath, alb_thumb)) + img.thumbnail(thumb_path, self.settings['thumb_size'], + fit=self.settings['thumb_fit'], + quality=self.settings['jpg_quality']) - dpath = os.path.join(dirpath, d) - alb_thumb = self.data[dpath].get('representative', '') + album = { + 'path': os.path.join(d, INDEX_PAGE), + 'title': paths[dpath]['title'], + 'thumb': os.path.join(d, thumb_name) + } + ctx['albums'].append(album) - if not alb_thumb or \ - not os.path.isfile(os.path.join(dpath, alb_thumb)): - alb_thumb = self.find_representative(dpath) + page = self.template.render(paths[relpath], **ctx).encode('utf-8') - thumb_name = os.path.join(self.settings['thumb_dir'], - self.settings['thumb_prefix'] + - alb_thumb) - thumb_path = os.path.join(dpath, thumb_name) - - if not os.path.exists(thumb_path): - img = Image(os.path.join(dpath, alb_thumb)) - img.thumbnail(thumb_path, self.settings['thumb_size'], - fit=self.settings['thumb_fit'], - quality=self.settings['jpg_quality']) - - album = { - 'path': os.path.join(d, INDEX_PAGE), - 'title': self.data[dpath]['title'], - 'thumb': os.path.join(d, thumb_name) - } - self.ctx['albums'].append(album) - - page = self.template.render(self.data[dirpath], - **self.ctx).encode('utf-8') - - # save page - f = open(os.path.join(dirpath, INDEX_PAGE), 'w') - f.write(page) - f.close() - - -def get_metadata(path): - """ Get album metadata from DESCRIPTION_FILE: - - - title - - representative image - - description - """ - - descfile = os.path.join(path, DESCRIPTION_FILE) - meta = {} - - if not os.path.isfile(descfile): - # default: get title from directory name - meta['title'] = os.path.basename(path).replace('_', ' ').\ - replace('-', ' ').capitalize() - else: - md = markdown.Markdown(extensions=['meta']) - - with codecs.open(descfile, "r", "utf-8") as f: - text = f.read() - - html = md.convert(text) - - meta = { - 'title': md.Meta.get('title', [''])[0], - 'description': html, - 'representative': md.Meta.get('representative', [''])[0] - } - - return meta + # save page + f = open(os.path.join(path, INDEX_PAGE), 'w') + f.write(page) + f.close()