Merge 2b2d6e0424 into 21ddc7cccd

src/sigal/plugins/extended_caching.py

@@ -38,14 +38,35 @@ from ..utils import get_mod_date
 logger = logging.getLogger(__name__)
 
 
+def _cache_key_global(path, name):
+    """Global (gallery) cache key function"""
+    return os.path.join(path, name)
+
+
+def _cache_key_local(_, name):
+    """Local (album) cache key function"""
+    return name
+
+
 def load_metadata(album):
     """Loads the metadata of all media in an album from cache"""
-    if not hasattr(album.gallery, "metadataCache"):
-        _restore_cache(album.gallery)
-    cache = album.gallery.metadataCache
+    plugin_settings = album.gallery.settings.get("extended_caching_options", {})
+    if plugin_settings.get("global_cache", True):
+        if not hasattr(album.gallery, "metadata_cache"):
+            logger.debug("Loading from global gallery cache")
+            cache_path = os.path.join(album.gallery.settings["destination"], ".metadata_cache")
+            _restore_cache(cache_path, album.gallery)
+        cache = album.gallery.metadata_cache
+        cache_key = _cache_key_global
+    else:
+        if not hasattr(album, "metadata_cache"):
+            logger.debug("Loading from local album cache %s", album.name)
+            _restore_cache(os.path.join(album.dst_path, ".metadata_cache"), album)
+        cache = album.metadata_cache
+        cache_key = _cache_key_local
 
     # load album metadata
-    key = os.path.join(album.path, "_index")
+    key = cache_key(album.path, "_index")
     if key in cache:
         data = cache[key]
 
@@ -62,7 +83,7 @@ def load_metadata(album):
 
     # load media metadata
     for media in album.medias:
-        key = os.path.join(media.path, media.dst_filename)
+        key = cache_key(media.path, media.dst_filename)
         if key in cache:
             data = cache[key]
 
@@ -92,36 +113,47 @@ def load_metadata(album):
                 media.markdown_metadata = data["markdown_metadata"]
 
 
-def _restore_cache(gallery):
+def _restore_cache(cache_path, cache_object):
     """Restores the metadata cache from the cache file"""
-    cachePath = os.path.join(gallery.settings["destination"], ".metadata_cache")
     try:
-        if os.path.exists(cachePath):
-            with open(cachePath, "rb") as cacheFile:
-                gallery.metadataCache = pickle.load(cacheFile)
-                logger.debug("Loaded cache with %d entries", len(gallery.metadataCache))
+        if os.path.exists(cache_path):
+            with open(cache_path, "rb") as cache_file:
+                cache_object.metadata_cache = pickle.load(cache_file)
+                logger.debug("Loaded cache with %d entries", len(cache_object.metadata_cache))
         else:
-            gallery.metadataCache = {}
+            cache_object.metadata_cache = {}
     except Exception as e:
         logger.warning("Could not load cache: %s", e)
-        gallery.metadataCache = {}
+        cache_object.metadata_cache = {}
 
 
-def save_cache(gallery):
+def store_metadata(gallery):
     """Stores the exif data of all images in the gallery"""
-
-    if hasattr(gallery, "metadataCache"):
-        cache = gallery.metadataCache
+    plugin_settings = gallery.settings.get("extended_caching_options", {})
+    global_cache = plugin_settings.get("global_cache", True)
+    if global_cache:
+        logger.debug("Using global gallery cache")
+        if not hasattr(gallery, "metadata_cache"):
+            gallery.metadata_cache = {}
+        cache_key = _cache_key_global
     else:
-        cache = gallery.metadataCache = {}
+        logger.debug("Using local album caches")
+        cache_key = _cache_key_local
 
     for album in gallery.albums.values():
+        if global_cache:
+            cache = gallery.metadata_cache
+        else:
+            if not hasattr(album, "metadata_cache"):
+                album.metadata_cache = {}
+            cache = album.metadata_cache
+
         try:
             data = {
                 "mod_date": int(get_mod_date(album.markdown_metadata_filepath)),
                 "markdown_metadata": album.markdown_metadata,
             }
-            cache[os.path.join(album.path, "_index")] = data
+            cache[cache_key(album.path, "_index")] = data
         except FileNotFoundError:
             pass
 
@@ -147,24 +179,33 @@ def save_cache(gallery):
                 data["meta_mod_date"] = meta_mod_date
                 data["markdown_metadata"] = media.markdown_metadata
 
-            cache[os.path.join(media.path, media.dst_filename)] = data
+            cache[cache_key(media.path, media.dst_filename)] = data
+
+        if not global_cache:
+            cache_path = os.path.join(album.dst_path, ".metadata_cache")
+            _save_cache(cache_path, cache)
+
+    if global_cache:
+        cache_path = os.path.join(gallery.settings["destination"], ".metadata_cache")
+        _save_cache(cache_path, gallery.metadata_cache)
 
-    cachePath = os.path.join(gallery.settings["destination"], ".metadata_cache")
 
+def _save_cache(cache_path, cache):
+    """Stores the metadata cache to the cache file"""
     if len(cache) == 0:
-        if os.path.exists(cachePath):
-            os.remove(cachePath)
+        if os.path.exists(cache_path):
+            os.remove(cache_path)
         return
 
     try:
-        with open(cachePath, "wb") as cacheFile:
-            pickle.dump(cache, cacheFile)
-            logger.debug("Stored cache with %d entries", len(gallery.metadataCache))
+        with open(cache_path, "wb") as cache_file:
+            pickle.dump(cache, cache_file)
+            logger.debug("Stored cache with %d entries", len(cache))
     except Exception as e:
-        logger.warn("Could not store cache: %s", e)
-        os.remove(cachePath)
+        logger.warning("Could not store cache: %s", e)
+        os.remove(cache_path)
 
 
 def register(settings):
-    signals.gallery_build.connect(save_cache)
+    signals.gallery_build.connect(store_metadata)
     signals.album_initialized.connect(load_metadata)

src/sigal/templates/sigal.conf.py

@@ -312,6 +312,11 @@ ignore_files = []
 #    'ask_password': False
 # }
 
+# Settings for extended caching plugin
+# extended_caching_options = {
+#     'global_cache': True # a single gallery cache, vs caching per album
+# }
+
 # Settings for nonmedia_files plugin.
 # nonmedia_files_options = {
 #     'ext_as_thumb': True,

tests/test_extended_caching.py

@@ -7,16 +7,16 @@ from sigal.plugins import extended_caching
 CURRENT_DIR = os.path.dirname(__file__)
 
 
-def test_save_cache(settings, tmpdir):
+def test_store_metadata(settings, tmpdir):
     settings["destination"] = str(tmpdir)
     gal = Gallery(settings, ncpu=1)
-    extended_caching.save_cache(gal)
+    extended_caching.store_metadata(gal)
 
-    cachePath = os.path.join(settings["destination"], ".metadata_cache")
+    cache_path = os.path.join(settings["destination"], ".metadata_cache")
 
-    assert os.path.isfile(cachePath)
+    assert os.path.isfile(cache_path)
 
-    with open(cachePath, "rb") as cacheFile:
+    with open(cache_path, "rb") as cacheFile:
         cache = pickle.load(cacheFile)
 
     # test exif
@@ -52,27 +52,132 @@ def test_save_cache(settings, tmpdir):
 
     # test if file disappears
     gal.albums["exifTest"].medias.append(Image("foooo.jpg", "exifTest", settings))
-    extended_caching.save_cache(gal)
-    with open(cachePath, "rb") as cacheFile:
+    extended_caching.store_metadata(gal)
+    with open(cache_path, "rb") as cacheFile:
         cache = pickle.load(cacheFile)
     assert "exifTest/foooo.jpg" not in cache
 
 
+def test_load_metadata(settings, tmpdir):
+    settings["destination"] = str(tmpdir)
+    gal1 = Gallery(settings, ncpu=1)
+    gal2 = Gallery(settings, ncpu=1)
+    extended_caching.store_metadata(gal1)
+    for album in gal2.albums.values():
+        extended_caching.load_metadata(album)
+        break  # only need to load one
+    assert gal1.metadata_cache == gal2.metadata_cache
+
+
 def test_restore_cache(settings, tmpdir):
     settings["destination"] = str(tmpdir)
     gal1 = Gallery(settings, ncpu=1)
     gal2 = Gallery(settings, ncpu=1)
-    extended_caching.save_cache(gal1)
-    extended_caching._restore_cache(gal2)
-    assert gal1.metadataCache == gal2.metadataCache
+    extended_caching.store_metadata(gal1)
+    cache_path = os.path.join(settings["destination"], ".metadata_cache")
+    extended_caching._restore_cache(cache_path, gal2)
+    assert gal1.metadata_cache == gal2.metadata_cache
 
     # test bad cache
-    cachePath = os.path.join(settings["destination"], ".metadata_cache")
-    with open(cachePath, "w") as f:
+    with open(cache_path, "w") as f:
         f.write("bad pickle file")
 
-    extended_caching._restore_cache(gal2)
-    assert gal2.metadataCache == {}
+    extended_caching._restore_cache(cache_path, gal2)
+    assert gal2.metadata_cache == {}
+
+
+def test_store_metadata_local(settings, tmpdir):
+    settings["destination"] = str(tmpdir)
+    settings['extended_caching_options'] = {'global_cache': False}
+    gal = Gallery(settings, ncpu=1)
+    extended_caching.store_metadata(gal)
+
+    for album in gal.albums.values():
+        if album.metadata_cache:
+            cache_path = os.path.join(album.dst_path, ".metadata_cache")
+            assert os.path.isfile(cache_path)
+            with open(cache_path, "rb") as cacheFile:
+                cache = pickle.load(cacheFile)
+
+    # test exif
+    cache_path = os.path.join(settings["destination"], "exifTest", ".metadata_cache")
+    assert os.path.isfile(cache_path)
+    with open(cache_path, "rb") as cacheFile:
+        cache = pickle.load(cacheFile)
+
+    album = gal.albums["exifTest"]
+    cache_img = cache["21.jpg"]
+    assert cache_img["exif"] == album.medias[0].exif
+    assert "markdown_metadata" not in cache_img
+    assert cache_img["file_metadata"] == album.medias[0].file_metadata
+
+    cache_img = cache["22.jpg"]
+    assert cache_img["exif"] == album.medias[1].exif
+    assert "markdown_metadata" not in cache_img
+    assert cache_img["file_metadata"] == album.medias[1].file_metadata
+
+    cache_img = cache["noexif.png"]
+    assert cache_img["exif"] == album.medias[2].exif
+    assert "markdown_metadata" not in cache_img
+    assert cache_img["file_metadata"] == album.medias[2].file_metadata
+
+    # test iptc and md
+    cache_path = os.path.join(settings["destination"], "iptcTest", ".metadata_cache")
+    assert os.path.isfile(cache_path)
+    with open(cache_path, "rb") as cacheFile:
+        cache = pickle.load(cacheFile)
+
+    album = gal.albums["iptcTest"]
+    assert cache["_index"]["markdown_metadata"] == album.markdown_metadata
+
+    cache_img = cache["1.jpg"]
+    assert cache_img["file_metadata"] == album.medias[0].file_metadata
+    assert "markdown_metadata" not in cache_img
+
+    cache_img = cache["2.jpg"]
+    assert cache_img["markdown_metadata"] == album.medias[1].markdown_metadata
+
+    # test if file disappears
+    gal.albums["exifTest"].medias.append(Image("foooo.jpg", "exifTest", settings))
+    extended_caching.store_metadata(gal)
+    cache_path = os.path.join(settings["destination"], "exifTest", ".metadata_cache")
+    with open(cache_path, "rb") as cacheFile:
+        cache = pickle.load(cacheFile)
+    assert "foooo.jpg" not in cache
+
+
+def test_restore_cache_local(settings, tmpdir):
+    settings["destination"] = str(tmpdir)
+    settings['extended_caching_options'] = {'global_cache': False}
+    gal1 = Gallery(settings, ncpu=1)
+    gal2 = Gallery(settings, ncpu=1)
+    extended_caching.store_metadata(gal1)
+    cache_path = os.path.join(settings["destination"], "exifTest", ".metadata_cache")
+    extended_caching._restore_cache(cache_path, gal2.albums["exifTest"])
+    assert not hasattr(gal1, "metadata_cache")
+    assert not hasattr(gal2, "metadata_cache")
+    assert gal1.albums["exifTest"].metadata_cache == gal2.albums["exifTest"].metadata_cache
+
+    # test bad cache
+    with open(cache_path, "w") as f:
+        f.write("bad pickle file")
+
+    extended_caching._restore_cache(cache_path, gal2.albums["exifTest"])
+    assert gal2.albums["exifTest"].metadata_cache == {}
+
+
+def test_load_metadata_local(settings, tmpdir):
+    settings["destination"] = str(tmpdir)
+    settings['extended_caching_options'] = {'global_cache': False}
+    gal1 = Gallery(settings, ncpu=1)
+    gal2 = Gallery(settings, ncpu=1)
+    extended_caching.store_metadata(gal1)
+    for album in gal2.albums.values():
+        extended_caching.load_metadata(album)
+    assert not hasattr(gal1, "metadata_cache")
+    assert not hasattr(gal2, "metadata_cache")
+    for al1, al2 in zip(gal1.albums.values(), gal2.albums.values()):
+        assert al1.metadata_cache == al2.metadata_cache
 
 
 def test_load_exif(settings, tmpdir):
@@ -80,7 +185,7 @@ def test_load_exif(settings, tmpdir):
     gal1 = Gallery(settings, ncpu=1)
     gal1.albums["exifTest"].medias[3].exif = "blafoo"
     # set mod_date in future, to force these values
-    gal1.metadataCache = {
+    gal1.metadata_cache = {
         "exifTest/21.jpg": {"exif": "Foo", "mod_date": 100000000000},
         "exifTest/22.jpg": {"exif": "Bar", "mod_date": 100000000000},
         "exifTest/22-nodate.jpg": {"exif": "Baz", "mod_date": 100000000000},
@@ -97,9 +202,9 @@ def test_load_exif(settings, tmpdir):
     assert get_media(gal1, "22.jpg").exif == "Bar"
     assert get_media(gal1, "noexif.png").exif == "blafoo"
 
-    # check if setting gallery.metadataCache works
+    # check if setting gallery.metadata_cache works
     gal2 = Gallery(settings, ncpu=1)
-    extended_caching.save_cache(gal1)
+    extended_caching.store_metadata(gal1)
     extended_caching.load_metadata(gal2.albums["exifTest"])
 
     assert get_media(gal2, "21.jpg").exif == "Foo"
@@ -110,14 +215,14 @@ def test_load_exif(settings, tmpdir):
 def test_load_metadata_missing(settings, tmpdir):
     settings["destination"] = str(tmpdir)
     gal = Gallery(settings, ncpu=1)
-    extended_caching.save_cache(gal)
-    assert gal.metadataCache
+    extended_caching.store_metadata(gal)
+    assert gal.metadata_cache
 
     # test if file disappears
     gal.albums["exifTest"].medias.append(Image("foooo.jpg", "exifTest", settings))
 
     # set mod_date to -1 to force cache update
-    gal.metadataCache = {
+    gal.metadata_cache = {
         "exifTest/_index": {
             "mod_date": -1,
         },
@@ -138,3 +243,19 @@ def test_load_metadata_missing(settings, tmpdir):
     extended_caching.load_metadata(gal.albums["dir1/test2"])
     assert gal.albums["dir1/test2"].medias[1].exif == "Bar"
     assert gal.albums["dir1/test2"].medias[1].markdown_metadata != "Bar"
+
+
+def test_empty_cache(settings, tmpdir):
+    cache_path = os.path.join(tmpdir, ".metadata_cache")
+    with open(cache_path, 'w') as f:
+        f.write("bad pickle file")
+    extended_caching._save_cache(cache_path, {})
+    assert not os.path.exists(cache_path)
+
+
+def test_uncachable(settings, tmpdir):
+    cache_path = os.path.join(tmpdir, ".metadata_cache")
+    with open(cache_path, 'w') as f:
+        f.write("bad pickle file")
+    extended_caching._save_cache(cache_path, {'func': lambda x: x+1})
+    assert not os.path.exists(cache_path)