From bb7c752019de884df58bb11168f25973f10da778 Mon Sep 17 00:00:00 2001 From: Glandos Date: Thu, 15 Feb 2018 22:55:47 +0100 Subject: [PATCH] Add compress_assets plugin: - The plugin - Configuration - Myself as author - Docs - Tests - Dependencies for Travis --- .travis.yml | 2 +- AUTHORS | 1 + docs/plugins.rst | 5 + sigal/plugins/compress_assets.py | 174 +++++++++++++++++++++++++++ sigal/templates/sigal.conf.py | 5 + tests/test_compress_assets_plugin.py | 62 ++++++++++ 6 files changed, 248 insertions(+), 1 deletion(-) create mode 100644 sigal/plugins/compress_assets.py create mode 100644 tests/test_compress_assets_plugin.py diff --git a/.travis.yml b/.travis.yml index d5612b8..ca89df5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,7 +32,7 @@ addons: install: - pip install -U pip setuptools wheel - pip install -q $PILLOW - - pip install pytest pytest-cov codecov feedgenerator + - pip install pytest pytest-cov codecov feedgenerator zopfli brotli - pip install . before_script: diff --git a/AUTHORS b/AUTHORS index c8364b9..13916c8 100644 --- a/AUTHORS +++ b/AUTHORS @@ -15,6 +15,7 @@ alphabetical order): - David Siroky - Edwin Steele - François D. (@franek) +- @Glandos - Giel van Schijndel - Jamie Starke - @jdn06 diff --git a/docs/plugins.rst b/docs/plugins.rst index 221c9d3..275173d 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -89,6 +89,11 @@ Adjust plugin .. automodule:: sigal.plugins.adjust +Compress assets plugin +================ + +.. automodule:: sigal.plugins.compress_assets + Copyright plugin ================ diff --git a/sigal/plugins/compress_assets.py b/sigal/plugins/compress_assets.py new file mode 100644 index 0000000..fe9eb92 --- /dev/null +++ b/sigal/plugins/compress_assets.py @@ -0,0 +1,174 @@ +"""Plugin to compress static files for faster HTTP transfer. + +Your web server must be configured in order to select pre-compressed +files instead of the required one, to save CPU. For example, in nginx: + +:: + + server { + gzip_static; + } + +Currently, 3 methods are supported: + +- ``gzip``: No dependency required. + This is the fastest, but also largest output. +- ``zopfli``: Requires zopfli_. + gzip compatible output with optimized size. +- ``brotli``: Requires brotli_. + Brotli is the best compressor for web usage. + +.. _zopfli: https://pypi.python.org/pypi/zopfli +.. _brotli: https://pypi.python.org/pypi/Brotli + +Settings available as dictionary in ``compress_assets_options``: + +- ``method``: One of ``gzip``, ``zopfli`` or ``brotli``. +- ``suffixes``: List of file suffixes eligible to compression. + Default to ``['htm', 'html', 'css', 'js', 'svg']`` + +""" + +import logging +import gzip +import shutil +import os + +from sigal import signals +from click import progressbar + +logger = logging.getLogger(__name__) + +DEFAULT_SETTINGS = { + 'suffixes': ['htm', 'html', 'css', 'js', 'svg'], + 'method': 'gzip', + } + + +class BaseCompressor: + suffix = None + + def __init__(self, settings): + self.suffixes_to_compress = settings.get('suffixes', DEFAULT_SETTINGS['suffixes']) + + def do_compress(self, filename, compressed_filename): + ''' + Perform actual compression. + This should be implemented by subclasses. + ''' + raise NotImplementedError + + def compress(self, filename): + ''' + Compress a file, only if needed. + ''' + compressed_filename = self.get_compressed_filename(filename) + if not compressed_filename: + return + + self.do_compress(filename, compressed_filename) + + def get_compressed_filename(self, filename): + ''' + If the given filename should be compressed, returns the compressed filename. + A file can be compressed if: + - It is a whitelisted extension + - The compressed file does not exist + - The compressed file exists by is older than the file itself + Otherwise, it returns False. + ''' + if not os.path.splitext(filename)[1][1:] in self.suffixes_to_compress: + return False + + file_stats = None + compressed_stats = None + compressed_filename = u'{}.{}'.format(filename, self.suffix) + try: + file_stats = os.stat(filename) + compressed_stats = os.stat(compressed_filename) + except OSError: # FileNotFoundError is for Python3 only + pass + + if file_stats and compressed_stats: + return compressed_filename if file_stats.st_mtime > compressed_stats.st_mtime else False + else: + return compressed_filename + + +class GZipCompressor(BaseCompressor): + suffix = 'gz' + + def do_compress(self, filename, compressed_filename): + with open(filename, 'rb') as f_in, gzip.open(compressed_filename, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + + +class ZopfliCompressor(BaseCompressor): + suffix = 'gz' + + def do_compress(self, filename, compressed_filename): + import zopfli.gzip + with open(filename, 'rb') as f_in, open(compressed_filename, 'wb') as f_out: + f_out.write(zopfli.gzip.compress(f_in.read())) + + +class BrotliCompressor(BaseCompressor): + suffix = 'br' + + def do_compress(self, filename, compressed_filename): + import brotli + with open(filename, 'rb') as f_in, open(compressed_filename, 'wb') as f_out: + f_out.write(brotli.compress(f_in.read(), mode=brotli.MODE_TEXT)) + + +def get_compressor(settings): + name = settings.get('method', DEFAULT_SETTINGS['method']) + if name == 'gzip': + return GZipCompressor(settings) + elif name == 'zopfli': + try: + import zopfli.gzip + return ZopfliCompressor(settings) + except ImportError: + logging.error('Unable to import zopfli module') + + elif name == 'brotli': + try: + import brotli + return BrotliCompressor(settings) + except ImportError: + logger.error('Unable to import brotli module') + + else: + logger.error('No such compressor {}'.format(name)) + + +def compress_assets(assets_directory, compressor): + assets = [] + for current_directory, _, filenames in os.walk(assets_directory): + for filename in filenames: + assets.append(os.path.join(current_directory, filename)) + + with progressbar(assets, label='Compressing theme assets') as progress_compress: + for filename in progress_compress: + compressor.compress(filename) + + +def compress_gallery(gallery): + logging.info('Compressing assets for %s', gallery.title) + compress_settings = gallery.settings.get('compress_assets_options', DEFAULT_SETTINGS) + compressor = get_compressor(compress_settings) + + if compressor is None: + return + + with progressbar(gallery.albums.values(), label='Compressing albums static files') as progress_compress: + for album in progress_compress: + compressor.compress(os.path.join(album.dst_path, album.output_file)) + + compress_assets(os.path.join(gallery.settings['destination'], 'static'), compressor) + + +def register(settings): + if settings['write_html']: + signals.gallery_build.connect(compress_gallery) diff --git a/sigal/templates/sigal.conf.py b/sigal/templates/sigal.conf.py index 8fe1889..da5b2b8 100644 --- a/sigal/templates/sigal.conf.py +++ b/sigal/templates/sigal.conf.py @@ -254,3 +254,8 @@ ignore_files = [] # 'policy': 'public-read', # 'overwrite': False # } + +# Settings for compressing static assets +# compress_assets_options = { +# 'method': 'gzip' # Or 'zopfli' or 'brotli' +# } diff --git a/tests/test_compress_assets_plugin.py b/tests/test_compress_assets_plugin.py new file mode 100644 index 0000000..5e98ecf --- /dev/null +++ b/tests/test_compress_assets_plugin.py @@ -0,0 +1,62 @@ +# -*- coding:utf-8 -*- + +import os + +import blinker +import pytest + +from sigal import init_plugins, signals +from sigal.gallery import Gallery +from sigal.plugins import compress_assets + +CURRENT_DIR = os.path.dirname(__file__) + + +@pytest.fixture(autouse=True) +def disconnect_signals(): + yield None + for name in dir(signals): + if not name.startswith('_'): + try: + sig = getattr(signals, name) + if isinstance(sig, blinker.Signal): + sig.receivers.clear() + except Exception: + pass + + +def make_gallery(settings, tmpdir, method): + settings['destination'] = str(tmpdir) + if "sigal.plugins.compress_assets" not in settings["plugins"]: + settings['plugins'] += ["sigal.plugins.compress_assets"] + + # Set method + settings.setdefault('compress_assets_options', {})['method'] = method + + compress_options = compress_assets.DEFAULT_SETTINGS.copy() + # The key was created by the previous setdefault if needed + compress_options.update(settings['compress_assets_options']) + + init_plugins(settings) + gal = Gallery(settings) + gal.build() + + return compress_options + + +@pytest.mark.parametrize("method,compress_suffix,test_import", + [('gzip', 'gz', None), + ('zopfli', 'gz', 'zopfli.gzip'), + ('brotli', 'br', 'brotli')]) +def test_compress(settings, tmpdir, method, compress_suffix, test_import): + if test_import: + pytest.importorskip(test_import) + compress_options = make_gallery(settings, tmpdir, 'gzip') + + suffixes = compress_options['suffixes'] + + for path, dirs, files in os.walk(settings['destination']): + for file in files: + path_exists = os.path.exists('{}.{}'.format(os.path.join(path, file), 'gz')) + file_ext = os.path.splitext(file)[1][1:] + assert path_exists if file_ext in suffixes else not path_exists