From 6785d9379ea6902bb2abc3410fa4d95b92484147 Mon Sep 17 00:00:00 2001 From: user Date: Tue, 13 Mar 2018 13:40:47 -0300 Subject: [PATCH 1/7] Adding OptimizedPipelineStorage which speeds up the compressing process and appends a hash to the resources' URL in order to invalidate caching mechanisms automatically. --- pipeline/storage.py | 198 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 198 insertions(+) diff --git a/pipeline/storage.py b/pipeline/storage.py index c75d0ebd..62d7cea6 100644 --- a/pipeline/storage.py +++ b/pipeline/storage.py @@ -1,6 +1,8 @@ from __future__ import unicode_literals import gzip +import hashlib +import os from io import BytesIO @@ -98,3 +100,199 @@ class PipelineCachedStorage(PipelineMixin, CachedStaticFilesStorage): class NonPackagingPipelineCachedStorage(NonPackagingMixin, PipelineCachedStorage): pass + + +class OptimizedPipelineStorage(PipelineMixin, StaticFilesStorage): + """This storage compresses only the packages which had modifications in + their source files, or that have not been compressed yet. This speeds up + the collectstatic process, since must of the time we modify only a few + javascript/css files at a time. + + It also appends the a md5 hash to the compressed files' url so any existing + cache mechanisms are naturally invalidated.""" + + compressed_packages = [] + unchanged_packages = [] + packager = None + HASH_CACHE_KEY = 'pipeline_compressed_hash_key' + SOURCES_DUMP_KEY = 'pipeline_dumped_sources_key' + + def url(self, name): + """Append the produced hash to the resource url so existing cache + mechanisms are naturally invalidated.""" + url = super(OptimizedPipelineStorage, self).url(name) + _hash = self.get_compressed_files_hash() + return '{url}?{_hash}'.format(url=url, _hash=_hash) if _hash else url + + def post_process(self, paths, dry_run=False, **options): + if dry_run: + return + + from pipeline.packager import Packager + self.packager = Packager(storage=self) + + for package_name in self.packager.packages['css']: + package = self.packager.package_for('css', package_name) + output_file = package.output_filename + + if self.packing and self._is_outdated(package_name, package): + print('COMPRESSING {} package...'.format(package_name)) + self.packager.pack_stylesheets(package) + self.compressed_packages.append(package_name) + else: + self.unchanged_packages.append(package_name) + + paths[output_file] = (self, output_file) + yield output_file, output_file, True + + for package_name in self.packager.packages['js']: + package = self.packager.package_for('js', package_name) + output_file = package.output_filename + + if self.packing and self._is_outdated(package_name, package): + print('COMPRESSING {} package...'.format(package_name)) + self.packager.pack_javascripts(package) + self.compressed_packages.append(package_name) + else: + self.unchanged_packages.append(package_name) + + paths[output_file] = (self, output_file) + yield output_file, output_file, True + + super_class = super(PipelineMixin, self) + if hasattr(super_class, 'post_process'): + for name, hashed_name, processed in super_class.post_process( + paths.copy(), dry_run, **options): + yield name, hashed_name, processed + + self._finalize() + + def _is_outdated(self, package_name, package): + outdated = False + + for path in package.paths: + # Needs to run for every path in order to generate the individual + # file hashes. + if self._is_content_changed(path) and not outdated: + outdated = True + + if not outdated: + previous_paths = self._get_previous_compressed_sources(package_name) + if not previous_paths or set(previous_paths) != set(package.paths): + outdated = True + + from django.conf import settings + output_path = os.path.join(settings.STATIC_ROOT, package.output_filename) + return outdated or not os.path.exists(output_path) + + def _is_content_changed(self, path): + """Verifies if the content of :path change based on the hash that was + produced during the last collecstatic run.""" + from django.conf import settings + changed = True + infile_path = os.path.join(self.location, path) + outfile_path = os.path.join(settings.STATIC_ROOT, path) + infile_hash_path = outfile_path + '.hash' + + with open(infile_path) as infile_file: + current_hash = hashlib.md5(infile_file.read()).hexdigest() + + from django.core.cache import caches + DEFAULT_CACHE = caches['default'] + old_hash = DEFAULT_CACHE.get(infile_hash_path) + changed = current_hash != old_hash + DEFAULT_CACHE.set(infile_hash_path, current_hash, None) + return changed + + def _finalize(self): + self._dump_sources() + print('\n=== {} results ==='.format(self.__class__.__name__)) + total_removed = self._remove_sources() + self._write_hash() + print('{} removed files used in the compressing'.format(total_removed)) + print('{} new compressed packages: {}'.format( + len(self.compressed_packages), self.compressed_packages)) + print('{} unchanged packages: {}'.format( + len(self.unchanged_packages), self.unchanged_packages)) + print('=== End {} results ==='.format(self.__class__.__name__)) + + def _remove_sources(self): + """We do not want to expose our source files, thus they are removed + from the STATIC_ROOT directory, keeping only the compressed files.""" + from django.conf import settings + sources = [] + + for package_name in self.packager.packages['js']: + package = self.packager.package_for('js', package_name) + sources.extend(package.paths) + + for package_name in self.packager.packages['css']: + package = self.packager.package_for('css', package_name) + sources.extend(package.paths) + + removed = 0 + for source in sources: + source_path = os.path.join(settings.STATIC_ROOT, source) + if os.path.exists(source_path): + os.remove(source_path) + removed += 1 + + return removed + + def _dump_sources(self): + """We dump the list of compressed source files so we can compare if + there is any difference (new files or removed files) in the next + collectstatic run.""" + from django.core.cache import caches + DEFAULT_CACHE = caches['default'] + + packages = {} + + for package_name in self.packager.packages['js']: + package = self.packager.package_for('js', package_name) + packages[package_name] = package.paths + + for package_name in self.packager.packages['css']: + package = self.packager.package_for('css', package_name) + packages[package_name] = package.paths + # cache forever + DEFAULT_CACHE.set(self.SOURCES_DUMP_KEY, packages, None) + + def _get_previous_compressed_sources(self, package_name): + from django.core.cache import caches + DEFAULT_CACHE = caches['default'] + return DEFAULT_CACHE.get(self.SOURCES_DUMP_KEY, {}).\ + get(package_name) + + def _write_hash(self): + """Writes a single md5 hash considering all the content from the + source files. This is useful to force any cache mechanism to update + their registries.""" + from django.conf import settings + from django.core.cache import caches + DEFAULT_CACHE = caches['default'] + output_filenames = [] + + for package_name in self.packager.packages['js']: + package = self.packager.package_for('js', package_name) + output_filenames.append(package.output_filename) + + for package_name in self.packager.packages['css']: + package = self.packager.package_for('css', package_name) + output_filenames.append(package.output_filename) + + contents = [] + for output_filename in output_filenames: + abs_path = os.path.join(settings.STATIC_ROOT, output_filename) + with open(abs_path) as output_file: + contents.append(output_file.read()) + + digest = hashlib.md5(''.join(contents)).hexdigest() + print('New hash: {}'.format(digest)) + DEFAULT_CACHE.set(self.HASH_CACHE_KEY, digest, None) # cache forever + + @staticmethod + def get_compressed_files_hash(): + from django.core.cache import caches + DEFAULT_CACHE = caches['default'] + return DEFAULT_CACHE.get(OptimizedPipelineStorage.HASH_CACHE_KEY) From bced82c0f4760513373445b68c167265599a5257 Mon Sep 17 00:00:00 2001 From: user Date: Tue, 13 Mar 2018 13:53:53 -0300 Subject: [PATCH 2/7] Fixing unit test expected result. --- tests/assets/compressors/yuglify.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/assets/compressors/yuglify.js b/tests/assets/compressors/yuglify.js index 77ca92f1..c2d59b46 100644 --- a/tests/assets/compressors/yuglify.js +++ b/tests/assets/compressors/yuglify.js @@ -1 +1 @@ -(function(){(function(){window.concat=function(){console.log(arguments)}})(),function(){window.cat=function(){console.log("hello world")}}()}).call(this); +(function(){window.concat=function(){console.log(arguments)},window.cat=function(){console.log("hello world")}}).call(this) From 07ccad95024d262f3b29dc87072aa2e9060824f4 Mon Sep 17 00:00:00 2001 From: yguarata Date: Tue, 13 Mar 2018 20:56:37 -0300 Subject: [PATCH 3/7] Updating documentation with the new OptimizedPipelineStorage. --- docs/storages.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/storages.rst b/docs/storages.rst index c17d41a3..f461453e 100644 --- a/docs/storages.rst +++ b/docs/storages.rst @@ -26,6 +26,23 @@ Also available if you want versioning :: STATICFILES_STORAGE = 'pipeline.storage.NonPackagingPipelineCachedStorage' +Optimized storage with cache invalidation +----------------------------------------- + +There is also an optimized storage which compress only the packages with +modifications. This speeds up the collectstatic process substantially when you +have multiple packages in your pipeline configuration. + +In addition, this storage produces a hash from the compressed files which is +appended to the resources' URL. Thus, whenever you modify your files and +compress them, the cache registries (from cache server ou browser) will be +automatically invalidated :: + + STATICFILES_STORAGE = 'pipeline.storage.OptimizedPipelineStorage' + +File finders (staticfiles with DEBUG = False) +============================================= + If you use staticfiles with ``DEBUG = False`` (i.e. for integration tests with `Selenium `_) you should install the finder that allows staticfiles to locate your outputted assets : :: From 5996d6adeb556f10e67992e5627a0cbf86ed875f Mon Sep 17 00:00:00 2001 From: user Date: Wed, 14 Mar 2018 12:22:01 -0300 Subject: [PATCH 4/7] Adding templatetags for prefetching resources. --- docs/usage.rst | 30 +++++++++++++- pipeline/templates/pipeline/prefetch.html | 3 ++ pipeline/templatetags/pipeline.py | 50 +++++++++++++++++++++++ 3 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 pipeline/templates/pipeline/prefetch.html diff --git a/docs/usage.rst b/docs/usage.rst index f453a93b..b8ccec89 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -9,7 +9,7 @@ Describes how to use Pipeline when it is installed and configured. Templatetags ============ -Pipeline includes two template tags: ``stylesheet`` and ``javascript``, +Pipeline includes two main template tags: ``stylesheet`` and ``javascript``, in a template library called ``pipeline``. They are used to output the ```` and ``