diff --git a/.gitignore b/.gitignore index d60c9c29..f9930b5e 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,6 @@ *.~* *.pyc + +# OS X +.DS_Store diff --git a/docs/external_assets.rst b/docs/external_assets.rst new file mode 100644 index 00000000..426c5c14 --- /dev/null +++ b/docs/external_assets.rst @@ -0,0 +1,46 @@ +.. _external_assets: + +=============== +External Assets +=============== + +An external assets bundle is used to manage images, webfonts and other assets +that you wouldn't normally include in another bundle. Files will have a cache +buster applied (see :doc:`URL Expiry `), and the +:ref:`cssrewrite ` filter can modify css files to point to +the versioned filenames. + + +Registering external files +-------------------------- + +An external assets bundle takes any number of input patterns and one output +directory. + +.. code-block:: python + + ExternalAssets('images/*', 'more_images/*', output='versioned_images') + +The output directory is relative to the ``directory`` setting of your +:doc:`environment `. All files found matching the input patterns +will be copied (with rewritten filenames) to this directory. + + +Using rewritten files +--------------------- + +CSS files using the :ref:`cssrewrite ` filter will be +automatically adapted to use the versioned filenames. + +.. code-block:: python + + Bundle('style.css', filters=['cssrewrite']) + + +If you need to get the specific url for a file, you can request it from the +bundle directly using :meth:`ExternalAssets.url` directly. + +.. code-block:: python + + >>> env['images'].url('logo.png') + /static/logo.c49de0ce.png diff --git a/docs/generic/index.rst b/docs/generic/index.rst index 9279b1db..6decab63 100644 --- a/docs/generic/index.rst +++ b/docs/generic/index.rst @@ -89,6 +89,7 @@ Further Reading /environment /bundles + /external_assets /script /builtin_filters /custom_filters diff --git a/docs/index.rst b/docs/index.rst index 4a98e987..6e8ddc32 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -23,6 +23,7 @@ of framework used: environment bundles + external_assets script builtin_filters expiring diff --git a/src/webassets/__init__.py b/src/webassets/__init__.py index eb604dc9..929ebfe0 100644 --- a/src/webassets/__init__.py +++ b/src/webassets/__init__.py @@ -1,6 +1,7 @@ __version__ = (0, 8, 'dev') -# Make a couple frequently used things available right here. +# Make a few frequently used things available right here. from bundle import Bundle +from external import ExternalAssets from env import Environment diff --git a/src/webassets/bundle.py b/src/webassets/bundle.py index 4f2147b0..a69240f3 100644 --- a/src/webassets/bundle.py +++ b/src/webassets/bundle.py @@ -6,6 +6,7 @@ from merge import (FileHunk, UrlHunk, FilterTool, merge, merge_filters, select_filters, MoreThanOneFilterError) from updater import SKIP_CACHE +from container import Container, has_placeholder, is_url from exceptions import BundleError, BuildError from utils import cmp_debug_levels @@ -13,18 +14,7 @@ __all__ = ('Bundle', 'get_all_bundle_files',) -def is_url(s): - if not isinstance(s, str): - return False - parsed = urlparse.urlsplit(s) - return bool(parsed.scheme and parsed.netloc) and len(parsed.scheme) > 1 - - -def has_placeholder(s): - return '%(version)s' in s - - -class Bundle(object): +class Bundle(Container): """A bundle is the unit webassets uses to organize groups of media files, which filters to apply and where to store them. @@ -46,6 +36,7 @@ class Bundle(object): """ def __init__(self, *contents, **options): + super(Container, self).__init__() self.env = None self.contents = contents self.output = options.pop('output', None) @@ -86,13 +77,6 @@ def _set_filters(self, value): self._filters = [get_filter(f) for f in filters] filters = property(_get_filters, _set_filters) - def _get_contents(self): - return self._contents - def _set_contents(self, value): - self._contents = value - self._resolved_contents = None - contents = property(_get_contents, _set_contents) - def _get_extra(self): if not self._extra and not has_files(self): # If this bundle has no extra values of it's own, and only @@ -111,61 +95,6 @@ def _set_extra(self, value): template tags, and can be used to attach things like a CSS 'media' value.""") - def resolve_contents(self, env=None, force=False): - """Return an actual list of source files. - - What the user specifies as the bundle contents cannot be - processed directly. There may be glob patterns of course. We - may need to search the load path. It's common for third party - extensions to provide support for referencing assets spread - across multiple directories. - - This passes everything through :class:`Environment.resolver`, - through which this process can be customized. - - At this point, we also validate source paths to complain about - missing files early. - - The return value is a list of 2-tuples ``(original_item, - abspath)``. In the case of urls and nested bundles both tuple - values are the same. - - Set ``force`` to ignore any cache, and always re-resolve - glob patterns. - """ - env = self._get_env(env) - - # TODO: We cache the values, which in theory is problematic, since - # due to changes in the env object, the result of the globbing may - # change. Not to mention that a different env object may be passed - # in. We should find a fix for this. - if getattr(self, '_resolved_contents', None) is None or force: - resolved = [] - for item in self.contents: - try: - result = env.resolver.resolve_source(item) - except IOError, e: - raise BundleError(e) - if not isinstance(result, list): - result = [result] - - # Exclude the output file. - # TODO: This will not work for nested bundle contents. If it - # doesn't work properly anyway, should be do it in the first - # place? If there are multiple versions, it will fail as well. - # TODO: There is also the question whether we can/should - # exclude glob duplicates. - if self.output: - try: - result.remove(self.resolve_output(env)) - except (ValueError, BundleError): - pass - - resolved.extend(map(lambda r: (item, r), result)) - - self._resolved_contents = resolved - return self._resolved_contents - def _get_depends(self): return self._depends def _set_depends(self, value): @@ -228,17 +157,6 @@ def get_version(self, env=None, refresh=False): self.version = version return self.version - def resolve_output(self, env=None, version=None): - """Return the full, absolute output path. - - If a %(version)s placeholder is used, it is replaced. - """ - env = self._get_env(env) - output = env.resolver.resolve_output_to_path(self.output, self) - if has_placeholder(output): - output = output % {'version': version or self.get_version(env)} - return output - def __hash__(self): """This is used to determine when a bundle definition has changed so that a rebuild is required. diff --git a/src/webassets/container.py b/src/webassets/container.py new file mode 100644 index 00000000..ed6e5e95 --- /dev/null +++ b/src/webassets/container.py @@ -0,0 +1,105 @@ +import urlparse +import os +from exceptions import ContainerError, BundleError + +try: + # Current version of glob2 does not let us access has_magic :/ + import glob2 as glob + from glob import has_magic +except ImportError: + import glob + from glob import has_magic + +def has_placeholder(s): + return '%(version)s' in s + +def is_url(s): + if not isinstance(s, str): + return False + parsed = urlparse.urlsplit(s) + return bool(parsed.scheme and parsed.netloc) and len(parsed.scheme) > 1 + +class Container(object): + + def __init__(self): + pass + + def _get_contents(self): + return self._contents + def _set_contents(self, value): + self._contents = value + self._resolved_contents = None + contents = property(_get_contents, _set_contents) + + def _get_env(self, env): + # Note how bool(env) can be False, due to __len__. + env = env if env is not None else self.env + if env is None: + raise ContainerError('Container not connected to an environment') + return env + + def resolve_output(self, env=None, version=None): + """Return the full, absolute output path. + + If a %(version)s placeholder is used, it is replaced. + """ + env = self._get_env(env) + output = env.resolver.resolve_output_to_path(self.output, self) + if has_placeholder(output): + output = output % {'version': version or self.get_version(env)} + return output + + def resolve_contents(self, env=None, force=False): + """Return an actual list of source files. + + What the user specifies as the bundle contents cannot be + processed directly. There may be glob patterns of course. We + may need to search the load path. It's common for third party + extensions to provide support for referencing assets spread + across multiple directories. + + This passes everything through :class:`Environment.resolver`, + through which this process can be customized. + + At this point, we also validate source paths to complain about + missing files early. + + The return value is a list of 2-tuples ``(original_item, + abspath)``. In the case of urls and nested bundles both tuple + values are the same. + + Set ``force`` to ignore any cache, and always re-resolve + glob patterns. + """ + env = self._get_env(env) + + # TODO: We cache the values, which in theory is problematic, since + # due to changes in the env object, the result of the globbing may + # change. Not to mention that a different env object may be passed + # in. We should find a fix for this. + if getattr(self, '_resolved_contents', None) is None or force: + resolved = [] + for item in self.contents: + try: + result = env.resolver.resolve_source(item) + except IOError, e: + raise BundleError(e) + if not isinstance(result, list): + result = [result] + + # Exclude the output file. + # TODO: This will not work for nested bundle contents. If it + # doesn't work properly anyway, should be do it in the first + # place? If there are multiple versions, it will fail as well. + # TODO: There is also the question whether we can/should + # exclude glob duplicates. + if self.output: + try: + result.remove(self.resolve_output(env)) + except (ValueError, BundleError): + pass + + resolved.extend(map(lambda r: (item, r), result)) + + self._resolved_contents = resolved + return self._resolved_contents diff --git a/src/webassets/env.py b/src/webassets/env.py index 8b3ecc9e..0929a2de 100644 --- a/src/webassets/env.py +++ b/src/webassets/env.py @@ -2,6 +2,10 @@ import urlparse from itertools import chain import warnings + +from bundle import Bundle, is_url +from external import ExternalAssets + try: import glob2 as glob from glob import has_magic @@ -9,7 +13,6 @@ import glob from glob import has_magic -from bundle import Bundle, is_url from cache import get_cache from version import get_versioner, get_manifest from updater import get_updater @@ -267,6 +270,17 @@ def resolve_source(self, item): return self.search_for_source(item) + def resolve_source_to_path(self, file_name): + """Given ``item`` from a Bundle's contents, this has to + return the final value to use, usually an absolute + filesystem path. Unlike :meth:`search_for_source` this + will only return the first matching path it finds. + """ + source = self.resolve_source(file_name) + if isinstance(source, list): + return source[0] + return source + def resolve_output_to_path(self, target, bundle): """Given ``target``, this has to return the absolute filesystem path to which the output file of ``bundle`` @@ -339,6 +353,7 @@ class BaseEnvironment(object): def __init__(self, **config): self._named_bundles = {} self._anon_bundles = [] + self.external_assets = None self._config = self.config_storage_class(self) self.resolver = self.resolver_class(self) @@ -406,7 +421,7 @@ def register(self, name, *args, **kwargs): if len(args) == 0: raise TypeError('at least two arguments are required') else: - if len(args) == 1 and not kwargs and isinstance(args[0], Bundle): + if len(args) == 1 and not kwargs and (isinstance(args[0], Bundle) or isinstance(args[0], ExternalAssets)): bundle = args[0] else: bundle = Bundle(*args, **kwargs) diff --git a/src/webassets/exceptions.py b/src/webassets/exceptions.py index a642c966..6c9d2229 100644 --- a/src/webassets/exceptions.py +++ b/src/webassets/exceptions.py @@ -1,11 +1,19 @@ -__all__ = ('BundleError', 'BuildError', 'FilterError', - 'EnvironmentError', 'ImminentDeprecationWarning') +__all__ = ('BundleError', 'BuildError', 'ContainerError', 'FilterError', + 'EnvironmentError', 'ExternalAssetsError', 'ImminentDeprecationWarning', ) class EnvironmentError(Exception): pass +class ExternalAssetsError(Exception): + pass + + +class ContainerError(Exception): + pass + + class BundleError(Exception): pass diff --git a/src/webassets/ext/jinja2.py b/src/webassets/ext/jinja2.py index d6a041fa..a8f71346 100644 --- a/src/webassets/ext/jinja2.py +++ b/src/webassets/ext/jinja2.py @@ -4,12 +4,26 @@ import jinja2 from jinja2.ext import Extension from jinja2 import nodes -from webassets import Bundle +from jinja2.utils import environmentfunction +from webassets import Bundle, ExternalAssets from webassets.loaders import GlobLoader, LoaderError from webassets.exceptions import ImminentDeprecationWarning -__all__ = ('assets', 'Jinja2Loader',) +__all__ = ('assets', 'Jinja2Loader', 'webasset_tag') + + +@environmentfunction +def webasset_tag(env, file_path): + for bundle in env.assets_environment: + if isinstance(bundle, ExternalAssets): + # see if our file has a versioned version available + try: + asset_path = bundle.versioned_folder(file_path) + return env.assets_environment.resolver.resolve_output_to_url(asset_path) + break + except IOError: + return file_path class AssetsExtension(Extension): diff --git a/src/webassets/external.py b/src/webassets/external.py new file mode 100644 index 00000000..b17f594f --- /dev/null +++ b/src/webassets/external.py @@ -0,0 +1,100 @@ +import os +from os import path +from merge import FileHunk + +from exceptions import ExternalAssetsError, BuildError +from container import Container + +__all__ = ('ExternalAssets',) + + +class ExternalAssets(Container): + + def __init__(self, *contents, **options): + super(Container, self).__init__() + self.env = None + self.contents = contents + self.output = options.pop('output', None) + if options: + raise TypeError("got unexpected keyword argument '%s'" % + options.keys()[0]) + self.extra_data = {} + + def __repr__(self): + return "<%s folders=%s>" % ( + self.__class__.__name__, + self.contents, + ) + + def get_versioned_file(self, file_name): + version = self.get_version(file_name) + bits = file_name.split('.') + bits.insert(len(bits) - 1, version) + return '.'.join(bits) + + def versioned_folder(self, file_name): + if self.output: + output_folder = self.output + else: + output_folder = self.env.config.get('external_assets_output_folder', None) + if output_folder is None: + raise ExternalAssetsError( + 'You must set an output folder for these ExternalAssets ' + 'or the external_assets_output_folder config value') + versioned = self.get_versioned_file(file_name) + return path.join(output_folder, path.basename(versioned)) + + def get_resolved_path(self, file_name): + return self.env.resolver.resolve_source(file_name) + + def write_file(self, file_name): + hunk = FileHunk(file_name) + output_path = path.join(self.env.directory, self.versioned_folder(file_name)) + output_dir = path.dirname(output_path) + if not path.exists(output_dir): + os.makedirs(output_dir) + hunk.save(output_path) + if self.env.manifest: + self.env.manifest.remember_file(file_name, self.env, self.get_version(file_name)) + + def write_files(self, external_assets_path): + resolved_paths = self.get_resolved_path(external_assets_path) + if type(resolved_paths) is not list: + resolved_paths = [resolved_paths] + for path in resolved_paths: + self.write_file(path) + + def build(self, env=None, force=None, disable_cache=None): + # Prepare contents + resolved_contents = self.resolve_contents(env, force=True) + if not resolved_contents: + raise BuildError('empty external assets cannot be built') + for path in self.contents: + self.write_files(path) + + def show_manifest(self): + if self.env.manifest: + print self.env.manifest.get_manifest() + + def url(self, file_name): + versioned = self.versioned_folder(file_name) + url = self.env.resolver.resolve_output_to_url(versioned) + file_path = self.env.resolver.resolve_source_to_path(file_name) + versioned_path = self.env.resolver.resolve_output_to_path(versioned, self) + if not path.exists(versioned_path): + self.write_file(file_path) + return url + + def get_version(self, file_name): + version = None + if self.env.manifest: + version = self.env.manifest.query_file(file_name, self.env) + if version is None: + version = self.env.versions.determine_file_version(file_name, self.env) + return version + + @property + def is_container(self): + """ExternalAssets cannot be containers + """ + return False diff --git a/src/webassets/filter/cssrewrite/__init__.py b/src/webassets/filter/cssrewrite/__init__.py index f10d7a67..0a16f3ee 100644 --- a/src/webassets/filter/cssrewrite/__init__.py +++ b/src/webassets/filter/cssrewrite/__init__.py @@ -1,6 +1,7 @@ import os, urlparse from os.path import join from webassets.utils import common_path_prefix +from webassets.external import ExternalAssets import urlpath try: from collections import OrderedDict @@ -30,7 +31,7 @@ class CSSRewrite(CSSUrlRewriter): No configuration is necessary. - The filter also supports a manual mode:: + The filter also supports a manual mode, using either ``replace`` or ``external``:: get_filter('cssrewrite', replace={'old_directory':'/custom/path/'}) @@ -53,6 +54,7 @@ class CSSRewrite(CSSUrlRewriter): def __init__(self, replace=False): super(CSSRewrite, self).__init__() self.replace = replace + self.external = [] def unique(self): # Allow mixing the standard version of this filter, and replace mode. @@ -72,8 +74,17 @@ def input(self, _in, out, **kw): replace_dict[replurl] = sub self.replace_dict = replace_dict + # see if we have external assets in the environment + for bundle in self.env: + if isinstance(bundle, ExternalAssets): + self.external.append(bundle) + #pass + return super(CSSRewrite, self).input(_in, out, **kw) + def _is_abs_url(self, url): + return url.startswith('/') and (url.startswith('http://') or url.startswith('https://')) + def replace_url(self, url): # Replace mode: manually adjust the location of files if callable(self.replace): @@ -86,13 +97,49 @@ def replace_url(self, url): # Only apply the first match break - # Default mode: auto correct relative urls else: - # If path is an absolute one, keep it - if not url.startswith('/') and not (url.startswith('http://') or url.startswith('https://')): - # rewritten url: relative path from new location (output) - # to location of referenced file (source + current url) - url = urlpath.relpath(self.output_url, - urlparse.urljoin(self.source_url, url)) - - return url \ No newline at end of file + # External mode: use ExternalAssets objects + # to fetch replacements + if len(self.external): + # If path is an absolute one, keep it + if not self._is_abs_url(url): + + replacement = None + + file_path = urlpath.pathjoin(self.source_path, url) + asset_path = None + for external_assets in self.external: + # see if our file has a versioned version available + try: + asset_path = external_assets.versioned_folder(file_path) + break + except IOError: + pass + + if asset_path is not None: + if self.env.url: + # see if it's a complete url (rather than a folder) + # otherwise we want a relative path in the CSS + if self._is_abs_url(self.env.url): + replacement = urlparse.urljoin(self.env.url, asset_path) + else: + replacement = urlpath.relpathto(self.env.directory, self.output_path, asset_path) + else: + replacement = urlpath.relpathto(self.env.directory, self.output_path, asset_path) + + if replacement is None: + url = urlpath.relpath(self.output_url, + urlparse.urljoin(self.source_url, url)) + else: + url = replacement + + else: + # Default mode: auto correct relative urls + # If path is an absolute one, keep it + if not self._is_abs_url(url): + # rewritten url: relative path from new location (output) + # to location of referenced file (source + current url) + url = urlpath.relpath(self.output_url, + urlparse.urljoin(self.source_url, url)) + + return url diff --git a/src/webassets/merge.py b/src/webassets/merge.py index f10b6e73..24e32876 100644 --- a/src/webassets/merge.py +++ b/src/webassets/merge.py @@ -70,6 +70,13 @@ def __repr__(self): def mtime(self): pass + def save(self, filename): + f = open(filename, 'wb') + try: + f.write(self.data()) + finally: + f.close() + def data(self): f = open(self.filename, 'rb') try: diff --git a/src/webassets/script.py b/src/webassets/script.py index ff56539d..80b27b4d 100644 --- a/src/webassets/script.py +++ b/src/webassets/script.py @@ -8,6 +8,7 @@ from webassets.exceptions import BuildError from webassets.updater import TimestampUpdater from webassets.merge import MemoryHunk +from webassets.external import ExternalAssets from webassets.version import get_manifest from webassets.cache import FilesystemCache from webassets.utils import set, StringIO @@ -117,14 +118,18 @@ def __call__(self, bundles=None, output=None, directory=None, no_cache=None, raise CommandError( 'I do not know a bundle name named "%s".' % name) - # Make a list of bundles to build, and the filename to write to. + # Make a list of all the named bundles to build, and the filename to write to. + # TODO: It's not ok to use an internal property here. + bundles = [(n,b) for n, b in self.environment._named_bundles.items()] + if bundle_names: - # TODO: It's not ok to use an internal property here. - bundles = [(n,b) for n, b in self.environment._named_bundles.items() - if n in bundle_names] + # if bundle names have been specified, filter by those + bundles = [(n,b) for n, b in bundles + if n in bundle_names] else: - # Includes unnamed bundles as well. - bundles = [(None, b) for b in self.environment] + # Otherwise include unnamed bundles as well, if not already in + bundles = bundles + [(None, b) for b in self.environment + if b not in [b for (n, b) in bundles]] # Determine common prefix for use with ``directory`` option. if directory: @@ -157,13 +162,20 @@ def __call__(self, bundles=None, output=None, directory=None, no_cache=None, # Build. built = [] for bundle, overwrite_filename, name in to_build: + if isinstance(bundle, ExternalAssets): + bundle_type = 'external assets' + output_destination = bundle.output or\ + self.environment.config.get('external_assets_output_folder', None) + else: + bundle_type = 'bundle' + output_destination = overwrite_filename or bundle.output if name: - # A name is not necessary available of the bundle was + # A name is not necessarily available if the bundle was # registered without one. - self.log.info("Building bundle: %s (to %s)" % ( - name, overwrite_filename or bundle.output)) + self.log.info("Building %s: %s (to %s)" % ( + bundle_type, name, output_destination)) else: - self.log.info("Building bundle: %s" % bundle.output) + self.log.info("Building %s: %s" % (bundle_type, output_destination)) try: if not overwrite_filename: @@ -188,6 +200,7 @@ def __call__(self, bundles=None, output=None, directory=None, no_cache=None, built.append(bundle) except BuildError, e: self.log.error("Failed, error was: %s" % e) + if len(built): self.event_handlers['post_build']() if len(built) != len(to_build): diff --git a/src/webassets/version.py b/src/webassets/version.py index fee1a1af..d09b8e39 100644 --- a/src/webassets/version.py +++ b/src/webassets/version.py @@ -7,8 +7,8 @@ import os import pickle - -from webassets.bundle import has_placeholder, is_url, get_all_bundle_files +from webassets.container import has_placeholder, is_url +from webassets.bundle import get_all_bundle_files from webassets.merge import FileHunk from webassets.utils import md5_constructor, RegistryMetaclass @@ -42,6 +42,9 @@ class Version(object): clazz=lambda: Version, attribute='determine_version', desc='a version implementation') + def determine_file_version(self, file_name, env=None): + raise NotImplementedError() + def determine_version(self, bundle, hunk=None, env=None): """Return a string that represents the current version of the given bundle. @@ -153,6 +156,12 @@ def __init__(self, length=8, hash=md5_constructor): self.length = length self.hasher = hash + def determine_file_version(self, file_name, env): + hunk = FileHunk(env.resolver.resolve_source_to_path(file_name)) + hasher = self.hasher() + hasher.update(hunk.data()) + return hasher.hexdigest()[:self.length] + def determine_version(self, bundle, env, hunk=None): if not hunk: if not has_placeholder(bundle.output): @@ -209,6 +218,11 @@ def remember(self, bundle, env, version): def query(self, bundle, env): raise NotImplementedError() + def remember_file(self, file_name, env, version): + raise NotImplementedError() + + def query_file(self, file_name, env): + raise NotImplementedError() get_manifest = Manifest.resolve @@ -246,6 +260,19 @@ def query(self, bundle, env): self._load_manifest() return self.manifest.get(bundle.output, None) + def remember_file(self, file_name, env, version): + self.manifest[file_name] = version + self._save_manifest() + + def query_file(self, file_name, env): + if env.auto_build: + self._load_manifest() + return self.manifest.get(file_name, None) + + def get_manifest(self): + self._load_manifest() + return self.manifest + def _load_manifest(self): if os.path.exists(self.filename): with open(self.filename, 'rb') as f: