diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 67122c0..0000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "pelican-plugins"] - path = pelican-plugins - url = https://github.com/getpelican/pelican-plugins diff --git a/pelican-plugins b/pelican-plugins deleted file mode 160000 index 5c5f965..0000000 --- a/pelican-plugins +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 5c5f965c984d9cb9324c1c7b251d448992c4bda5 diff --git a/pelicanconf.py b/pelicanconf.py index ca55adc..3fc3922 100644 --- a/pelicanconf.py +++ b/pelicanconf.py @@ -24,6 +24,23 @@ DEFAULT_LANG = 'fr' # theme THEME = "./theme/" +# Plugins +PLUGIN_PATHS = ["plugins"] +PLUGINS = [ + 'i18n_subsites', + # 'pdf-img', +] +# Readers +READERS = {"html": None} +# Everythings in french +JINJA_ENVIRONMENT = {'extensions': ['jinja2.ext.i18n']} +# Default theme language. +I18N_TEMPLATES_LANG = "en" +# Your language. +DEFAULT_LANG = "fr" +OG_LOCALE = "fr" +LOCALE = ("fr", "fr_FR.utf8") + USE_GOOGLE_FONTS = False # Pages, articles and static diff --git a/plugins/i18n_subsites/README.rst b/plugins/i18n_subsites/README.rst new file mode 100644 index 0000000..340109b --- /dev/null +++ b/plugins/i18n_subsites/README.rst @@ -0,0 +1,165 @@ +======================= + I18N Sub-sites Plugin +======================= + +This plugin extends the translations functionality by creating +internationalized sub-sites for the default site. + +This plugin is designed for Pelican 3.4 and later. + +What it does +============ + +1. When the content of the main site is being generated, the settings + are saved and the generation stops when content is ready to be + written. While reading source files and generating content objects, + the output queue is modified in certain ways: + + - translations that will appear as native in a different (sub-)site + will be removed + - untranslated articles will be transformed to drafts if + ``I18N_UNTRANSLATED_ARTICLES`` is ``'hide'`` (default), removed if + ``'remove'`` or kept as they are if ``'keep'``. + - untranslated pages will be transformed into hidden pages if + ``I18N_UNTRANSLATED_PAGES`` is ``'hide'`` (default), removed if + ``'remove'`` or kept as they are if ``'keep'``.'' + - additional content manipulation similar to articles and pages can + be specified for custom generators in the ``I18N_GENERATOR_INFO`` + setting. + +2. For each language specified in the ``I18N_SUBSITES`` dictionary the + settings overrides are applied to the settings from the main site + and a new sub-site is generated in the same way as with the main + site until content is ready to be written. +3. When all (sub-)sites are waiting for content writing, all removed + contents, translations and static files are interlinked across the + (sub-)sites. +4. Finally, all the output is written. + +Setting it up +============= + +For each extra used language code, a language-specific settings overrides +dictionary must be given (but can be empty) in the ``I18N_SUBSITES`` dictionary + +.. code-block:: python + + PLUGINS = ['i18n_subsites', ...] + + # mapping: language_code -> settings_overrides_dict + I18N_SUBSITES = { + 'cz': { + 'SITENAME': 'Hezkej blog', + } + } + +You must also have the following in your pelican configuration + +.. code-block:: python + JINJA_ENVIRONMENT = { + 'extensions': ['jinja2.ext.i18n'], + } + + + +Default and special overrides +----------------------------- +The settings overrides may contain arbitrary settings, however, there +are some that are handled in a special way: + +``SITEURL`` + Any overrides to this setting should ensure that there is some level + of hierarchy between all (sub-)sites, because Pelican makes all URLs + relative to ``SITEURL`` and the plugin can only cross-link between + the sites using this hierarchy. For instance, with the main site + ``http://example.com`` a sub-site ``http://example.com/de`` will + work, but ``http://de.example.com`` will not. If not overridden, the + language code (the language identifier used in the ``lang`` + metadata) is appended to the main ``SITEURL`` for each sub-site. +``OUTPUT_PATH``, ``CACHE_PATH`` + If not overridden, the language code is appended as with ``SITEURL``. + Separate cache paths are required as parser results depend on the locale. +``STATIC_PATHS``, ``THEME_STATIC_PATHS`` + If not overridden, they are set to ``[]`` and all links to static + files are cross-linked to the main site. +``THEME``, ``THEME_STATIC_DIR`` + If overridden, the logic with ``THEME_STATIC_PATHS`` does not apply. +``DEFAULT_LANG`` + This should not be overridden as the plugin changes it to the + language code of each sub-site to change what is perceived as translations. + +Localizing templates +-------------------- + +Most importantly, this plugin can use localized templates for each +sub-site. There are two approaches to having the templates localized: + +- You can set a different ``THEME`` override for each language in + ``I18N_SUBSITES``, e.g. by making a copy of a theme ``my_theme`` to + ``my_theme_lang`` and then editing the templates in the new + localized theme. This approach means you don't have to deal with + gettext ``*.po`` files, but it is harder to maintain over time. +- You use only one theme and localize the templates using the + `jinja2.ext.i18n Jinja2 extension + `_. For a kickstart + read this `guide <./localizing_using_jinja2.rst>`_. + +Additional context variables +............................ + +It may be convenient to add language buttons to your theme in addition +to the translation links of articles and pages. These buttons could, +for example, point to the ``SITEURL`` of each (sub-)site. For this +reason the plugin adds these variables to the template context: + +``main_lang`` + The language of the main site — the original ``DEFAULT_LANG`` +``main_siteurl`` + The ``SITEURL`` of the main site — the original ``SITEURL`` +``lang_siteurls`` + An ordered dictionary, mapping all used languages to their + ``SITEURL``. The ``main_lang`` is the first key with ``main_siteurl`` + as the value. This dictionary is useful for implementing global + language buttons that show the language of the currently viewed + (sub-)site too. +``extra_siteurls`` + An ordered dictionary, subset of ``lang_siteurls``, the current + ``DEFAULT_LANG`` of the rendered (sub-)site is not included, so for + each (sub-)site ``set(extra_siteurls) == set(lang_siteurls) - + set([DEFAULT_LANG])``. This dictionary is useful for implementing + global language buttons that do not show the current language. +``relpath_to_site`` + A function that returns a relative path from the first (sub-)site to + the second (sub-)site where the (sub-)sites are identified by the + language codes given as two arguments. + +If you don't like the default ordering of the ordered dictionaries, +use a Jinja2 filter to alter the ordering. + +All the siteurls above are always absolute even in the case of +``RELATIVE_URLS == True`` (it would be to complicated to replicate the +Pelican internals for local siteurls), so you may rather use something +like ``{{ SITEURL }}/{{ relpath_to_site(DEFAULT_LANG, main_lang }}`` +to link to the main site. + +This short `howto <./implementing_language_buttons.rst>`_ shows two +example implementations of language buttons. + +Usage notes +=========== +- It is **mandatory** to specify ``lang`` metadata for each article + and page as ``DEFAULT_LANG`` is later changed for each sub-site, so + content without ``lang`` metadata would be rendered in every + (sub-)site. +- As with the original translations functionality, ``slug`` metadata + is used to group translations. It is therefore often convenient to + compensate for this by overriding the content URL (which defaults to + slug) using the ``url`` and ``save_as`` metadata. You could also + give articles e.g. ``name`` metadata and use it in ``ARTICLE_URL = + '{name}.html'``. + +Development +=========== + +- A demo and a test site is in the ``gh-pages`` branch and can be seen + at http://smartass101.github.io/pelican-plugins/ diff --git a/plugins/i18n_subsites/__init__.py b/plugins/i18n_subsites/__init__.py new file mode 100644 index 0000000..7dfbde0 --- /dev/null +++ b/plugins/i18n_subsites/__init__.py @@ -0,0 +1 @@ +from .i18n_subsites import * diff --git a/plugins/i18n_subsites/i18n_subsites.py b/plugins/i18n_subsites/i18n_subsites.py new file mode 100644 index 0000000..dc27799 --- /dev/null +++ b/plugins/i18n_subsites/i18n_subsites.py @@ -0,0 +1,462 @@ +"""i18n_subsites plugin creates i18n-ized subsites of the default site + +This plugin is designed for Pelican 3.4 and later +""" + + +import os +import six +import logging +import posixpath + +from copy import copy +from itertools import chain +from operator import attrgetter +try: + from collections.abc import OrderedDict +except ImportError: + from collections import OrderedDict +from contextlib import contextmanager +from six.moves.urllib.parse import urlparse + +import gettext +import locale + +from pelican import signals +from pelican.generators import ArticlesGenerator, PagesGenerator +from pelican.settings import configure_settings +try: + from pelican.contents import Draft +except ImportError: + from pelican.contents import Article as Draft + + +# Global vars +_MAIN_SETTINGS = None # settings dict of the main Pelican instance +_MAIN_LANG = None # lang of the main Pelican instance +_MAIN_SITEURL = None # siteurl of the main Pelican instance +_MAIN_STATIC_FILES = None # list of Static instances the main Pelican instance +_SUBSITE_QUEUE = {} # map: lang -> settings overrides +_SITE_DB = OrderedDict() # OrderedDict: lang -> siteurl +_SITES_RELPATH_DB = {} # map: (lang, base_lang) -> relpath +# map: generator -> list of removed contents that need interlinking +_GENERATOR_DB = {} +_NATIVE_CONTENT_URL_DB = {} # map: source_path -> content in its native lang +_LOGGER = logging.getLogger(__name__) + + +@contextmanager +def temporary_locale(temp_locale=None): + '''Enable code to run in a context with a temporary locale + + Resets the locale back when exiting context. + Can set a temporary locale if provided + ''' + orig_locale = locale.setlocale(locale.LC_ALL) + if temp_locale is not None: + locale.setlocale(locale.LC_ALL, temp_locale) + yield + locale.setlocale(locale.LC_ALL, orig_locale) + + +def initialize_dbs(settings): + '''Initialize internal DBs using the Pelican settings dict + + This clears the DBs for e.g. autoreload mode to work + ''' + global _MAIN_SETTINGS, _MAIN_SITEURL, _MAIN_LANG, _SUBSITE_QUEUE + _MAIN_SETTINGS = settings + _MAIN_LANG = settings['DEFAULT_LANG'] + _MAIN_SITEURL = settings['SITEURL'] + _SUBSITE_QUEUE = settings.get('I18N_SUBSITES', {}).copy() + prepare_site_db_and_overrides() + # clear databases in case of autoreload mode + _SITES_RELPATH_DB.clear() + _NATIVE_CONTENT_URL_DB.clear() + _GENERATOR_DB.clear() + + +def prepare_site_db_and_overrides(): + '''Prepare overrides and create _SITE_DB + + _SITE_DB.keys() need to be ready for filter_translations + ''' + _SITE_DB.clear() + _SITE_DB[_MAIN_LANG] = _MAIN_SITEURL + # make sure it works for both root-relative and absolute + main_siteurl = '/' if _MAIN_SITEURL == '' else _MAIN_SITEURL + for lang, overrides in _SUBSITE_QUEUE.items(): + if 'SITEURL' not in overrides: + overrides['SITEURL'] = posixpath.join(main_siteurl, lang) + _SITE_DB[lang] = overrides['SITEURL'] + # default subsite hierarchy + if 'OUTPUT_PATH' not in overrides: + overrides['OUTPUT_PATH'] = os.path.join( + _MAIN_SETTINGS['OUTPUT_PATH'], lang) + if 'CACHE_PATH' not in overrides: + overrides['CACHE_PATH'] = os.path.join( + _MAIN_SETTINGS['CACHE_PATH'], lang) + if 'STATIC_PATHS' not in overrides: + overrides['STATIC_PATHS'] = [] + if ('THEME' not in overrides and 'THEME_STATIC_DIR' not in overrides and + 'THEME_STATIC_PATHS' not in overrides): + relpath = relpath_to_site(lang, _MAIN_LANG) + overrides['THEME_STATIC_DIR'] = posixpath.join( + relpath, _MAIN_SETTINGS['THEME_STATIC_DIR']) + overrides['THEME_STATIC_PATHS'] = [] + # to change what is perceived as translations + overrides['DEFAULT_LANG'] = lang + + +def subscribe_filter_to_signals(settings): + '''Subscribe content filter to requested signals''' + for sig in settings.get('I18N_FILTER_SIGNALS', []): + sig.connect(filter_contents_translations) + + +def initialize_plugin(pelican_obj): + '''Initialize plugin variables and Pelican settings''' + if _MAIN_SETTINGS is None: + initialize_dbs(pelican_obj.settings) + subscribe_filter_to_signals(pelican_obj.settings) + + +def get_site_path(url): + '''Get the path component of an url, excludes siteurl + + also normalizes '' to '/' for relpath to work, + otherwise it could be interpreted as a relative filesystem path + ''' + path = urlparse(url).path + if path == '': + path = '/' + return path + + +def relpath_to_site(lang, target_lang): + '''Get relative path from siteurl of lang to siteurl of base_lang + + the output is cached in _SITES_RELPATH_DB + ''' + path = _SITES_RELPATH_DB.get((lang, target_lang), None) + if path is None: + siteurl = _SITE_DB.get(lang, _MAIN_SITEURL) + target_siteurl = _SITE_DB.get(target_lang, _MAIN_SITEURL) + path = posixpath.relpath(get_site_path(target_siteurl), + get_site_path(siteurl)) + _SITES_RELPATH_DB[(lang, target_lang)] = path + return path + + +def save_generator(generator): + '''Save the generator for later use + + initialize the removed content list + ''' + _GENERATOR_DB[generator] = [] + + +def article2draft(article): + '''Transform an Article to Draft''' + draft = Draft(article._content, article.metadata, article.settings, + article.source_path, article._context) + draft.status = 'draft' + return draft + + +def page2hidden_page(page): + '''Transform a Page to a hidden Page''' + page.status = 'hidden' + return page + + +class GeneratorInspector(object): + '''Inspector of generator instances''' + + generators_info = { + ArticlesGenerator: { + 'translations_lists': ['translations', 'drafts_translations'], + 'contents_lists': [('articles', 'drafts')], + 'hiding_func': article2draft, + 'policy': 'I18N_UNTRANSLATED_ARTICLES', + }, + PagesGenerator: { + 'translations_lists': ['translations', 'hidden_translations'], + 'contents_lists': [('pages', 'hidden_pages')], + 'hiding_func': page2hidden_page, + 'policy': 'I18N_UNTRANSLATED_PAGES', + }, + } + + def __init__(self, generator): + '''Identify the best known class of the generator instance + + The class ''' + self.generator = generator + self.generators_info.update(generator.settings.get( + 'I18N_GENERATORS_INFO', {})) + for cls in generator.__class__.__mro__: + if cls in self.generators_info: + self.info = self.generators_info[cls] + break + else: + self.info = {} + + def translations_lists(self): + '''Iterator over lists of content translations''' + return (getattr(self.generator, name) for name in + self.info.get('translations_lists', [])) + + def contents_list_pairs(self): + '''Iterator over pairs of normal and hidden contents''' + return (tuple(getattr(self.generator, name) for name in names) + for names in self.info.get('contents_lists', [])) + + def hiding_function(self): + '''Function for transforming content to a hidden version''' + hiding_func = self.info.get('hiding_func', lambda x: x) + return hiding_func + + def untranslated_policy(self, default): + '''Get the policy for untranslated content''' + return self.generator.settings.get(self.info.get('policy', None), + default) + + def all_contents(self): + '''Iterator over all contents''' + translations_iterator = chain(*self.translations_lists()) + return chain(translations_iterator, + *(pair[i] for pair in self.contents_list_pairs() + for i in (0, 1))) + + +def filter_contents_translations(generator): + '''Filter the content and translations lists of a generator + + Filters out + 1) translations which will be generated in a different site + 2) content that is not in the language of the currently + generated site but in that of a different site, content in a + language which has no site is generated always. The filtering + method bay be modified by the respective untranslated policy + ''' + inspector = GeneratorInspector(generator) + current_lang = generator.settings['DEFAULT_LANG'] + langs_with_sites = _SITE_DB.keys() + removed_contents = _GENERATOR_DB[generator] + + for translations in inspector.translations_lists(): + for translation in translations[:]: # copy to be able to remove + if translation.lang in langs_with_sites: + translations.remove(translation) + removed_contents.append(translation) + + hiding_func = inspector.hiding_function() + untrans_policy = inspector.untranslated_policy(default='hide') + for (contents, other_contents) in inspector.contents_list_pairs(): + for content in other_contents: # save any hidden native content first + if content.lang == current_lang: # in native lang + # save the native URL attr formatted in the current locale + _NATIVE_CONTENT_URL_DB[content.source_path] = content.url + for content in contents[:]: # copy for removing in loop + if content.lang == current_lang: # in native lang + # save the native URL attr formatted in the current locale + _NATIVE_CONTENT_URL_DB[content.source_path] = content.url + elif content.lang in langs_with_sites and untrans_policy != 'keep': + contents.remove(content) + if untrans_policy == 'hide': + other_contents.append(hiding_func(content)) + elif untrans_policy == 'remove': + removed_contents.append(content) + + +def install_templates_translations(generator): + '''Install gettext translations in the jinja2.Environment + + Only if the 'jinja2.ext.i18n' jinja2 extension is enabled + the translations for the current DEFAULT_LANG are installed. + ''' + if 'JINJA_ENVIRONMENT' in generator.settings: # pelican 3.7+ + jinja_extensions = generator.settings['JINJA_ENVIRONMENT'].get( + 'extensions', []) + else: + jinja_extensions = generator.settings['JINJA_EXTENSIONS'] + + if 'jinja2.ext.i18n' in jinja_extensions: + domain = generator.settings.get('I18N_GETTEXT_DOMAIN', 'messages') + localedir = generator.settings.get('I18N_GETTEXT_LOCALEDIR') + if localedir is None: + localedir = os.path.join(generator.theme, 'translations') + current_lang = generator.settings['DEFAULT_LANG'] + if current_lang == generator.settings.get('I18N_TEMPLATES_LANG', + _MAIN_LANG): + translations = gettext.NullTranslations() + else: + langs = [current_lang] + try: + translations = gettext.translation(domain, localedir, langs) + except (IOError, OSError): + _LOGGER.error(( + "Cannot find translations for language '{}' in '{}' with " + "domain '{}'. Installing NullTranslations.").format( + langs[0], localedir, domain)) + translations = gettext.NullTranslations() + newstyle = generator.settings.get('I18N_GETTEXT_NEWSTYLE', True) + generator.env.install_gettext_translations(translations, newstyle) + + +def add_variables_to_context(generator): + '''Adds useful iterable variables to template context''' + context = generator.context # minimize attr lookup + context['relpath_to_site'] = relpath_to_site + context['main_siteurl'] = _MAIN_SITEURL + context['main_lang'] = _MAIN_LANG + context['lang_siteurls'] = _SITE_DB + current_lang = generator.settings['DEFAULT_LANG'] + extra_siteurls = _SITE_DB.copy() + extra_siteurls.pop(current_lang) + context['extra_siteurls'] = extra_siteurls + + +def interlink_translations(content): + '''Link content to translations in their main language + + so the URL (including localized month names) of the different subsites + will be honored + ''' + lang = content.lang + # sort translations by lang + content.translations.sort(key=attrgetter('lang')) + for translation in content.translations: + relpath = relpath_to_site(lang, translation.lang) + url = _NATIVE_CONTENT_URL_DB[translation.source_path] + translation.override_url = posixpath.join(relpath, url) + + +def interlink_translated_content(generator): + '''Make translations link to the native locations + + for generators that may contain translated content + ''' + inspector = GeneratorInspector(generator) + for content in inspector.all_contents(): + interlink_translations(content) + + +def interlink_removed_content(generator): + '''For all contents removed from generation queue update interlinks + + link to the native location + ''' + current_lang = generator.settings['DEFAULT_LANG'] + for content in _GENERATOR_DB[generator]: + url = _NATIVE_CONTENT_URL_DB[content.source_path] + relpath = relpath_to_site(current_lang, content.lang) + content.override_url = posixpath.join(relpath, url) + + +def interlink_static_files(generator): + '''Add links to static files in the main site if necessary''' + if generator.settings['STATIC_PATHS'] != []: + return # customized STATIC_PATHS + try: # minimize attr lookup + static_content = generator.context['static_content'] + except KeyError: + static_content = generator.context['filenames'] + relpath = relpath_to_site(generator.settings['DEFAULT_LANG'], _MAIN_LANG) + for staticfile in _MAIN_STATIC_FILES: + if staticfile.get_relative_source_path() not in static_content: + staticfile = copy(staticfile) # prevent override in main site + staticfile.override_url = posixpath.join(relpath, staticfile.url) + try: + generator.add_source_path(staticfile, static=True) + except TypeError: + generator.add_source_path(staticfile) + + +def save_main_static_files(static_generator): + '''Save the static files generated for the main site''' + global _MAIN_STATIC_FILES + # test just for current lang as settings change in autoreload mode + if static_generator.settings['DEFAULT_LANG'] == _MAIN_LANG: + _MAIN_STATIC_FILES = static_generator.staticfiles + + +def update_generators(): + '''Update the context of all generators + + Ads useful variables and translations into the template context + and interlink translations + ''' + for generator in _GENERATOR_DB.keys(): + install_templates_translations(generator) + add_variables_to_context(generator) + interlink_static_files(generator) + interlink_removed_content(generator) + interlink_translated_content(generator) + + +def get_pelican_cls(settings): + '''Get the Pelican class requested in settings''' + cls = settings['PELICAN_CLASS'] + if isinstance(cls, six.string_types): + module, cls_name = cls.rsplit('.', 1) + module = __import__(module) + cls = getattr(module, cls_name) + return cls + + +def create_next_subsite(pelican_obj): + '''Create the next subsite using the lang-specific config + + If there are no more subsites in the generation queue, update all + the generators (interlink translations and removed content, add + variables and translations to template context). Otherwise get the + language and overrides for next the subsite in the queue and apply + overrides. Then generate the subsite using a PELICAN_CLASS + instance and its run method. Finally, restore the previous locale. + ''' + global _MAIN_SETTINGS + if len(_SUBSITE_QUEUE) == 0: + _LOGGER.debug( + 'i18n: Updating cross-site links and context of all generators.') + update_generators() + _MAIN_SETTINGS = None # to initialize next time + else: + with temporary_locale(): + settings = _MAIN_SETTINGS.copy() + lang, overrides = _SUBSITE_QUEUE.popitem() + settings.update(overrides) + settings = configure_settings(settings) # to set LOCALE, etc. + cls = get_pelican_cls(settings) + + new_pelican_obj = cls(settings) + _LOGGER.debug(("Generating i18n subsite for language '{}' " + "using class {}").format(lang, cls)) + new_pelican_obj.run() + + +# map: signal name -> function name +_SIGNAL_HANDLERS_DB = { + 'get_generators': initialize_plugin, + 'article_generator_pretaxonomy': filter_contents_translations, + 'page_generator_finalized': filter_contents_translations, + 'get_writer': create_next_subsite, + 'static_generator_finalized': save_main_static_files, + 'generator_init': save_generator, +} + + +def register(): + '''Register the plugin only if required signals are available''' + for sig_name in _SIGNAL_HANDLERS_DB.keys(): + if not hasattr(signals, sig_name): + _LOGGER.error(( + 'The i18n_subsites plugin requires the {} ' + 'signal available for sure in Pelican 3.4.0 and later, ' + 'plugin will not be used.').format(sig_name)) + return + + for sig_name, handler in _SIGNAL_HANDLERS_DB.items(): + sig = getattr(signals, sig_name) + sig.connect(handler) diff --git a/plugins/i18n_subsites/implementing_language_buttons.rst b/plugins/i18n_subsites/implementing_language_buttons.rst new file mode 100644 index 0000000..55b7bf3 --- /dev/null +++ b/plugins/i18n_subsites/implementing_language_buttons.rst @@ -0,0 +1,128 @@ +----------------------------- +Implementing language buttons +----------------------------- + +Each article with translations has translations links, but that's the +only way to switch between language subsites. + +For this reason it is convenient to add language buttons to the top +menu bar to make it simple to switch between the language subsites on +all pages. + +Example designs +--------------- + +Language buttons showing other available languages +.................................................. + +The ``extra_siteurls`` dictionary is a mapping of all other (not the +``DEFAULT_LANG`` of the current (sub-)site) languages to the +``SITEURL`` of the respective (sub-)sites + +.. code-block:: jinja + + +