My fork for pelican i18n sub-sites

i18n_subsites.py

"""i18n_subsites plugin creates i18n-ized subsites of the default site

This plugin is designed for Pelican 3.4 and later
"""


import os
import six
import logging
import posixpath

from copy import copy
from itertools import chain
from operator import attrgetter
try:
    from collections.abc import OrderedDict
except ImportError:
    from collections import OrderedDict
from contextlib import contextmanager
from six.moves.urllib.parse import urlparse

import gettext
import locale

from pelican import signals
from pelican.generators import ArticlesGenerator, PagesGenerator
from pelican.settings import configure_settings
try:
    from pelican.contents import Draft
except ImportError:
    from pelican.contents import Article as Draft


# Global vars
_MAIN_SETTINGS = None     # settings dict of the main Pelican instance
_MAIN_LANG = None         # lang of the main Pelican instance
_MAIN_SITEURL = None      # siteurl of the main Pelican instance
_MAIN_STATIC_FILES = None # list of Static instances the main Pelican instance
_SUBSITE_QUEUE = {}   # map: lang -> settings overrides
_SITE_DB = OrderedDict()           # OrderedDict: lang -> siteurl
_SITES_RELPATH_DB = {}       # map: (lang, base_lang) -> relpath
# map: generator -> list of removed contents that need interlinking
_GENERATOR_DB = {}
_NATIVE_CONTENT_URL_DB = {} # map: source_path -> content in its native lang
_LOGGER = logging.getLogger(__name__)

_L10N_KEY = '_i18n_subsites_translations'
_NULL_TRANSLATIONS = gettext.NullTranslations()

@contextmanager
def temporary_locale(temp_locale=None):
    '''Enable code to run in a context with a temporary locale

    Resets the locale back when exiting context.
    Can set a temporary locale if provided
    '''
    orig_locale = locale.setlocale(locale.LC_ALL)
    if temp_locale is not None:
        locale.setlocale(locale.LC_ALL, temp_locale)
    yield
    locale.setlocale(locale.LC_ALL, orig_locale)


def initialize_dbs(settings):
    '''Initialize internal DBs using the Pelican settings dict

    This clears the DBs for e.g. autoreload mode to work
    '''
    global _MAIN_SETTINGS, _MAIN_SITEURL, _MAIN_LANG, _SUBSITE_QUEUE
    _MAIN_SETTINGS = settings
    _MAIN_LANG = settings['DEFAULT_LANG']
    _MAIN_SITEURL = settings['SITEURL']
    _SUBSITE_QUEUE = settings.get('I18N_SUBSITES', {}).copy()
    prepare_site_db_and_overrides()
    # clear databases in case of autoreload mode
    _SITES_RELPATH_DB.clear()
    _NATIVE_CONTENT_URL_DB.clear()
    _GENERATOR_DB.clear()


def prepare_site_db_and_overrides():
    '''Prepare overrides and create _SITE_DB

    _SITE_DB.keys() need to be ready for filter_translations
    '''
    _SITE_DB.clear()
    _SITE_DB[_MAIN_LANG] = _MAIN_SITEURL
    # make sure it works for both root-relative and absolute
    main_siteurl = '/' if _MAIN_SITEURL == '' else _MAIN_SITEURL
    for lang, overrides in _SUBSITE_QUEUE.items():
        if 'SITEURL' not in overrides:
            overrides['SITEURL'] = posixpath.join(main_siteurl, lang)
        _SITE_DB[lang] = overrides['SITEURL']
        # default subsite hierarchy
        if 'OUTPUT_PATH' not in overrides:
            overrides['OUTPUT_PATH'] = os.path.join(
                _MAIN_SETTINGS['OUTPUT_PATH'], lang)
        if 'CACHE_PATH' not in overrides:
            overrides['CACHE_PATH'] = os.path.join(
                _MAIN_SETTINGS['CACHE_PATH'], lang)
        if 'STATIC_PATHS' not in overrides:
            overrides['STATIC_PATHS'] = []
        if ('THEME' not in overrides and 'THEME_STATIC_DIR' not in overrides and
                'THEME_STATIC_PATHS' not in overrides):
            relpath = relpath_to_site(lang, _MAIN_LANG)
            overrides['THEME_STATIC_DIR'] = posixpath.join(
                relpath, _MAIN_SETTINGS['THEME_STATIC_DIR'])
            overrides['THEME_STATIC_PATHS'] = []
        # to change what is perceived as translations
        overrides['DEFAULT_LANG'] = lang


def subscribe_filter_to_signals(settings):
    '''Subscribe content filter to requested signals'''
    for sig in settings.get('I18N_FILTER_SIGNALS', []):
        sig.connect(filter_contents_translations)


def initialize_plugin(pelican_obj):
    '''Initialize plugin variables and Pelican settings'''
    if _MAIN_SETTINGS is None:
        initialize_dbs(pelican_obj.settings)
        subscribe_filter_to_signals(pelican_obj.settings)


def get_site_path(url):
    '''Get the path component of an url, excludes siteurl

    also normalizes '' to '/' for relpath to work,
    otherwise it could be interpreted as a relative filesystem path
    '''
    path = urlparse(url).path
    if path == '':
        path = '/'
    return path


def relpath_to_site(lang, target_lang):
    '''Get relative path from siteurl of lang to siteurl of base_lang

    the output is cached in _SITES_RELPATH_DB
    '''
    path = _SITES_RELPATH_DB.get((lang, target_lang), None)
    if path is None:
        siteurl = _SITE_DB.get(lang, _MAIN_SITEURL)
        target_siteurl = _SITE_DB.get(target_lang, _MAIN_SITEURL)
        path = posixpath.relpath(get_site_path(target_siteurl),
                                 get_site_path(siteurl))
        _SITES_RELPATH_DB[(lang, target_lang)] = path
    return path


def save_generator(generator):
    '''Save the generator for later use

    initialize the removed content list
    '''
    _GENERATOR_DB[generator] = []


def article2draft(article):
    '''Transform an Article to Draft'''
    draft = Draft(article._content, article.metadata, article.settings,
                  article.source_path, article._context)
    draft.status = 'draft'
    return draft


def page2hidden_page(page):
    '''Transform a Page to a hidden Page'''
    page.status = 'hidden'
    return page


class GeneratorInspector(object):
    '''Inspector of generator instances'''

    generators_info = {
        ArticlesGenerator: {
            'translations_lists': ['translations', 'drafts_translations'],
            'contents_lists': [('articles', 'drafts')],
            'hiding_func': article2draft,
            'policy': 'I18N_UNTRANSLATED_ARTICLES',
        },
        PagesGenerator: {
            'translations_lists': ['translations', 'hidden_translations'],
            'contents_lists': [('pages', 'hidden_pages')],
            'hiding_func': page2hidden_page,
            'policy': 'I18N_UNTRANSLATED_PAGES',
        },
    }

    def __init__(self, generator):
        '''Identify the best known class of the generator instance

        The class '''
        self.generator = generator
        self.generators_info.update(generator.settings.get(
            'I18N_GENERATORS_INFO', {}))
        for cls in generator.__class__.__mro__:
            if cls in self.generators_info:
                self.info = self.generators_info[cls]
                break
        else:
            self.info = {}

    def translations_lists(self):
        '''Iterator over lists of content translations'''
        return (getattr(self.generator, name) for name in
                self.info.get('translations_lists', []))

    def contents_list_pairs(self):
        '''Iterator over pairs of normal and hidden contents'''
        return (tuple(getattr(self.generator, name) for name in names)
                for names in self.info.get('contents_lists', []))

    def hiding_function(self):
        '''Function for transforming content to a hidden version'''
        hiding_func = self.info.get('hiding_func', lambda x: x)
        return hiding_func

    def untranslated_policy(self, default):
        '''Get the policy for untranslated content'''
        return self.generator.settings.get(self.info.get('policy', None),
                                           default)

    def all_contents(self):
        '''Iterator over all contents'''
        translations_iterator = chain(*self.translations_lists())
        return chain(translations_iterator,
                     *(pair[i] for pair in self.contents_list_pairs()
                       for i in (0, 1)))


def filter_contents_translations(generator):
    '''Filter the content and translations lists of a generator

    Filters out
        1) translations which will be generated in a different site
        2) content that is not in the language of the currently
        generated site but in that of a different site, content in a
        language which has no site is generated always. The filtering
        method bay be modified by the respective untranslated policy
    '''
    inspector = GeneratorInspector(generator)
    current_lang = generator.settings['DEFAULT_LANG']
    langs_with_sites = _SITE_DB.keys()
    removed_contents = _GENERATOR_DB[generator]

    for translations in inspector.translations_lists():
        for translation in translations[:]:    # copy to be able to remove
            if translation.lang in langs_with_sites:
                translations.remove(translation)
                removed_contents.append(translation)

    hiding_func = inspector.hiding_function()
    untrans_policy = inspector.untranslated_policy(default='hide')
    for (contents, other_contents) in inspector.contents_list_pairs():
        for content in other_contents: # save any hidden native content first
            if content.lang == current_lang: # in native lang
                # save the native URL attr formatted in the current locale
                _NATIVE_CONTENT_URL_DB[content.source_path] = content.url
        for content in contents[:]:        # copy for removing in loop
            if content.lang == current_lang: # in native lang
                # save the native URL attr formatted in the current locale
                _NATIVE_CONTENT_URL_DB[content.source_path] = content.url
            elif content.lang in langs_with_sites and untrans_policy != 'keep':
                contents.remove(content)
                if untrans_policy == 'hide':
                    other_contents.append(hiding_func(content))
                elif untrans_policy == 'remove':
                    removed_contents.append(content)


def install_translations(generator):
    """
    Install gettext translations in the jinja2.Environment

    Only if the 'jinja2.ext.i18n' jinja2 extension is enabled
    the translations for the current DEFAULT_LANG are installed.
    """
    if 'JINJA_ENVIRONMENT' in generator.settings: # pelican 3.7+
        jinja_extensions = generator.settings['JINJA_ENVIRONMENT'].get(
            'extensions', [])
    else:
        jinja_extensions = generator.settings['JINJA_EXTENSIONS']

    if 'jinja2.ext.i18n' in jinja_extensions:
        current_lang = generator.settings['DEFAULT_LANG']

        if _L10N_KEY in _MAIN_SETTINGS:
            translations = _MAIN_SETTINGS[_L10N_KEY].get(
                current_lang, _NULL_TRANSLATIONS)
        else:
            translations = _NULL_TRANSLATIONS

        newstyle = generator.settings.get('I18N_GETTEXT_NEWSTYLE', True)
        generator.env.install_gettext_translations(translations, newstyle)


def add_variables_to_context(generator):
    '''Adds useful iterable variables to template context'''
    context = generator.context             # minimize attr lookup
    context['relpath_to_site'] = relpath_to_site
    context['main_siteurl'] = _MAIN_SITEURL
    context['main_lang'] = _MAIN_LANG
    context['lang_siteurls'] = _SITE_DB
    current_lang = generator.settings['DEFAULT_LANG']
    extra_siteurls = _SITE_DB.copy()
    extra_siteurls.pop(current_lang)
    context['extra_siteurls'] = extra_siteurls


def interlink_translations(content):
    '''Link content to translations in their main language

    so the URL (including localized month names) of the different subsites
    will be honored
    '''
    lang = content.lang
    # sort translations by lang
    content.translations.sort(key=attrgetter('lang'))
    for translation in content.translations:
        relpath = relpath_to_site(lang, translation.lang)
        url = _NATIVE_CONTENT_URL_DB[translation.source_path]
        translation.override_url = posixpath.join(relpath, url)


def interlink_translated_content(generator):
    '''Make translations link to the native locations

    for generators that may contain translated content
    '''
    inspector = GeneratorInspector(generator)
    for content in inspector.all_contents():
        interlink_translations(content)


def interlink_removed_content(generator):
    '''For all contents removed from generation queue update interlinks

    link to the native location
    '''
    current_lang = generator.settings['DEFAULT_LANG']
    for content in _GENERATOR_DB[generator]:
        url = _NATIVE_CONTENT_URL_DB[content.source_path]
        relpath = relpath_to_site(current_lang, content.lang)
        content.override_url = posixpath.join(relpath, url)


def interlink_static_files(generator):
    '''Add links to static files in the main site if necessary'''
    if generator.settings['STATIC_PATHS'] != []:
        return                               # customized STATIC_PATHS
    try: # minimize attr lookup
        static_content = generator.context['static_content']
    except KeyError:
        static_content = generator.context['filenames']
    relpath = relpath_to_site(generator.settings['DEFAULT_LANG'], _MAIN_LANG)
    for staticfile in _MAIN_STATIC_FILES:
        if staticfile.get_relative_source_path() not in static_content:
            staticfile = copy(staticfile) # prevent override in main site
            staticfile.override_url = posixpath.join(relpath, staticfile.url)
            try:
                generator.add_source_path(staticfile, static=True)
            except TypeError:
                generator.add_source_path(staticfile)


def save_main_static_files(static_generator):
    '''Save the static files generated for the main site'''
    global _MAIN_STATIC_FILES
    # test just for current lang as settings change in autoreload mode
    if static_generator.settings['DEFAULT_LANG'] == _MAIN_LANG:
        _MAIN_STATIC_FILES = static_generator.staticfiles


def update_generators():
    '''Update the context of all generators

    Ads useful variables and translations into the template context
    and interlink translations
    '''
    for generator in _GENERATOR_DB.keys():
        install_translations(generator)
        add_variables_to_context(generator)
        interlink_static_files(generator)
        interlink_removed_content(generator)
        interlink_translated_content(generator)


def get_pelican_cls(settings):
    '''Get the Pelican class requested in settings'''
    cls = settings['PELICAN_CLASS']
    if isinstance(cls, six.string_types):
        module, cls_name = cls.rsplit('.', 1)
        module = __import__(module)
        cls = getattr(module, cls_name)
    return cls


def create_next_subsite(pelican_obj):
    '''Create the next subsite using the lang-specific config

    If there are no more subsites in the generation queue, update all
    the generators (interlink translations and removed content, add
    variables and translations to template context). Otherwise get the
    language and overrides for next the subsite in the queue and apply
    overrides.  Then generate the subsite using a PELICAN_CLASS
    instance and its run method. Finally, restore the previous locale.
    '''
    global _MAIN_SETTINGS
    if len(_SUBSITE_QUEUE) == 0:
        _LOGGER.debug(
            'i18n: Updating cross-site links and context of all generators.')
        update_generators()
        _MAIN_SETTINGS = None             # to initialize next time
    else:
        with temporary_locale():
            lang, overrides = _SUBSITE_QUEUE.popitem()
            translations = _load_translations(_MAIN_SETTINGS, lang)
            settings = _merge_dict(_MAIN_SETTINGS, overrides, translations)
            settings = configure_settings(settings)      # to set LOCALE, etc.
            cls = get_pelican_cls(settings)

            new_pelican_obj = cls(settings)
            _LOGGER.debug(("Generating i18n subsite for language '{}' "
                           "using class {}").format(lang, cls))
            new_pelican_obj.run()


def _load_translations(settings, current_lang):
    """
    Load the translations and cache them on the main settings object.
    """
    domain = settings.get('I18N_GETTEXT_DOMAIN', 'messages')
    localedir = settings.get('I18N_GETTEXT_LOCALEDIR')
    if localedir is None:
        localedir = os.path.join(pelican_obj.theme, 'translations')

    if current_lang == settings.get('I18N_TEMPLATES_LANG', _MAIN_LANG):
        translations = _NULL_TRANSLATIONS
    else:
        langs = [current_lang]
        try:
            translations = gettext.translation(domain, localedir, langs)
        except (IOError, OSError):
            _LOGGER.error((
                "Cannot find translations for language '{}' in '{}' with "
                "domain '{}'. Installing NullTranslations.").format(
                    langs[0], localedir, domain))
            translations = _NULL_TRANSLATIONS

    if _L10N_KEY not in settings:
        settings[_L10N_KEY] = {}

    settings[_L10N_KEY][current_lang] = translations
    return translations


def _merge_dict(target, source, translations):
    """
    Update the values in `target` based on the values from source.
    """
    result = {}
    for key, value in target.items():
        if key not in source:
            # Keep the original value as it's not found.
            if value and isinstance(value, str):
                update = translations.gettext(value)
            else:
                update = copy(value)

            result[key] = update
            continue

        update = source[key]

        if isinstance(value, dict):
            update = _merge_dict(value, update, translations)

        result[key] = update

    return result


# map: signal name -> function name
_SIGNAL_HANDLERS_DB = {
    'get_generators': initialize_plugin,
    'article_generator_pretaxonomy': filter_contents_translations,
    'page_generator_finalized': filter_contents_translations,
    'get_writer': create_next_subsite,
    'static_generator_finalized': save_main_static_files,
    'generator_init': save_generator,
}


def register():
    '''Register the plugin only if required signals are available'''
    for sig_name in _SIGNAL_HANDLERS_DB.keys():
        if not hasattr(signals, sig_name):
            _LOGGER.error((
                'The i18n_subsites plugin requires the {} '
                'signal available for sure in Pelican 3.4.0 and later, '
                'plugin will not be used.').format(sig_name))
            return

    for sig_name, handler in _SIGNAL_HANDLERS_DB.items():
        sig = getattr(signals, sig_name)
        sig.connect(handler)

pelican.conf

# This is needed to enable the translation subsites.
# Most values should be translates using gettext inside SITE.
# But only text type is supported for translation.
# So if we have a tupple or list we add it here.
L10N_LANGS = {
    'en': {'name': 'English', 'base_url': ''},
    'de': {'name': 'Deutsch', 'base_url': '/de'},
    'es': {'name': 'Español', 'base_url': '/es'},
}

I18N_UNTRANSLATED_ARTICLES = 'remove'

I18N_SUBSITES = {
'de': { 'SITE': {
    'menu': [
        # Text, URL, category.
        ('Produkt', '/de/product/', 'product'),
        ('Preisgestaltung', '/de/pricing/', 'pricing'),
        ('Support', '/de/support/', 'support'),
        ('Neuigkeiten', '/de/articles/category/news/', 'news'),
        ('Kontakt', '/de/about/contact.html', 'contact'),
        ],
    }},

'es': { 'SITE': {
    'menu': [
        # Text, URL, category.
        ('Producto', '/es/product/', 'product'),
        ('Precios', '/es/pricing/', 'pricing'),
        ('Soporte', '/es/support/', 'support'),
        ('Noticias', '/es/articles/category/news/', 'news'),
        ('Contacto', '/es/about/contact.html', 'contact'),
        ],
    }},
}

# Only enable the translations on request via env variable.
# This is done as building the translation takes extra time.
if os.environ.get('PELICAN_I18N', 'yes') == 'no':
    I18N_SUBSITES = {}
    L10N_LANGS = {'en': L10N_LANGS['en']}


DEFAULT_LANG = 'en'
ARTICLE_TRANSLATION_ID = 'slug'
PAGE_TRANSLATION_ID = 'slug'
LOCALE = "C"
I18N_GETTEXT_L18NDIR = 'translations/'
I18N_GETTEXT_LOCALEDIR = 'locale/'
I18N_GETTEXT_DOMAIN = 'sftpplus_com'