2499.patch

From 0e2560e045dad843950461affad51d69c4b9f9d2 Mon Sep 17 00:00:00 2001
From: Randy Barlow <[email protected]>
Date: Wed, 23 Mar 2016 20:58:52 +0000
Subject: [PATCH] Streamer download requests use permanently stored pem files.

This commit begins a shift towards storing repository PEM data (CA
certificates, client certificates, and client keys) on the
filesystem, and passing paths to those certs to Nectar rather than
allowing nectar to write them in temporary files.

This commit provides the infrastructure for this change, but only
converts the streamer to using it. All other downloads in Pulp
still take the old, now deprecated path.

https://pulp.plan.io/issues/1771

fixes #1771
---
 server/pulp/plugins/importer.py                    |  56 ++++-
 server/pulp/plugins/util/nectar_config.py          |  94 ++++++---
 server/pulp/server/controllers/repository.py       |  13 +-
 server/pulp/server/db/model/__init__.py            |  93 ++++++++-
 streamer/pulp/streamer/server.py                   |   7 +-
 10 files changed, 654 insertions(+), 57 deletions(-)

diff --git a/server/pulp/plugins/importer.py b/server/pulp/plugins/importer.py
index 25360b6..1d2cdaf 100644
--- a/plugins/importer.py
+++ b/plugins/importer.py
@@ -6,7 +6,8 @@
 from nectar.downloaders.local import LocalFileDownloader
 from nectar.downloaders.threaded import HTTPThreadedDownloader
 
-from pulp.plugins.util.nectar_config import importer_config_to_nectar_config
+from pulp.plugins.util.nectar_config import (importer_config_to_nectar_config,
+                                             importer_to_nectar_config)
 
 
 class Importer(object):
@@ -17,8 +18,32 @@ class Importer(object):
     """
 
     @staticmethod
+    def build_downloader(url, nectar_config):
+        """
+        Return a Nectar downloader for a URL with the given nectar config.
+
+        :param url:           The URL is used to determine the scheme so the correct type of
+                              downloader can be created.
+        :type  url:           basestring
+        :param nectar_config: The configuration that should be used with the downloader
+        :type  nectar_config: nectar.config.DownloaderConfig
+        :return:              A configured downloader.
+        :rtype:               nectar.downloaders.base.Downloader
+        :raise ValueError:    When the URL scheme is not supported.
+        """
+        url = urlparse(url)
+        scheme = url.scheme.lower()
+        if scheme == 'file':
+            return LocalFileDownloader(nectar_config)
+        if scheme in ('http', 'https'):
+            return HTTPThreadedDownloader(nectar_config)
+        raise ValueError(_('Scheme "{s}" not supported').format(s=url.scheme))
+
+    @staticmethod
     def get_downloader(config, url, working_dir=None, **options):
         """
+        DEPRECATED. Use get_downloader_for_db_importer instead.
+
         Get a configured downloader.
 
         :param config: A plugin configuration.
@@ -33,14 +58,29 @@ def get_downloader(config, url, working_dir=None, **options):
         :rtype: nectar.downloaders.base.Downloader
         :raise ValueError: when the URL scheme is not supported.
         """
-        url = urlparse(url)
         nectar_config = importer_config_to_nectar_config(config.flatten(), working_dir=working_dir)
-        scheme = url.scheme.lower()
-        if scheme == 'file':
-            return LocalFileDownloader(nectar_config)
-        if scheme in ('http', 'https'):
-            return HTTPThreadedDownloader(nectar_config)
-        raise ValueError(_('Scheme "{s}" not supported').format(s=url.scheme))
+        return Importer.build_downloader(url, nectar_config)
+
+    @staticmethod
+    def get_downloader_for_db_importer(importer, url, working_dir=None, **options):
+        """
+        Get a configured downloader for the given DB model of an Importer.
+
+        :param importer:    The database representation of an Importer for which we need a
+                            downloader.
+        :type  importer:    pulp.server.db.model.Importer
+        :param url:         A URL.
+        :type url:          str
+        :param working_dir: Allow the caller to override the working directory used.
+        :type  working_dir: str
+        :param options:     Extended configuration.
+        :type options:      dict
+        :return:            A configured downloader.
+        :rtype:             nectar.downloaders.base.Downloader
+        :raise ValueError:  When the URL scheme is not supported.
+        """
+        nectar_config = importer_to_nectar_config(importer, working_dir=working_dir)
+        return Importer.build_downloader(url, nectar_config)
 
     # -- plugin lifecycle -----------------------------------------------------
 
diff --git a/server/pulp/plugins/util/nectar_config.py b/server/pulp/plugins/util/nectar_config.py
index ac75a94..1eef4bf 100644
--- a/plugins/util/nectar_config.py
+++ b/plugins/util/nectar_config.py
@@ -1,7 +1,7 @@
 """
 Contains functions related to working with the Nectar downloading library.
 """
-
+import copy
 from functools import partial
 
 from nectar.config import DownloaderConfig
@@ -10,43 +10,89 @@
 from pulp.server.managers.repo import _common as common_utils
 
 
-def importer_config_to_nectar_config(importer_config, working_dir=None):
+# Mapping of importer config key to downloader config key
+IMPORTER_DOWNLADER_CONFIG_MAP = (
+    (constants.KEY_SSL_CA_CERT, 'ssl_ca_cert'),
+    (constants.KEY_SSL_VALIDATION, 'ssl_validation'),
+    (constants.KEY_SSL_CLIENT_CERT, 'ssl_client_cert'),
+    (constants.KEY_SSL_CLIENT_KEY, 'ssl_client_key'),
+
+    (constants.KEY_PROXY_HOST, 'proxy_url'),
+    (constants.KEY_PROXY_PORT, 'proxy_port'),
+    (constants.KEY_PROXY_USER, 'proxy_username'),
+    (constants.KEY_PROXY_PASS, 'proxy_password'),
+
+    (constants.KEY_BASIC_AUTH_USER, 'basic_auth_username'),
+    (constants.KEY_BASIC_AUTH_PASS, 'basic_auth_password'),
+
+    (constants.KEY_MAX_DOWNLOADS, 'max_concurrent'),
+    (constants.KEY_MAX_SPEED, 'max_speed'),
+)
+
+
+def importer_to_nectar_config(importer, working_dir=None):
     """
-    Translates the Pulp standard importer configuration into a DownloaderConfig instance.
+    Translates a Pulp Importer into a DownloaderConfig instance.
+
+    This function replaces importer_config_to_nectar_config. The primary difference is that it
+    requires the database Importer model and is able to use that to configure Nectar to use the
+    permanently stored TLS certificates and key rather than having Nectar write them out as
+    temporary files. For now, this function uses the deprected function to avoid duplicating code.
+    Once we have confirmed that the other function is no longer used by anything outside of this
+    module it can be removed and its functionality can be moved here.
 
-    :param importer_config: use the PluginCallConfiguration.flatten method to retrieve a
-           single dict view on the configuration
-    :type  importer_config: dict
+    :param importer:    The Importer that has a config to be translated to a Nectar config.
+    :type  importer:    pulp.server.db.model.Importer
     :param working_dir: Allow the caller to override the working directory used
-    :type working_dir: str
+    :type  working_dir: str
 
     :rtype: nectar.config.DownloaderConfig
     """
+    download_config_kwargs = {}
+
+    # If the Importer config has TLS certificates and keys, we need to remove them and configure
+    # nectar to use the permanently stored paths on the filesystem.
+    config = copy.copy(importer.config)
+    if constants.KEY_SSL_CA_CERT in config:
+        del config[constants.KEY_SSL_CA_CERT]
+        download_config_kwargs['ssl_ca_cert_path'] = importer.tls_ca_cert_path
+    if constants.KEY_SSL_CLIENT_CERT in config:
+        del config[constants.KEY_SSL_CLIENT_CERT]
+        download_config_kwargs['ssl_client_cert_path'] = importer.tls_client_cert_path
+    if constants.KEY_SSL_CLIENT_KEY in config:
+        del config[constants.KEY_SSL_CLIENT_KEY]
+        download_config_kwargs['ssl_client_key_path'] = importer.tls_client_key_path
+
+    return importer_config_to_nectar_config(config, working_dir, download_config_kwargs)
+
+
+def importer_config_to_nectar_config(importer_config, working_dir=None,
+                                     download_config_kwargs=None):
+    """
+    DEPRECATED. Use importer_to_nectar_config instead.
 
-    # Mapping of importer config key to downloader config key
-    translations = (
-        (constants.KEY_SSL_CA_CERT, 'ssl_ca_cert'),
-        (constants.KEY_SSL_VALIDATION, 'ssl_validation'),
-        (constants.KEY_SSL_CLIENT_CERT, 'ssl_client_cert'),
-        (constants.KEY_SSL_CLIENT_KEY, 'ssl_client_key'),
+    Translates the Pulp standard importer configuration into a DownloaderConfig instance.
 
-        (constants.KEY_PROXY_HOST, 'proxy_url'),
-        (constants.KEY_PROXY_PORT, 'proxy_port'),
-        (constants.KEY_PROXY_USER, 'proxy_username'),
-        (constants.KEY_PROXY_PASS, 'proxy_password'),
+    :param importer_config:        use the PluginCallConfiguration.flatten method to retrieve a
+                                   single dict view on the configuration
+    :type  importer_config:        dict
+    :param working_dir:            Allow the caller to override the working directory used
+    :type  working_dir:            str
+    :param download_config_kwargs: Any additional keyword arguments you would like to include in the
+                                   download config.
+    :type  download_config_kwargs: dict
 
-        (constants.KEY_BASIC_AUTH_USER, 'basic_auth_username'),
-        (constants.KEY_BASIC_AUTH_PASS, 'basic_auth_password'),
+    :rtype: nectar.config.DownloaderConfig
+    """
+    if download_config_kwargs is None:
+        download_config_kwargs = {}
 
-        (constants.KEY_MAX_DOWNLOADS, 'max_concurrent'),
-        (constants.KEY_MAX_SPEED, 'max_speed'),
-    )
     if working_dir is None:
         working_dir = common_utils.get_working_directory()
 
-    download_config_kwargs = {'working_dir': working_dir}
+    download_config_kwargs['working_dir'] = working_dir
     adder = partial(_safe_add_arg, importer_config, download_config_kwargs)
-    map(adder, translations)
+    map(adder, IMPORTER_DOWNLADER_CONFIG_MAP)
 
     download_config = DownloaderConfig(**download_config_kwargs)
     return download_config
diff --git a/server/pulp/server/controllers/repository.py b/server/pulp/server/controllers/repository.py
index 698bd14..1894af9 100644
--- a/server/controllers/repository.py
+++ b/server/controllers/repository.py
@@ -454,14 +454,17 @@ def queue_delete(repo_id):
 
 def get_importer_by_id(object_id):
     """
-    Get a plugin and call configuration using the document ID
+    Get a plugin, call configuration, and Importer document object using the document ID
     of the repository-importer association document.
 
     :param object_id: The document ID.
-    :type object_id: str
+    :type  object_id: str
+
     :return: A tuple of:
-        (pulp.plugins.importer.Importer, pulp.plugins.config.PluginCallConfiguration)
-    :rtype: tuple
+        (pulp.plugins.importer.Importer, pulp.plugins.config.PluginCallConfiguration,
+         pulp.server.db.model.Importer)
+    :rtype:  tuple
+
     :raise pulp.plugins.loader.exceptions.PluginNotFound: not found.
     """
     try:
@@ -474,7 +477,7 @@ def get_importer_by_id(object_id):
         raise plugin_exceptions.PluginNotFound()
     plugin, cfg = plugin_api.get_importer_by_id(document.importer_type_id)
     call_conf = PluginCallConfiguration(cfg, document.config)
-    return plugin, call_conf
+    return plugin, call_conf, document
 
 
 @celery.task(base=Task, name='pulp.server.tasks.repository.delete')
diff --git a/server/pulp/server/db/model/__init__.py b/server/pulp/server/db/model/__init__.py
index 4b1b29d..4e2fff8 100644
--- a/server/db/model/__init__.py
+++ b/server/db/model/__init__.py
@@ -1,10 +1,11 @@
 import copy
-from gettext import gettext as _
 import logging
 import os
 import random
+import shutil
 import uuid
 from collections import namedtuple
+from gettext import gettext as _
 from hashlib import sha256
 from hmac import HMAC
 
@@ -13,9 +14,11 @@
 from mongoengine import signals
 
 from pulp.common import constants, dateutils, error_codes
+from pulp.common.plugins import importer_constants
 from pulp.plugins.model import Repository as plugin_repo
+from pulp.plugins.util import misc
 from pulp.server import exceptions
-from pulp.server.constants import SUPER_USER_ROLE
+from pulp.server.constants import LOCAL_STORAGE, SUPER_USER_ROLE
 from pulp.server.content.storage import FileStorage, SharedStorage
 from pulp.server.async.emit import send as send_taskstatus_message
 from pulp.server.db.connection import UnsafeRetry
@@ -257,6 +260,92 @@ def pre_delete(cls, sender, document, **kwargs):
             repo=document.repo_id))
         query_set.delete()
 
+    def delete(self):
+        """
+        Delete the Importer. Remove any documents it has stored.
+        """
+        if os.path.exists(self._local_storage_path):
+            shutil.rmtree(self._local_storage_path)
+        super(Importer, self).delete()
+
+    def save(self):
+        """
+        Save the Importer. Additionally, write any pki documents from its config into disk storage
+        for use by requests.
+        """
+        super(Importer, self).save()
+        # A map of Importer config key names to file paths for the TLS PEM settings.
+        pem_keys_paths = (
+            (importer_constants.KEY_SSL_CA_CERT, self.tls_ca_cert_path),
+            (importer_constants.KEY_SSL_CLIENT_CERT, self.tls_client_cert_path),
+            (importer_constants.KEY_SSL_CLIENT_KEY, self.tls_client_key_path))
+        for key, path in pem_keys_paths:
+            self._write_pem_file(key, path)
+
+    @property
+    def tls_ca_cert_path(self):
+        """
+        Return the path where the TLS CA certificate should be stored for this Importer.
+
+        :rtype: basestring
+        """
+        return os.path.join(self._pki_path, 'ca.crt')
+
+    @property
+    def tls_client_cert_path(self):
+        """
+        Return the path where the TLS client certificate should be stored for this Importer.
+
+        :rtype: basestring
+        """
+        return os.path.join(self._pki_path, 'client.crt')
+
+    @property
+    def tls_client_key_path(self):
+        """
+        Return the path where the TLS client key should be stored for this Importer.
+
+        :rtype: basestring
+        """
+        return os.path.join(self._pki_path, 'client.key')
+
+    @property
+    def _local_storage_path(self):
+        """
+        Return the path that the Importer should use for local storage.
+
+        :rtype: basestring
+        """
+        return os.path.join(
+            LOCAL_STORAGE, 'importers',
+            '{}-{}'.format(self.repo_id, self.importer_type_id))
+
+    @property
+    def _pki_path(self):
+        """
+        Return the path that all pki files should be stored within for this Importer.
+
+        :rtype: basestring
+        """
+        return os.path.join(self._local_storage_path, 'pki')
+
+    def _write_pem_file(self, config_key, path):
+        """
+        Write the PEM data from self.config[config_key] to the given path, if the key is defined and
+        is "truthy".
+
+        :param config_key: The key corresponding to a value in self.config to write to path.
+        :type  config_key: basestring
+        :param path:       The path to write the PEM data to.
+        :type  path:       basestring
+        """
+        if config_key in self.config and self.config[config_key]:
+            if not os.path.exists(self._pki_path):
+                misc.mkdir(os.path.dirname(self._pki_path))
+                os.mkdir(self._pki_path, 0700)
+            with os.fdopen(os.open(path, os.O_WRONLY | os.O_CREAT, 0600), 'w') as pem_file:
+                pem_file.write(self.config[config_key])
+
 
 signals.pre_delete.connect(Importer.pre_delete, sender=Importer)
 
diff --git a/streamer/pulp/streamer/server.py b/streamer/pulp/streamer/server.py
index 9aaafcd..902ae58 100644
--- a/streamer/server.py
+++ b/streamer/server.py
@@ -208,9 +208,10 @@ def _download(self, catalog_entry, request, responder):
         :type  responder:       Responder
         """
         # Configure the primary downloader for alternate content sources
-        importer, config = repo_controller.get_importer_by_id(catalog_entry.importer_id)
-        primary_downloader = importer.get_downloader(config, catalog_entry.url, working_dir='/tmp',
-                                                     **catalog_entry.data)
+        plugin_importer, config, db_importer = repo_controller.get_importer_by_id(
+            catalog_entry.importer_id)
+        primary_downloader = plugin_importer.get_downloader_for_db_importer(
+            db_importer, catalog_entry.url, working_dir='/tmp')
         pulp_request = request.getHeader(PULP_STREAM_REQUEST_HEADER)
         listener = StreamerListener(request, self.config, catalog_entry, pulp_request)
         primary_downloader.session = self.session