Create offline package install bundles for Continuum Analytics Anaconda

anaconda_offline_package_bundler.py

DESCRIPTION = \
"This script builds a tar'd bundle of Anaconda packages and their dependencies suitable for installing on an offline server."

EPILOG = \
'''
The basic work flow is as follows,
 - The user manually runs a conda 'dry-run' install command on an online server to generate a json file containing the required packages and their dependency packages.
 - The script loads the 'dry-run' json file(s) and performs the following actions,
     - Create a 'channel' directory per online channel, as discovered from the json file(s).
     - Download the packages and their dependency packages into the channel directory.
     - 'conda index' the contents of each channel directory.
     - Generate a README file with basic install instructions to be executed on the offline server.
     - Bundles the channel directories into a single TAR file suitable for transfer to the offline server.

The script logic is built around the following assumptions,
 - This script is run using the Anaconda installed python executable.
 - The Anaconda version and OS type are the same on the online and offline servers.
 - Anaconda is installed in the same path location on both the online and offline servers. (Only impacts README instructions)
 - The working directory is the same on both the online and offline servers. (Only impacts README instructions)
 - The 'dry-run' json file names comply to glob pattern '*_channel_pkgs.json'
 - The 'dry-run' json file(s) are found in the working directory.

So what you need to do is,
  - Generate the 'dry-run' install json file(s) as per the examples below, on the online server.
  - Run this script on the online server.
  - Copy the resulting 'anaconda_offline_channels.tar' file to the offline server.
  - Follow the instructions in the 'anaconda_offline_channels.README' file.


'dry-run' examples
------------------

${HOME}/anaconda3/bin/conda install --dry-run \\
cx_oracle \\
psycopg2 \\
ujson \\
--json > /tmp/anaconda_channel_pkgs.json

${HOME}/anaconda3/bin/conda install --dry-run --channel conda-forge \\
altair \\
python-xxhash \\
--json > /tmp/condaforge_channel_pkgs.json

%HOMEPATH%\\AppData\\Local\\Continuum\\anaconda3\\Scripts\\conda.exe install --dry-run ^
cx_oracle ^
psycopg2 ^
ujson ^
--json > %TEMP%\\anaconda_channel_pkgs.json

Tested on
---------
- Anaconda2-5.1.0-Linux-x86_64 on Ubuntu 18.04
- Anaconda3-5.1.0-Linux-x86_64 on Ubuntu 18.04
- Anaconda3-5.1.0-Windows-x86_64 on Windows 10
'''
#------------------------------------------------------------------------------

__author__ = 'Dave Coutts'
__license__ = 'Apache'
__version__ = '1.1.0'
__maintainer__ = 'https://github.com/davecoutts'
__status__ = 'Production'

#------------------------------------------------------------------------------

import os
import sys
import glob
import json
import os.path
import tarfile
import argparse
import platform
import requests
import tempfile
from jinja2 import Template
from conda_build.index import update_index

#------------------------------------------------------------------------------

parser = argparse.ArgumentParser(
    epilog=EPILOG, 
    description=DESCRIPTION, 
    formatter_class=argparse.RawTextHelpFormatter
)

parser.add_argument(
    '-w', 
    '--working-directory',
    action='store',
    dest='working_directory',
    default=tempfile.gettempdir(),
    help='Specify the working directory. Default is the system temporary directory.'
)

args = parser.parse_args()

#------------------------------------------------------------------------------

WORKING_DIR = args.working_directory

BASE_NAME = 'anaconda_offline_channels'

CHANNELS_BASE_DIR = os.path.join(WORKING_DIR, BASE_NAME)
CHANNELS_TAR_FILE = '{}.tar'.format(CHANNELS_BASE_DIR)

README_FILE = '{}.README'.format(BASE_NAME)
README_FILE_PATH = os.path.join(WORKING_DIR, README_FILE)

PYTHON_EXEC_PATH = sys.executable

if platform.system() == 'Windows':

    CONDA_PATH = os.path.join(os.path.dirname(PYTHON_EXEC_PATH), os.path.join('Scripts', 'conda.exe'))

else:

    CONDA_PATH = os.path.join(os.path.dirname(PYTHON_EXEC_PATH), 'conda')

channel_directories = set()

#------------------------------------------------------------------------------

DRYRUN_FILES = glob.glob(os.path.join(WORKING_DIR, '*_channel_pkgs.json'))

if len(DRYRUN_FILES) == 0:

    sys.exit("No '*_channel_pkgs.json' files found in '{}'".format(WORKING_DIR))

#------------------------------------------------------------------------------

# Read the conda dry-run json file(s) and download the required packages and their dependency packages.

for dryrun_file in DRYRUN_FILES:

    with open(dryrun_file) as json_file:

        data = json.load(json_file)

        if 'actions' in data:

            for package in data['actions']['FETCH']:

                channel_dir = os.path.join(CHANNELS_BASE_DIR, package['channel'])

                channel_directories.add(channel_dir)

                platform_dir = os.path.join(channel_dir, package['platform'])

                download_file_name = '{}.tar.bz2'.format(package['dist_name'])

                download_file_path = os.path.join(platform_dir, download_file_name)

                if os.path.isfile(download_file_path):

                    print('Package already downloaded: {}'.format(download_file_path))

                    continue

                if not os.path.isdir(platform_dir):

                    os.makedirs(platform_dir)

                download_url = '/'.join(
                    [package['base_url'],
                     package['platform'],
                     download_file_name]
                )

                print('Downloading package file  : {}'.format(download_file_name))

                response = requests.get(download_url, allow_redirects=True)

                if response.status_code == requests.codes.ok:

                    with open(download_file_path, 'wb') as dlfile:

                        dlfile.write(response.content)

                else:

                    print('Could not download: {}'.format(download_url))

#------------------------------------------------------------------------------

# conda index the downloaded files

for channel_dir in channel_directories:

    # conda index expects to see a 'noarch' directory regardless of whether it is used or not.
    noarch_dir = os.path.join(channel_dir, 'noarch')

    if not os.path.isdir(noarch_dir):

        os.makedirs(noarch_dir)

        update_index(noarch_dir)

    update_index(channel_dir)

#------------------------------------------------------------------------------

# Render the README file with offline server install instructions.

template_text = \
"""
# Run this on the offline server
# Extract tar file, attach local channels, install packages, detach local channels

{{python_exec_path}} -m tarfile -e {{channels_tar_file}} {{working_dir}}

{% for channel_dir in channel_directories %}
{{conda_path}} config --add channels file:///{{channel_dir}}
{% endfor %}

{{conda_path}} info


{{conda_path}} install --offline package_name_A package_name_B package_name_C

{% for channel_dir in channel_directories %}
{{conda_path}} config --remove channels file:///{{channel_dir}}
{% endfor %}

{{conda_path}} info

"""

template = Template(template_text)

rendered = template.render(
    channel_directories=channel_directories,
    channels_tar_file=CHANNELS_TAR_FILE,
    python_exec_path=PYTHON_EXEC_PATH,
    working_dir=WORKING_DIR,
    conda_path=CONDA_PATH
)

with open(README_FILE_PATH, 'wt') as fh:

    fh.write(rendered)

#------------------------------------------------------------------------------

# Create the channel(s) files tar set.

with tarfile.open(CHANNELS_TAR_FILE, 'w') as tar:

    tar.add(CHANNELS_BASE_DIR, arcname=BASE_NAME)
    tar.add(README_FILE_PATH, arcname=README_FILE)

#------------------------------------------------------------------------------

print("\nCopy the '{}' file to the offline server.".format(CHANNELS_TAR_FILE))
print("Follow the instructions in '{}'.\n".format(README_FILE))

#------------------------------------------------------------------------------

thanks, it also works in Anaconda3-5.2.0-Linux-x86_64 on Ubuntu 16.04, great!!!!