Skip to content

Instantly share code, notes, and snippets.

@jtriley
Created August 9, 2012 19:28
Show Gist options
  • Save jtriley/3307331 to your computer and use it in GitHub Desktop.
Save jtriley/3307331 to your computer and use it in GitHub Desktop.
Example StarCluster plugin that configures environment variables on all nodes
# This is an example config that assumes scenv.py is either in
# $HOME/.starcluster/plugins or lives somewhere in your $PYTHONPATH
[plugin env]
setup_class = scenv.EnvPlugin
# add as many key=value pairs as you like separated by ','
env_vars_list = 'PYTHONPATH=/path/to/python/mods, MYENV=some_value'
[cluster default]
...
plugins = env
# Install this file to $HOME/.starcluster/plugins/scenv.py or somewhere on your $PYTHONPATH
from starcluster.clustersetup import ClusterSetup
from starcluster.logger import log
class EnvPlugin(ClusterSetup):
def __init__(self, env_vars_list):
self.env_vars_list = '\n'.join([v.strip() for v in env_vars_list.split(',')])
def run(self, nodes, master, user, user_shell, volumes):
log.info("Setting environment on all nodes...")
for node in nodes:
env_file = node.ssh.remote_file('/etc/profile.d/scenv.sh', 'w')
env_file.write(self.env_vars_list)
env_file.close()
@daskinne
Copy link

import os

from starcluster.clustersetup import ClusterSetup
from starcluster.logger import log

class EnvarPlugin(ClusterSetup):
    """
    Plugin that configures environment variables for a cluster
    """
    #vars_hash = {}
    var_str = ""
    envar_location = None
    def __init__(self, var_list=None,var_file=None,user_envar_location=None):
        en_vars = []
        if not var_list is None:
            en_vars = var_list.split(',')
        elif not var_file is None:
            log.info("Adding vars from: %s " % var_file)
            vfile_name = os.path.expanduser(var_file or '') or None
            if vfile_name is not None: 
                fh = open(vfile_name,"r")
                line = fh.readline()
                while not line == "":
                    en_vars.append(line.replace('\n',''))
                    line = fh.readline()
                fh.close()
        for var in en_vars:
            split_index = var.find('=')
            if not split_index == -1:
                #self.vars_hash[var[0:split_index]] = var[split_index:]
                self.var_str += 'export '+var[0:split_index]+'="'+var[split_index+1:].replace('"','\"')+'";\n'
        #log.info("VarCMD %s " % self.var_str)
        self.envar_location = user_envar_location
    def run(self, nodes, master, user, user_shell, volumes):
        if not self.var_str == "":
            for node in nodes:
                log.info("Adding vars to: %s " % node.alias)
                node.ssh.execute('echo \''+self.var_str.replace('\'', '\\\'')+'\' >> .bashrc')
                if self.envar_location is not None:
                    node.ssh.execute('echo \''+self.var_str.replace('\'', '\\\'')+'\' >> '+self.envar_location)

    def on_add_node(self, node, nodes, master, user, user_shell, volumes):
        if not self.var_str == "":
            log.info("Adding vars to: %s " % node.alias)
            node.ssh.execute('echo \''+self.var_str+' >> .bashrc')
            if self.envar_location is not None:
                node.ssh.execute('echo \''+self.var_str.replace('\'', '\\\'')+'\' >> '+self.envar_location)

@scollis
Copy link

scollis commented Jan 21, 2016

should there be an 'export' in the config example?
eg I am getting this:

starcluster sshmaster pyart_sc
Last login: Thu Jan 21 21:09:54 2016 from ip-172-31-55-102.ec2.internal
-bash: PATH=/home/ubuntu/anaconda2/bin:$PATH: No such file or directory

@scollis
Copy link

scollis commented Jan 22, 2016

Trying the comment and no love.. nothing written to .bashrc and no env variables added.. trying to replace base python with anaconda

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment