provisioner, WIP

provision.groovy

#!/usr/bin/env groovy
/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 * <p/>
 * http://www.apache.org/licenses/LICENSE-2.0
 * <p/>
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import groovy.json.JsonSlurper;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.Writer;
import org.apache.hadoop.fs.Path;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.bigtop.itest.shell.Shell

def LOG = LogFactory.getLog(this.getClass());

def USAGE = """\
        USAGE:
        This script provisions the skeleton of a hadoop file system.
    It takes a single argument: The json schema (a list of lists),
    of 4 element tuples.  For an example , see the bigtop init-hcfs.json
    file.  The main elements of the JSON file are:

    dir: list of dirs to create with permissions.
    user: list of users to setup home dirs with permissions.
    root_user: The root owner of distributed FS, to run shell commands.

    To run this code as a standalone groovy script, you can make the full
    classpath,  and you can invoke with something like:

    --classpath usr/lib/hadoop/hadoop-common-2.0.6-alpha.jar:
    \$M2_HOME/org/apache/bigtop/itest/itest-common/
    0.8.0-SNAPSHOT/itest-common-0.8.0-SNAPSHOT.jar:
    /usr/lib/hadoop/lib/guava-11.0.2.jar:
    /etc/hadoop/conf/:
    /usr/lib/hadoop/hadoop-common-2.0.6-alpha.jar:
    /usr/lib/hadoop/lib/commons-configuration-1.6.jar:
    /usr/lib/hadoop/lib/commons-lang-2.5.jar:
    /usr/lib/hadoop/hadoop-auth.jar:
    /usr/lib/hadoop/lib/slf4j-api-1.6.1.jar:
    /usr/lib/hadoop-hdfs/hadoop-hdfs.jar:
    /usr/lib/hadoop/lib/protobuf-java-2.4.0a.jar

"""

/**
 * The HCFS generic provisioning process:
 *   1) Create a file system skeleton.
 *   2) Create users with home dirs in /user.
 *   In the future, we can add more args here.
 **/


File validateInputs(Map inputs){

  if(! inputs["json-file"].exists()) {
    System.err.println("Cannot find file " + args[0]);
    System.exit(1) 1;
  }

}

Map handleInput(stringargs) {

  def inputs=[
      "json-file":null
  ];

  if(! args.size == expec_args.size()) {
    System.err.println(USAGE);
    System.err.println(expect_args);
    System.exit(1) ;
  }

  /**
   * Put the inputs into a map here...
   * */
  inputs["json-file"]=new File(args[0]);

  return inputs;
}

File readJson(jsonfile){

  def v = new JsonSlurper();
  def jsonParser = new JsonSlurper();
  def parsedData = jsonParser.parse( new FileReader(jsonfile));
  //return all 3 as objects root_user,dir,user
  [
      parsedData.get("root_user") as String,
      parsedData.get("dir") as [],
      parsedData.get("user") as []
  ]
}


provision_dirs(root_user, dirs){
  LOG.info("\nPROVISIONING WITH FILE SYSTEM : " + fs.getClass()+"\n");
  dirs.each() {
    System.out.println("here " + it);
    def (name,mode,user,group) = it;
    Path file = new Path(name);
    LOG.info("mkdirs " + name + " " + mode);
    LOG.info("Owner " + name + " " + user + " " + group);
    if(mode == null) {
      fs.mkdirs(new Path(name));
    }
    else {
      fs.mkdirs(new Path(name), new FsPermission((short)mode));
    }
    if(user != null){
      fs.setOwner(new Path(name), user, group);
    }
    else
      LOG.info("Skipping ... user null");
  }
}

provision_users(List users, Configuration conf){
  users.each() {
    def (user,permission,group) = it;
    LOG.info("current user: " + user);
    Path homedir = new Path("/user/"+user);
    //perms should be ALL, RX,RX ^^

    fs.mkdirs(homedir);
    fs.setOwner(homedir, user, group);
    FsPermission perm = FsPermission.createImmutable((short)permission);
    fs.setPermission(homedir, perm);
  }
}

oozie_libs(Shell sh){

  LOG.info("Now running some basic shell commands for setting up oozie shared libraries.");

  if ( new File(HIVE_HOME).exists()) {
    sh.exec("hadoop fs -put " +
        "${HIVE_HOME}/lib/*jar " +
        "${OOZIE_LIBS}/lib/hive");
  }
  if ( new File(MAPRED_HOME).exists()) {
    sh.exec("hadoop fs -put " +
        "${MAPRED_HOME}/hadoop-streaming*.jar " +
        "${OOZIE_LIBS}/lib/mapreduce-streaming");
  }
  sh.exec("hadoop fs -put " +
      "${MAPRED_HOME}/hadoop-distcp*.jar " +
      "${OOZIE_LIBS}/lib/distcp");

  if ( new File(PIG_HOME).exists()) {
    sh.exec("hadoop fs -put " +
        "${PIG_HOME}/{lib/,}*jar " +
        "${OOZIE_LIBS}/lib/pig");
  }

}

Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
Shell sh = new Shell("/bin/bash -s",root_user);

PIG_HOME="/usr/lib/pig"
HIVE_HOME="/usr/lib/hive"
MAPRED_HOME="/usr/lib/hadoop-mapreduce"
OOZIE_LIBS="/usr/lib/oozie/share"
 /**
 * The lines below here probably are not likely to change.
 * */

Map inputs = handleInput(args);
def (hcfs_json) = validateInputs(inputs);
def (String root_user, List dirs, List users) = readJson(hcfs_json);
provision_dirs(root_user,dirs);
provision_users(users)
oozie_libs(sh);