Skip to content

Instantly share code, notes, and snippets.

@amiller
Created January 26, 2011 08:17
Show Gist options
  • Save amiller/796409 to your computer and use it in GitHub Desktop.
Save amiller/796409 to your computer and use it in GitHub Desktop.
diff --git a/hadoopy/_freeze.py b/hadoopy/_freeze.py
index 0923e30..445ed88 100644
--- a/hadoopy/_freeze.py
+++ b/hadoopy/_freeze.py
@@ -56,12 +56,17 @@ def _wrap_string(s):
def _run(path):
print(path)
+
subprocess.call(path.split())
def _copytree(src, dst):
"""Similar to shutils.copytree, except that dst is already there
"""
+ try:
+ os.mkdir(dst)
+ except OSError:
+ pass # It must already exist
for file in os.listdir(src):
try:
shutil.copy2('%s/%s' % (src, file), '%s/%s' % (dst, file))
diff --git a/hadoopy/_hdfs.py b/hadoopy/_hdfs.py
index a16c253..5771cc9 100755
--- a/hadoopy/_hdfs.py
+++ b/hadoopy/_hdfs.py
@@ -40,7 +40,7 @@ def ls(path):
Raises:
IOError: An error occurred listing the directory (e.g., not available).
"""
- out, err = subprocess.Popen('hadoop fs -ls %s' % path, env={}, shell=True,
+ out, err = subprocess.Popen('hadoop fs -ls %s' % path, shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
if err:
diff --git a/hadoopy/_runner.py b/hadoopy/_runner.py
index d3bad03..4ca9d4b 100644
--- a/hadoopy/_runner.py
+++ b/hadoopy/_runner.py
@@ -161,7 +161,7 @@ def launch(in_name, out_name, script_path, mapper=True, redu
# Run command and wait till it has completed
if not pretend:
print('HadooPY: Running[%s]' % (' '.join(cmd)))
- subprocess.check_call(cmd)
+ subprocess.check_call(cmd,**kw)
return ' '.join(cmd)
@@ -230,5 +230,6 @@ def launch_frozen(in_name, out_name, script_path, **kw):
kw['copy_script'] = False
kw['files'] = files
kw['add_python'] = False
+ kw['shell'] = True
launch_cmd = launch(in_name, out_name, script_path, **kw)
return launch_cmd
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment