executor.diff

1 file changed, 6 insertions(+)
 .../{executors/python.py => plugins/spark/pyspark_executor.py}      | 6 ++++++

modified   home/dan/projects/romanesco/romanesco/executors/python.py -> home/dan/projects/romanesco/romanesco/plugins/spark/pyspark_executor.py
@@ -4,8 +4,14 @@ import sys
 
 
 def run(task, inputs, outputs, task_inputs, task_outputs, **kwargs):
+    from . import SC_KEY
+
     custom = imp.new_module("custom")
 
+    sc = kwargs[SC_KEY]
+    custom.__dict__['sc'] = sc
+    custom.__dict__['spark_context'] = sc
+
     for name in inputs:
         custom.__dict__[name] = inputs[name]["script_data"]