fhoering

python3.6 -m venv venv
source venv/bin/activate
mkdir -p userlib/userlib
echo -n > userlib/__init__.py
echo -n > userlib/userlib/__init__.py
echo -n > userlib/userlib/startup.py
echo "import setuptools
 
setuptools.setup(
    name='userlib',

fhoering

import json
import subprocess
import sys

from pex.fetcher import PyPIFetcher
from pex.pex_builder import PEXBuilder
from pex.resolvable import Resolvable
from pex.resolver import resolve_multi, Unsatisfiable, Untranslateable
from pex.resolver_options import ResolverOptionsBuilder

fhoering

import json
import subprocess
import sys
import os
import zipfile


def upload_editable_requirement_from_current_venv(sc):
    for requirement_dir in get_editable_requirements():
        add_lib_to_spark_context(requirement_dir, sc)

fhoering

import os
import sys
import numpy as np
from pyspark import SparkConf, SparkContext
 
def create_spark_context():
    pex_file = os.path.basename([path for path in sys.path if path.endswith('.pex')][0])
    conf = SparkConf() \
        .setMaster("yarn") \
        .set("spark.submit.deployMode", "client") \

fhoering

$ pex pyspark==2.3.2 numpy userlib -o myarchive.pex                 
$./myarchive.pex -m userlib.startup  
14