Testing out some handy dandy pyspark debugging. Note that the output script displays file, line number and code being executed as well as a sample of the result.

1-driver.py

import pyspark

from sparkbuddy.monkey import patch_pyspark
patch_pyspark()


def is_even(val):
    if val % 2 == 0:
        return [val]
    return []


def square(val):
    return val ** 2


with pyspark.SparkContext() as sc:
    logs = sc.parallelize([str(x) for x in xrange(1000)])
    blah = (logs
            .map(lambda l: int(l))
            .map(square)
            .flatMap(is_even))
    blah = blah.map(lambda l: l / 100.0)
    print blah.collect()

2-output.txt

/Users/mikesukmanowsky/code/sparkbuddy/test_driver.py:18: logs = sc.parallelize([str(x) for x in xrange(1000)])
['0', '1', '2', '3', '4']

/Users/mikesukmanowsky/code/sparkbuddy/test_driver.py:20: .map(lambda l: int(l))
[0, 1, 2, 3, 4]

/Users/mikesukmanowsky/code/sparkbuddy/test_driver.py:21: .map(square)
[0, 1, 4, 9, 16]

/Users/mikesukmanowsky/code/sparkbuddy/test_driver.py:22: .flatMap(is_even))
[0, 4, 16, 36, 64]

/Users/mikesukmanowsky/code/sparkbuddy/test_driver.py:23: blah = blah.map(lambda l: l / 100.0)
[0.0, 0.04, 0.16, 0.36, 0.64]

[0.0, 0.04, 0.16, 0.36, 0.64, 1.0, 1.44, 1.96, 2.56, 3.24, 4.0, 4.84, 5.76, 6.76, 7.84, 9.0, 10.24, 11.56, 12.96, 14.44, 16.0, 17.64, 19.36, 21.16, 23.04, 25.0, 27.04, 29.16, 31.36, 33.64, 36.0, 38.44, 40.96, 43.56, 46.24, 49.0, 51.84, 54.76, 57.76, 60.84, 64.0, 67.24, 70.56, 73.96, 77.44, 81.0, 84.64, 88.36, 92.16, 96.04, 100.0, 104.04, 108.16, 112.36, 116.64, 121.0, 125.44, 129.96, 134.56, 139.24, 144.0, 148.84, 153.76, 158.76, 163.84, 169.0, 174.24, 179.56, 184.96, 190.44, 196.0, 201.64, 207.36, 213.16, 219.04, 225.0, 231.04, 237.16, 243.36, 249.64, 256.0, 262.44, 268.96, 275.56, 282.24, 289.0, 295.84, 302.76, 309.76, 316.84, 324.0, 331.24, 338.56, 345.96, 353.44, 361.0, 368.64, 376.36, 384.16, 392.04, 400.0, 408.04, 416.16, 424.36, 432.64, 441.0, 449.44, 457.96, 466.56, 475.24, 484.0, 492.84, 501.76, 510.76, 519.84, 529.0, 538.24, 547.56, 556.96, 566.44, 576.0, 585.64, 595.36, 605.16, 615.04, 625.0, 635.04, 645.16, 655.36, 665.64, 676.0, 686.44, 696.96, 707.56, 718.24, 729.0, 739.84, 750.76, 761.76, 772.84, 784.0, 795.24, 806.56, 817.96, 829.44, 841.0, 852.64, 864.36, 876.16, 888.04, 900.0, 912.04, 924.16, 936.36, 948.64, 961.0, 973.44, 985.96, 998.56, 1011.24, 1024.0, 1036.84, 1049.76, 1062.76, 1075.84, 1089.0, 1102.24, 1115.56, 1128.96, 1142.44, 1156.0, 1169.64, 1183.36, 1197.16, 1211.04, 1225.0, 1239.04, 1253.16, 1267.36, 1281.64, 1296.0, 1310.44, 1324.96, 1339.56, 1354.24, 1369.0, 1383.84, 1398.76, 1413.76, 1428.84, 1444.0, 1459.24, 1474.56, 1489.96, 1505.44, 1521.0, 1536.64, 1552.36, 1568.16, 1584.04, 1600.0, 1616.04, 1632.16, 1648.36, 1664.64, 1681.0, 1697.44, 1713.96, 1730.56, 1747.24, 1764.0, 1780.84, 1797.76, 1814.76, 1831.84, 1849.0, 1866.24, 1883.56, 1900.96, 1918.44, 1936.0, 1953.64, 1971.36, 1989.16, 2007.04, 2025.0, 2043.04, 2061.16, 2079.36, 2097.64, 2116.0, 2134.44, 2152.96, 2171.56, 2190.24, 2209.0, 2227.84, 2246.76, 2265.76, 2284.84, 2304.0, 2323.24, 2342.56, 2361.96, 2381.44, 2401.0, 2420.64, 2440.36, 2460.16, 2480.04, 2500.0, 2520.04, 2540.16, 2560.36, 2580.64, 2601.0, 2621.44, 2641.96, 2662.56, 2683.24, 2704.0, 2724.84, 2745.76, 2766.76, 2787.84, 2809.0, 2830.24, 2851.56, 2872.96, 2894.44, 2916.0, 2937.64, 2959.36, 2981.16, 3003.04, 3025.0, 3047.04, 3069.16, 3091.36, 3113.64, 3136.0, 3158.44, 3180.96, 3203.56, 3226.24, 3249.0, 3271.84, 3294.76, 3317.76, 3340.84, 3364.0, 3387.24, 3410.56, 3433.96, 3457.44, 3481.0, 3504.64, 3528.36, 3552.16, 3576.04, 3600.0, 3624.04, 3648.16, 3672.36, 3696.64, 3721.0, 3745.44, 3769.96, 3794.56, 3819.24, 3844.0, 3868.84, 3893.76, 3918.76, 3943.84, 3969.0, 3994.24, 4019.56, 4044.96, 4070.44, 4096.0, 4121.64, 4147.36, 4173.16, 4199.04, 4225.0, 4251.04, 4277.16, 4303.36, 4329.64, 4356.0, 4382.44, 4408.96, 4435.56, 4462.24, 4489.0, 4515.84, 4542.76, 4569.76, 4596.84, 4624.0, 4651.24, 4678.56, 4705.96, 4733.44, 4761.0, 4788.64, 4816.36, 4844.16, 4872.04, 4900.0, 4928.04, 4956.16, 4984.36, 5012.64, 5041.0, 5069.44, 5097.96, 5126.56, 5155.24, 5184.0, 5212.84, 5241.76, 5270.76, 5299.84, 5329.0, 5358.24, 5387.56, 5416.96, 5446.44, 5476.0, 5505.64, 5535.36, 5565.16, 5595.04, 5625.0, 5655.04, 5685.16, 5715.36, 5745.64, 5776.0, 5806.44, 5836.96, 5867.56, 5898.24, 5929.0, 5959.84, 5990.76, 6021.76, 6052.84, 6084.0, 6115.24, 6146.56, 6177.96, 6209.44, 6241.0, 6272.64, 6304.36, 6336.16, 6368.04, 6400.0, 6432.04, 6464.16, 6496.36, 6528.64, 6561.0, 6593.44, 6625.96, 6658.56, 6691.24, 6724.0, 6756.84, 6789.76, 6822.76, 6855.84, 6889.0, 6922.24, 6955.56, 6988.96, 7022.44, 7056.0, 7089.64, 7123.36, 7157.16, 7191.04, 7225.0, 7259.04, 7293.16, 7327.36, 7361.64, 7396.0, 7430.44, 7464.96, 7499.56, 7534.24, 7569.0, 7603.84, 7638.76, 7673.76, 7708.84, 7744.0, 7779.24, 7814.56, 7849.96, 7885.44, 7921.0, 7956.64, 7992.36, 8028.16, 8064.04, 8100.0, 8136.04, 8172.16, 8208.36, 8244.64, 8281.0, 8317.44, 8353.96, 8390.56, 8427.24, 8464.0, 8500.84, 8537.76, 8574.76, 8611.84, 8649.0, 8686.24, 8723.56, 8760.96, 8798.44, 8836.0, 8873.64, 8911.36, 8949.16, 8987.04, 9025.0, 9063.04, 9101.16, 9139.36, 9177.64, 9216.0, 9254.44, 9292.96, 9331.56, 9370.24, 9409.0, 9447.84, 9486.76, 9525.76, 9564.84, 9604.0, 9643.24, 9682.56, 9721.96, 9761.44, 9801.0, 9840.64, 9880.36, 9920.16, 9960.04]