drj42

Org-mode Org-mode Reference Card

Org mode reference card

Created from the plain text reference card on orgmode.org Download this file, and open it in Emacs org-mode!

drj42

#!/usr/bin/env perl
use Mojolicious::Lite;

# Instructions:
# $ cpanm Mojolicious
# $ morbo grave_names.pl
#
# browse to http://localhost:3000

# Documentation browser under "/perldoc"

drj42

from luigi.format import Format
import csvkit

class CSVOutputProcessor(object):
    """
    A simple CSV output processor to be hooked into Format's
    `pipe_writer`. 

    If `cols` are given, the names are used as CSV header, otherwise no
    explicit header is written.

drj42

"""
Launch pyspark with the flag:

--packages com.databricks:spark-csv_2.10:1.0.3
""" 
from pyspark.sql import SQLContext
from pyspark.sql.types import StructField, StringType, StructType

def create_schema(fields):
    """ Create a dataframe schema from a list of field names.

drj42

import luigi


class Foo(luigi.Task):
    message = 'Foo'


class RunOnceTask(luigi.Task):
    my_task = luigi.TaskParameter()

drj42

"""sftp.py - PySftp connections wrapped up in a luigi.Target.

TODO: get rid of the redundant stuff, write some tests, contribute to luigi
      upstream.
"""
# -*- coding: utf-8 -*-

import io
import os
import random

drj42

# Removes a lot of the boiler plate involved in converting a pyspark dataframe
# to and from an rdd, in order to do a reduceByKey operation. 
#
# Lifted from:
# - http://codereview.stackexchange.com/questions/115082/generic-reduceby-or-groupby-aggregate-functionality-with-spark-dataframe


from pyspark.sql import Row
from pyspark.sql.functions import struct
from pyspark.sql import DataFrame

drj42

import hashlib
import hmac
import mimetypes

import binascii
from calendar import timegm
from datetime import datetime
import time
from email.utils import formatdate
from urllib.parse import quote, urlparse

drj42

import asyncio
import random

q = asyncio.Queue()


async def producer(num):
    while True:
        await q.put(num + random.random())
        await asyncio.sleep(random.random())

drj42

import asyncio
from datetime import datetime, timezone
import os

def utc_now():
    # utcnow returns a naive datetime, so we have to set the timezone manually <sigh>
    return datetime.utcnow().replace(tzinfo=timezone.utc)
  
class Terminator:
  pass