seandavi

Apache Airflow setup

base installations

sudo apt-get update
sudo apt-get install python3 python3-pip virtualenv
# sudo apt-get install emacs tmux 

seandavi

"""parallel bulk indexing

Indexes a json file using parallel_bulk.
"""

import elasticsearch
from elasticsearch import Elasticsearch
from elasticsearch.helpers import parallel_bulk as pb

from collections import deque

seandavi

#!/bin/bash
# Start EKS cluster on AWS

# Install eksctl (https://eksctl.io/)
#   On mac, homebrew
brew tap weaveworks/tap
brew install weaveworks/tap/eksctl


# start cluster (takes a few minutes)

seandavi

---
title: "Playing with OmicIDX"
author: "Sean Davis"
date: "3/14/2019"
output: 
  BiocStyle::html_document:
    toc_float: True
---

# Introduction to the OmicIDX API

seandavi

seandavi

FROM ubuntu:18.04

RUN apt-get update
RUN apt-get install -y wget 

# We do things this way to keep the docker image
# size down. See https://nickjanetakis.com/blog/docker-tip-3-chain-your-docker-run-instructions-to-shrink-your-images
RUN wget http://ftp-trace.ncbi.nlm.nih.gov/sra/sdk/2.9.2/sratoolkit.2.9.2-ubuntu64.tar.gz \
    && tar -xvzf sratoolkit.2.9.2-ubuntu64.tar.gz \
    && rm sratoolkit.2.9.2-ubuntu64.tar.gz

seandavi

from __future__ import print_function
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.io.filesystems import FileSystems
import urllib
import json
import argparse
import logging
logging.basicConfig(level=logging.INFO)

seandavi

---
title: "fusion genes"
output: 
  html_document:
    self_contained: true
---


```{r include=FALSE}
library(knitr)

seandavi

# requires python 2.7
# pip install apache_beam
from __future__ import print_function
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
import json
import argparse
import logging
import urllib2
import urllib

seandavi

{
  "sra_experiment_joined2": {
    "mappings": {
      "doc": {
        "properties": {
          "library_name": {
            "fields": {
              "keyword": {
                "ignore_above": 256,
                "type": "keyword"