output.json

(def test-graph  ;each pipeline can only be defined in an origin once?
  {:provider  {:account-file "blah.json" :project "hx-test" :region "us-central1"}
   :pipelines {"Pipeline1"
               {:transform-graph ["AbstractTransform/AbstractTransform.jar"]}
               "Pipeline2"
               {:transform-graph ["AbstractTransform/AbstractTransform2.jar"]}
               "Pipeline3"
               {:transform-graph ["PythonScripts/PipelinePy3.py"]}
               "Pipeline4"
               {:transform-graph ["AbstractTransform/Pipeline4.jar"]}
               "Pipeline5"
               {:transform-graph ["PythonScripts/Pipeline5.py"]}
               #_"RepairJob"
               #_{:transform-graph ["PythonScripts/RepairClickstream.py"]}
               }
   :sources   [{:package "ClickStream"} {:package "ShoppingCartClickStream"} {:package "CartTransaction"} {:package "EmailClickStream"}]
   :sinks     [{:package "S3Sink"} {:package "EnhancedAdwordsSink"} {:package "EnhancedAdwordsSink"} {:package "CartTrans-Redshift"} {:package "EmailClicks-RedShift"} {:package "RawEmailS3"}]
   :edges     [{:origin "Pipeline1" :targets ["Pipeline2"]}
               {:origin "ClickStream" :targets ["Pipeline1"]}
               {:origin        "Pipeline2"
                :error-sink    :google-storage ;implied sink of gcs directory "Pipeline2/errors/blah"
                :error-handler "RepairJob" ;implied source of gcs directory "Pipeline2/errors/blah"
                :targets       ["S3Sink" "EnhancedAdwordsSink"]}
               {:origin "ShoppingCartClickStream" :targets ["Pipeline3"]}
               {:origin "Pipeline3" :targets ["Pipeline2"]}
               {:origin "CartTransaction" :targets ["Pipeline4"]}
               {:origin "Pipeline4" :targets ["CartTrans-Redshift"  ]} ;implied {:name "CartTrans-Redshift"}, and write to a RedShift table and write to the post-processing queue of Pipeline2? {"Pipeline2" :type "post"}
               {:origin "EmailClickstream" :targets ["Pipeline5" "RawEmailS3"]}
               {:origin "Pipeline5" :targets ["Emailclicks-RedShift"]}]})




[ {
  "resource" : {
    "google_pubsub" : {
      "EmailClickstream-out" : {
        "name" : "EmailClickstream-out"
      }
    }
  }
}, {
  "resource" : {
    "google_pubsub" : {
      "EmailClickstream-err" : {
        "name" : "EmailClickstream-err"
      }
    }
  }
}, {
  "resource" : {
    "google_pubsub" : {
      "Pipeline5-out" : {
        "name" : "Pipeline5-out"
      }
    }
  }
}, {
  "resource" : {
    "google_pubsub" : {
      "Pipeline5-err" : {
        "name" : "Pipeline5-err"
      }
    }
  }
}, {
  "resource" : {
    "google_pubsub" : {
      "Pipeline3-out" : {
        "name" : "Pipeline3-out"
      }
    }
  }
}, {
  "resource" : {
    "google_pubsub" : {
      "Pipeline3-err" : {
        "name" : "Pipeline3-err"
      }
    }
  }
}, {
  "resource" : {
    "google_pubsub" : {
      "Pipeline2-out" : {
        "name" : "Pipeline2-out"
      }
    }
  }
}, {
  "resource" : {
    "google_pubsub" : {
      "Pipeline2-err" : {
        "name" : "Pipeline2-err"
      }
    }
  }
}, {
  "resource" : {
    "google_pubsub" : {
      "CartTransaction-out" : {
        "name" : "CartTransaction-out"
      }
    }
  }
}, {
  "resource" : {
    "google_pubsub" : {
      "CartTransaction-err" : {
        "name" : "CartTransaction-err"
      }
    }
  }
}, {
  "resource" : {
    "google_pubsub" : {
      "Pipeline4-out" : {
        "name" : "Pipeline4-out"
      }
    }
  }
}, {
  "resource" : {
    "google_pubsub" : {
      "Pipeline4-err" : {
        "name" : "Pipeline4-err"
      }
    }
  }
}, {
  "resource" : {
    "google_pubsub" : {
      "Pipeline1-out" : {
        "name" : "Pipeline1-out"
      }
    }
  }
}, {
  "resource" : {
    "google_pubsub" : {
      "Pipeline1-err" : {
        "name" : "Pipeline1-err"
      }
    }
  }
}, {
  "resource" : {
    "google_pubsub" : {
      "ShoppingCartClickStream-out" : {
        "name" : "ShoppingCartClickStream-out"
      }
    }
  }
}, {
  "resource" : {
    "google_pubsub" : {
      "ShoppingCartClickStream-err" : {
        "name" : "ShoppingCartClickStream-err"
      }
    }
  }
}, {
  "resource" : {
    "google_pubsub" : {
      "ClickStream-out" : {
        "name" : "ClickStream-out"
      }
    }
  }
}, {
  "resource" : {
    "google_pubsub" : {
      "ClickStream-err" : {
        "name" : "ClickStream-err"
      }
    }
  }
}, {
  "resource" : {
    "google_pubsub" : {
      "RawEmailS3-error" : {
        "name" : "RawEmailS3-error"
      }
    }
  }
}, {
  "resource" : {
    "google_pubsub" : {
      "Emailclicks-RedShift-error" : {
        "name" : "Emailclicks-RedShift-error"
      }
    }
  }
}, {
  "resource" : {
    "google_pubsub" : {
      "S3Sink-error" : {
        "name" : "S3Sink-error"
      }
    }
  }
}, {
  "resource" : {
    "google_pubsub" : {
      "EnhancedAdwordsSink-error" : {
        "name" : "EnhancedAdwordsSink-error"
      }
    }
  }
}, {
  "resource" : {
    "google_pubsub" : {
      "CartTrans-Redshift-error" : {
        "name" : "CartTrans-Redshift-error"
      }
    }
  }
}, {
  "resource" : {
    "google_dataflow" : {
      "EmailClickstream" : {
        "name" : "EmailClickstream",
        "class" : "Main.class",
        "depends_on" : [ "google_pubsub.Pipeline5-out", "google_pubsub.RawEmailS3-out" ],
        "project" : "hx-test"
      }
    }
  }
}, {
  "resource" : {
    "google_dataflow" : {
      "Pipeline5" : {
        "name" : "Pipeline5",
        "class" : "Main.class",
        "depends_on" : [ "google_pubsub.Emailclicks-RedShift-out", "google_pubsub.EmailClickstream-out", "google_dataflow.EmailClickstream" ],
        "project" : "hx-test"
      }
    }
  }
}, {
  "resource" : {
    "google_dataflow" : {
      "RawEmailS3" : {
        "name" : "RawEmailS3",
        "class" : "Main.class",
        "depends_on" : [ "google_pubsub.EmailClickstream-out", "google_dataflow.EmailClickstream" ],
        "project" : "hx-test"
      }
    }
  }
}, {
  "resource" : {
    "google_dataflow" : {
      "Emailclicks-RedShift" : {
        "name" : "Emailclicks-RedShift",
        "class" : "Main.class",
        "depends_on" : [ "google_pubsub.Pipeline5-out", "google_dataflow.Pipeline5" ],
        "project" : "hx-test"
      }
    }
  }
}, {
  "resource" : {
    "google_dataflow" : {
      "Pipeline3" : {
        "name" : "Pipeline3",
        "class" : "Main.class",
        "depends_on" : [ "google_pubsub.Pipeline2-out", "google_pubsub.ShoppingCartClickStream-out", "google_dataflow.ShoppingCartClickStream" ],
        "project" : "hx-test"
      }
    }
  }
}, {
  "resource" : {
    "google_dataflow" : {
      "Pipeline2" : {
        "name" : "Pipeline2",
        "class" : "Main.class",
        "depends_on" : [ "google_pubsub.S3Sink-out", "google_pubsub.EnhancedAdwordsSink-out", "google_pubsub.Pipeline3-out", "google_pubsub.Pipeline1-out", "google_dataflow.Pipeline3", "google_dataflow.Pipeline1" ],
        "project" : "hx-test"
      }
    }
  }
}, {
  "resource" : {
    "google_dataflow" : {
      "S3Sink" : {
        "name" : "S3Sink",
        "class" : "Main.class",
        "depends_on" : [ "google_pubsub.Pipeline2-out", "google_dataflow.Pipeline2" ],
        "project" : "hx-test"
      }
    }
  }
}, {
  "resource" : {
    "google_dataflow" : {
      "EnhancedAdwordsSink" : {
        "name" : "EnhancedAdwordsSink",
        "class" : "Main.class",
        "depends_on" : [ "google_pubsub.Pipeline2-out", "google_dataflow.Pipeline2" ],
        "project" : "hx-test"
      }
    }
  }
}, {
  "resource" : {
    "google_dataflow" : {
      "CartTransaction" : {
        "name" : "CartTransaction",
        "class" : "Main.class",
        "depends_on" : [ "google_pubsub.Pipeline4-out" ],
        "project" : "hx-test"
      }
    }
  }
}, {
  "resource" : {
    "google_dataflow" : {
      "Pipeline4" : {
        "name" : "Pipeline4",
        "class" : "Main.class",
        "depends_on" : [ "google_pubsub.CartTrans-Redshift-out", "google_pubsub.CartTransaction-out", "google_dataflow.CartTransaction" ],
        "project" : "hx-test"
      }
    }
  }
}, {
  "resource" : {
    "google_dataflow" : {
      "CartTrans-Redshift" : {
        "name" : "CartTrans-Redshift",
        "class" : "Main.class",
        "depends_on" : [ "google_pubsub.Pipeline4-out", "google_dataflow.Pipeline4" ],
        "project" : "hx-test"
      }
    }
  }
}, {
  "resource" : {
    "google_dataflow" : {
      "Pipeline1" : {
        "name" : "Pipeline1",
        "class" : "Main.class",
        "depends_on" : [ "google_pubsub.Pipeline2-out", "google_pubsub.ClickStream-out", "google_dataflow.ClickStream" ],
        "project" : "hx-test"
      }
    }
  }
}, {
  "resource" : {
    "google_dataflow" : {
      "ShoppingCartClickStream" : {
        "name" : "ShoppingCartClickStream",
        "class" : "Main.class",
        "depends_on" : [ "google_pubsub.Pipeline3-out" ],
        "project" : "hx-test"
      }
    }
  }
}, {
  "resource" : {
    "google_dataflow" : {
      "ClickStream" : {
        "name" : "ClickStream",
        "class" : "Main.class",
        "depends_on" : [ "google_pubsub.Pipeline1-out" ],
        "project" : "hx-test"
      }
    }
  }
} ]