High Level Overview of the SMRT Link Secondary Analysis System

advanced-jobs.dot

digraph {

  subgraph cluster_0 {
  c0_Job [shape=hexagon, color=blue, label="Import DataSet Job"]
  c0_store [shape=cylinder, label="DataStore"]
  c0_ep1 [shape=diamond, label="Path /path/to/alpha_subreadset.xml"]
  c0_ep1 -> c0_Job
  c0_Job -> c0_store
  c0_dsf_02 [shape=tab, label="DataStoreFile (SubreadSet)"]
  c0_dsf_04 [shape=tab, label="DataStoreFile (Report_01)"]
  c0_dsf_05 [shape=tab, label="DataStoreFile (Report_02)"]
  c0_dsf_03 [shape=tab, label="DataStoreFile (Log)"]
  c0_store -> c0_dsf_02
  c0_store -> c0_dsf_04
  c0_store -> c0_dsf_05
  c0_store -> c0_dsf_03
  c0_dsf_02 -> c0_dsf_04 [style=dotted]
  c0_dsf_02 -> c0_dsf_05 [style=dotted]
  }

  subgraph cluster_2 {
  c2_Job [shape=hexagon, color=blue, label="Import DataSet Job"]
  c2_store [shape=cylinder, label="DataStore"]
  c2_ep1 [shape=diamond, label="Path /path/to/beta_subreadset.xml"]
  c2_ep1 -> c2_Job
  c2_Job -> c2_store
  c2_dsf_02 [shape=tab, label="DataStoreFile (SubreadSet)"]
  c2_dsf_04 [shape=tab, label="DataStoreFile (Report_01)"]
  c2_dsf_05 [shape=tab, label="DataStoreFile (Report_02)"]
  c2_dsf_03 [shape=tab, label="DataStoreFile (Log)"]
  c2_store -> c2_dsf_02
  c2_store -> c2_dsf_04
  c2_store -> c2_dsf_05
  c2_store -> c2_dsf_03
  c2_dsf_02 -> c2_dsf_04 [style=dotted]
  c2_dsf_02 -> c2_dsf_05 [style=dotted]
  }



    subgraph cluster_3 {
  c3_Job [shape=hexagon, color=blue, label="Merge DataSets Job"]
  c3_store [shape=cylinder, label="DataStore"]
  c3_ep1 [shape=diamond, label="Path /path/to/gamma_subreadset.xml"]
  c3_ep1 -> c3_Job
  c3_Job -> c3_store
  c3_dsf_02 [shape=tab, label="DataStoreFile (Merged SubreadSet)"]
  c3_dsf_04 [shape=tab, label="DataStoreFile (Report_01)"]
  c3_dsf_05 [shape=tab, label="DataStoreFile (Report_02)"]
  c3_dsf_03 [shape=tab, label="DataStoreFile (Log)"]
  c3_store -> c3_dsf_02
  c3_store -> c3_dsf_04
  c3_store -> c3_dsf_05
  c3_store -> c3_dsf_03
  c3_dsf_02 -> c3_dsf_04 [style=dotted]
  c3_dsf_02 -> c3_dsf_05 [style=dotted]
  }



 subgraph cluster_4 {
  c4_Job [shape=hexagon, color=blue, label="Import DataSet Job"]
  c4_store [shape=cylinder, label="DataStore"]
  c4_ep1 [shape=diamond, label="Path /path/to/referenceset.xml"]
  c4_ep1 -> c4_Job
  c4_Job -> c4_store
  c4_dsf_02 [shape=tab, label="DataStoreFile (ReferenceSet)"]
  c4_dsf_03 [shape=tab, label="DataStoreFile (Log)"]
  c4_store -> c4_dsf_02
  c4_store -> c4_dsf_03
  }

 subgraph cluster_5 {
  c5_Job [shape=hexagon, color=blue, label="Copy (and filter) DataSet"]
  c5_store [shape=cylinder, label="DataStore"]
  c5_ep1 [shape=diamond, label="DataSet UUID=X,filter=rq >= 0.7"]
  c5_ep1 -> c5_Job
  c5_Job -> c5_store
  c5_dsf_02 [shape=tab, label="DataStoreFile (SubreadSet)"]
  c5_dsf_04 [shape=tab, label="DataStoreFile (Report_01)"]
  c5_dsf_05 [shape=tab, label="DataStoreFile (Report_02)"]
  c5_dsf_03 [shape=tab, label="DataStoreFile (Log)"]
  c5_store -> c5_dsf_02
  c5_store -> c5_dsf_04
  c5_store -> c5_dsf_05
  c5_store -> c5_dsf_03
  c5_dsf_02 -> c5_dsf_04 [style=dotted]
  c5_dsf_02 -> c5_dsf_05 [style=dotted]
  }

 subgraph cluster_6 {
  c6_Job [shape=hexagon, color=blue, label="Export DataSet(s) Zip Job"]
  c6_store [shape=cylinder, label="DataStore"]
  c6_ep1 [shape=diamond, label="DataSet UUIDs=X,Y,Z"]
  c6_ep1 -> c6_Job
  c6_Job -> c6_store
  c6_dsf_02 [shape=tab, label="DataStoreFile DataSet XML(s) Zip"]
  c6_dsf_03 [shape=tab, label="DataStoreFile (Log)"]
  c6_store -> c6_dsf_02
  c6_store -> c6_dsf_03
  }

  subgraph cluster_01 {
  c1_Job [shape=hexagon, color=blue, label="Analysis Job"]
  c1_store [shape=cylinder, label="DataStore"]
  c1_ep1 [shape=diamond, label="EntryPoint (SubreadSet)"]
  c1_ep2 [shape=diamond, label="EntryPoint (ReferenceSet)"]
  c1_ep1 -> c1_Job
  c1_ep2 -> c1_Job
  c1_Job -> c1_store
  c1_dsf_01 [shape=tab, label="DataStoreFile (Fasta)"]
  c1_dsf_02 [shape=tab, label="DataStoreFile (AlignmentSet)"]
  c1_dsf_03 [shape=tab, label="DataStoreFile (VCF)"]
  c1_dsf_04 [shape=tab, label="DataStoreFile (Report_01)"]
  c1_dsf_05 [shape=tab, label="DataStoreFile (Report_02)"]
  c1_dsf_06 [shape=tab, label="DataStoreFile (LOG)"]
  c1_store -> c1_dsf_01
  c1_store -> c1_dsf_02
  c1_store -> c1_dsf_03
  c1_store -> c1_dsf_04
  c1_store -> c1_dsf_05
  c1_store -> c1_dsf_06
  c1_ep1 -> c1_dsf_04 [style=dotted]
  c1_dsf_02 -> c1_dsf_05 [style=dotted]
  c1_ep2 -> c1_dsf_05 [style=dotted]
  }

  c3_dsf_02 -> c1_ep1
  c4_dsf_02 -> c1_ep2

  c0_dsf_02 -> c5_ep1

  c2_dsf_02 -> c3_ep1
  c5_dsf_02 -> c3_ep1

  c5_dsf_02 -> c6_ep1
} 

advanced-jobs.png

advanced-jobs.svg

ArchOverview.md

Arch Overview

Core Nouns of the PacBio System

1. Run (often created/edited from SMRT Link RunDesign, stored as XML)
2. CollectionMetadata a Run has a list of Collection (Primary Analysis will convert a CollectionMetadata to a SubreadSet)
3. PacBio DataSets SubreadSet, ReferenceSet, etc... These are thin-ish XML files that have general metadata as well as pointers to 'external resources' (e.g., BAM, Fasta files) and their companion index files.
4. SMRT Link Job A general (async) unit of work to perform operations on PacBio DataSets
5. ** DataStoreFile** a container for output files from a SMRT Link Job and contains metadata, such as file type, size, path. A list of DataStore Files is called a DataStore. This is the core output of SMRT Link Job.
6. ** Report** a Report is general model to capture Report metrics (also referred to as 'Attributes'), Report Tables and Report Plot Groups. A Report is a specific type of DataStoreFile and are used to communicate details of a SMRT Link Job to the SMRT Link UI (and webservices.)

Second tier models, such as Report View Rules, or Pipeline View Rules are not discussed here.

General Workflow starting from PA

ICS/PA takes a Run XML with a list of Collections, converts each CollectionMeta into a SubreadSet. The SubreadSet is copied from ICS/PA file system into the customer storage on NFS (accessible by the companion SMRT Link instance) and the SubreadSet XML is imported into SMRT Link using the import-dataset Job type in SMRT Link. The Reports for the SubreadSet XML emitted from the import-dataset job show up in RunQC as well as in DataManagement in SMRT Link.

Show below is a sketch of the dataflow.

General SMRT Link Job Model

Simplify, the general interface of a SMRT Link Job, for type T,

A Job takes T as input and produces a PB (T -> Job -> DataStore)

List of EntryPoint PB DataSet -> Job -> DataStore

A DataStore is a list of DataStore files.

Each DataStoreFile can be a different file types, such as, PB DataSet, VCF, ReportJSON, Fasta, etc... and also contains the specific ob id and UUID that generated the DataStoreFile.

During and after SMRT Link Job execution, the DataStoreFiles will be imorted into the db, the DataStoreFile. For a specific subset of file types (PB DataSet types), additional metadata will be stored in the SMRT Link database. Each DataSet has metadata about the specific dataset type as well as metadata about a possible 'parent' DataSet. The DataSet 'parentage' can be a result from copying, merging, analysis (the semantics are not consistent).

Report Details

Each ReportJson file type contains a list of PB Dataset UUIDs in the data model. This is used to communicate which DataSets are specific to the input(s) of a specific ReportJSON. Alternatively said, the EntryPoint PB DataSet(s) might not be directly used to compute the ReportJson* datastore file..

Example Jobs

NOTE, the dotted arrow represents the relation between the Report and the source input for the task at the Report JSON level. This is NOT captured at the SMRT Link Server level.

Import DataSet Job

Accessing the Reports and the source DataSet is clearly defined here by only depending on the Job Id.

I believe the Merge DataSet Job type is Similar.

Example Resequencing Job

Example: Larger Picture of DataFlow in SMRT Link using SMRT Link Jobs

Simple Example

To perform a standard Resequencing Job, the user can run two different import-dataset SMRT Link Jobs, then a pbsmrtpipe (i.e., 'Analysis') SMRT Link Job can be performed.

Steps:

1. Import SubreadSet
2. Import ReferenceSet
3. Run Analysis Job to run the Resequencing Analysis

(Each Job type is shown in its own box)

Advanced Example

To demonstrate a larger dataflow example, consider the following case. A user would like to import SubreadSet alpha and beta, perform filtering on beta, merge the datasets, perform a Resequencing analysis on the merged subreadset and export the filtered SubreadSet as a zip.

Steps:

1. Import ReferenceSet, SubreadSet alpha and Beta
2. Create a filtered SubreadSet from SubreadSet alpha
3. Create a Merged SubreadSet from SubreadSet Beta and the output of #2
4. Create an Analysis Job using #3 and ReferenceSet from #1
5. Create a DataSet XML(s) ZIP from the output of #3

This demonstrates graph nature of the design and composibility of different SMRT Link Job types. Note that data provenance is for free in the model.

connected-jobs.dot

digraph {

  subgraph cluster_0 {
  c0_Job [shape=hexagon, color=blue, label="Import DataSet"]
  c0_store [shape=cylinder, label="DataStore"]
  c0_ep1 [shape=diamond, label="Path /path/to/subreadset.xml"]
  c0_ep1 -> c0_Job
  c0_Job -> c0_store
  c0_dsf_02 [shape=tab, label="DataStoreFile (SubreadSet)"]
  c0_dsf_04 [shape=tab, label="DataStoreFile (Report_01)"]
  c0_dsf_05 [shape=tab, label="DataStoreFile (Report_02)"]
  c0_dsf_03 [shape=tab, label="DataStoreFile (Log)"]
  c0_store -> c0_dsf_02
  c0_store -> c0_dsf_04
  c0_store -> c0_dsf_05
  c0_store -> c0_dsf_03
  c0_dsf_02 -> c0_dsf_04 [style=dotted]
  c0_dsf_02 -> c0_dsf_05 [style=dotted]
  }

 subgraph cluster_2 {
  c2_Job [shape=hexagon, color=blue, label="Import DataSet"]
  c2_store [shape=cylinder, label="DataStore"]
  c2_ep1 [shape=diamond, label="Path /path/to/referenceset.xml"]
  c2_ep1 -> c2_Job
  c2_Job -> c2_store
  c2_dsf_02 [shape=tab, label="DataStoreFile (ReferenceSet)"]
  c2_dsf_03 [shape=tab, label="DataStoreFile (Log)"]
  c2_store -> c2_dsf_02
  c2_store -> c2_dsf_03
  }

  subgraph cluster_02 {
  c1_Job [shape=hexagon, color=blue, label="Analysis Job"]
  c1_store [shape=cylinder, label="DataStore"]
  c1_ep1 [shape=diamond, label="EntryPoint (SubreadSet)"]
  c1_ep2 [shape=diamond, label="EntryPoint (ReferenceSet)"]
  c1_ep1 -> c1_Job
  c1_ep2 -> c1_Job
  c1_Job -> c1_store
  c1_dsf_01 [shape=tab, label="DataStoreFile (Fasta)"]
  c1_dsf_02 [shape=tab, label="DataStoreFile (AlignmentSet)"]
  c1_dsf_03 [shape=tab, label="DataStoreFile (VCF)"]
  c1_dsf_04 [shape=tab, label="DataStoreFile (Report_01)"]
  c1_dsf_05 [shape=tab, label="DataStoreFile (Report_02)"]
  c1_dsf_06 [shape=tab, label="DataStoreFile (LOG)"]
  c1_store -> c1_dsf_01
  c1_store -> c1_dsf_02
  c1_store -> c1_dsf_03
  c1_store -> c1_dsf_04
  c1_store -> c1_dsf_05
  c1_store -> c1_dsf_06
  c1_ep1 -> c1_dsf_04 [style=dotted]
  c1_dsf_02 -> c1_dsf_05 [style=dotted]
  c1_ep2 -> c1_dsf_05 [style=dotted]
  }

  c2_dsf_02 -> c1_ep2
  c0_dsf_02 -> c1_ep1
  
} 

connected-jobs.png

connected-jobs.svg

DataSet-Job-Report-ScratchPad.md

Notes on SMRT Link DataSet, Job, DataStoreFile, Report models

Current Model for SMRT Link 'Job' model

Simplify, the general interface of a SMRT Link Job, for type T,

A Job takes T as input and produces a PB (T -> Job -> DataStore)

List of EntryPoint PB DataSet -> Job -> DataStore

A DataStore is a list of DataStore files.

Each DataStoreFile can be a different file types, such as, PB DataSet, VCF, ReportJSON, Fasta, etc... and also contains the specific ob id and UUID that generated the DataStoreFile.

During and after SMRT Link Job execution, the DataStoreFiles will be imorted into the db, the DataStoreFile. For a specific subset of file types (PB DataSet types), additional metadata will be stored in the SMRT Link database. Each DataSet has metadata about the specific dataset type as well as metadata about a possible 'parent' DataSet. The DataSet 'parentage' can be a result from copying, merging, analysis (the semantics are not consistent).

Report Details

Each ReportJson file type contains a list of PB Dataset UUIDs in the data model. This is used to communicate which DataSets are specific to the input(s) of a specific ReportJSON. Alternatively said, the EntryPoint PB DataSet(s) might not be directly used to compute the ReportJson* datastore file..

NOTE This is the core issue. Currently the system only communicates the DataSet Job Id

Example Jobs

NOTE, the dotted arrow represents the relation between the Report and the source input for the task at the Report JSON level. This is NOT captured at the SMRT Link Server level.

Import DataSet Job

Accessing the Reports and the source DataSet is clearly defined here by only depending on the Job Id.

I believe the Merge DataSet Job type is Similar.

Example Resequencing Job

The use case here is often to view all the output'ed Reports and links back to the source DataSet is not necessary.

However, viewing the AlignmentSet in DM will start yield unexpected results. This is why the SMRT Link has a workaround to filter all the Reports from the Job and only show the DataSets where for the AlignmentSet in interest. This works for small number of Reports, but DOES NOT work for an Job that outputs "many" reports (because of the explicit filtering necessary on the client side).

Example Demux Analysis Job

The model is more involved when N DataSets and N (or more) companion reports per DataSet are emitted.

The core issue is that a specific DataSet of the N will return

Job Output Access Points

From the file system access point, the DataStore* is accessible in the datastore.json file in the SMRT Link Job directory (the path of this is not consistent, but it's often in the root directory of the Job).

From the webservices, the datastore files are accessible from DM where DS-TYPE' is the DataSet type 'short name' (e.g., subreads) and DS-IDABLE is the DataSet (local) integer id, or (global) UUID.

smrt-link/datasets/<DS-TYPE>/<DS-IDABLE>/reports

NOTE The reports interface is the core issue because it assumes the Job Id link. I believe the SMRT Link UI is filtering to get around this, however, this is not scalable because the SMRT Link UI has to fetch the details of all the reports, then filter out based on the DataSet UUID in the report.

And from the Jobs context.

smrt-link/job-manager/jobs/<JOB-TYPE>/<JOB-IDABLE>/reports

The Job is not a problem and the interface does NOT need to be changed. Semantically, the interface captures exactly what is expected.

Possible Solution

Capture new Report -> DataSet(s) relation

• Add new table to capture DataStoreFile -> Set(DataStoreFile) relation
• On import parse Report and assign
• Update /smrt-link/<DS-TYPE>/<DS-IDABLE>/reports to also filter by 'parent' DataStore File UUID(s)

This is straight forward, but this now requires a join to get the reports for a specific DataSet.

Legacy Data

Need to handle legacy data, specifically for the smrt-link/datasets/<DS-TYPE>/<DS-IDABLE>/reports webservice endpoint.

Possible Solutions

1. During db migration on "start/upgrade", parse the Report JSON files and extract the DataSet UUID from the Report JSON file on disk and update the database.

   • BAD. This is expensive (could be parsing 1000's of report JSON files)
   • BAD. The dataset_uuids field might not be populated consistently. Not clear when that was added in the SMRT Link version.
   • GOOD. Potentially parsing the raw data removes guesswork at the job level (See #3 and the above issue is resolved)

2. Hide the details in the API and dispatch on lookup based on the Job Version. (i.e., if job > 6.0.0, do X else do Y to get reports)

   • BAD. Fundamentally has different semantic results.
   • This is probably very difficult to debug when it's not working as expected. The dispatch from different SL versions would requires an extra join to the engine_jobs table.
   • GOOD. Potentially least amount of db migration machinery

3. During db migration, attempt a thinner approach to migrate old data and assign Report -> DataSet relation based on the Job type

   • import-dataset (Look at the output DataStore file, get the (single) DataSet DataStoreFile, get the List of Reports, then update the DB
   • merge-datasets (Similar to import-dataset)
   • analysis job (Use the Entry Point(s) to get the UUID(s) and assign to all output Reports) (Note, this is not correct. The SL UI would still have to keep the legacy filtering model in place)
   • Other job types (Don't support?)

GOOD. Thin-ish migration BAD. Edge cases on capturing the

demux-pbsmrtpipe-job.dot

strict digraph {
  Job [shape=hexagon, color=blue, label="Demux Analysis Job"]
  store [shape=cylinder, label="DataStore"]
  ep1 [shape=diamond, label="EntryPoint (SubreadSet)"]
  ep2 [shape=diamond, label="EntryPoint (BarcodeSet)"]
  ep1 -> Job
  ep2 -> Job
  Job -> store
  s1 [shape=tab, label="DataStoreFile (SubreadSet_01)"]
  s2 [shape=tab, label="DataStoreFile (SubreadSet_02)"]
  s3 [shape=tab, label="DataStoreFile (SubreadSet_03)"]
  r1 [shape=tab, label="DataStoreFile (Report_01)"]
  r2 [shape=tab, label="DataStoreFile (Report_02)"]
  r3 [shape=tab, label="DataStoreFile (Report_03)"]
  dsf_01 [shape=tab, label="DataStoreFile (Log)"]
  store -> dsf_01
  store -> s1
  store -> s2
  store -> s3
  store -> r1
  store -> r2
  store -> r3
  s1 -> r1 [style=dotted]
  s2 -> r2 [style=dotted]
  s3 -> r3 [style=dotted]
} 

demux-pbsmrtpipe-job.png

demux-pbsmrtpipe-job.svg

ics.dot

strict digraph {

  Run [shape=component, label="PacBio Run (XML)"]

  c1 [shape=box, label="CollectionMetadata 1"]
  c2 [shape=box, label="CollectionMetadata 2"]
  c3 [shape=box, label="CollectionMetadata 3"]
  c4 [shape=box, label="CollectionMetadata 4"]

  Run -> c1
  Run -> c2
  Run -> c3
  Run -> c4

  p [shape=parallelogram, color=blue, label="Primary Analysis: Convert CollectionMeta to SubreadSet)"]

  f1 [shape=tab, label="PA File (SubreadSet XML) on PA file system"]

  copy_job [shape=parallelogram, color=blue, label="Primary Analysis: Copy to Customer FileSystem"]
  customer_subreadset [shape=tab, label="SubreadSet XML on Customer FileSystem"]
  import_job [shape=parallelogram, color=blue, label="Primary Analysis: Import SubreadSet XML into SMRT Link using import-dataset Job"]

  c1 -> p
  p -> f1

  f1 -> copy_job
  copy_job -> customer_subreadset
  customer_subreadset -> import_job

} 

ics.png

import-dataset-job.dot

strict digraph {
  Job [shape=hexagon, color=blue, label="Import DataSet"]
  store [shape=cylinder, label="DataStore"]
  ep1 [shape=diamond, label="Path /path/to/subreadset.xml"]
  ep1 -> Job
  Job -> store
  dsf_02 [shape=tab, label="DataStoreFile (SubreadSet)"]
  dsf_04 [shape=tab, label="DataStoreFile (Report_01)"]
  dsf_05 [shape=tab, label="DataStoreFile (Report_02)"]
  dsf_03 [shape=tab, label="DataStoreFile (Log)"]
  store -> dsf_02
  store -> dsf_04
  store -> dsf_05
  store -> dsf_03
  dsf_02 -> dsf_04 [style=dotted]
  dsf_02 -> dsf_05 [style=dotted]
} 

import-dataset-job.png

import-dataset-job.svg

Makefile

default: convert

convert:
	dot -Tpng demux-pbsmrtpipe-job.dot  -o demux-pbsmrtpipe-job.png
	dot -Tsvg demux-pbsmrtpipe-job.dot  -o demux-pbsmrtpipe-job.svg
	dot -Tpng import-dataset-job.dot -o import-dataset-job.png
	dot -Tsvg import-dataset-job.dot -o import-dataset-job.svg
	dot -Tpng pbsmrtpipe-job.dot -o pbsmrtpipe-job.png
	dot -Tsvg pbsmrtpipe-job.dot -o pbsmrtpipe-job.svg
	dot -Tpng connected-jobs.dot -o connected-jobs.png
	dot -Tsvg connected-jobs.dot -o connected-jobs.svg
	dot -Tpng advanced-jobs.dot -o advanced-jobs.png
	dot -Tsvg advanced-jobs.dot -o advanced-jobs.svg
	dot -Tpng ics.dot -o ics.png
	dot -Tsvg system-job-running.dot -o system-job-running.svg
	dot -Tpng system-job-running.dot -o system-job-running.png


clean:
	rm *.png

pbsmrtpipe-job.dot

strict digraph {
  Job [shape=hexagon, color=blue, label="Analysis Job"]
  store [shape=cylinder, label="DataStore"]
  ep1 [shape=diamond, label="EntryPoint (SubreadSet)"]
  ep2 [shape=diamond, label="EntryPoint (ReferenceSet)"]
  ep1 -> Job
  ep2 -> Job
  Job -> store
  dsf_01 [shape=tab, label="DataStoreFile (Fasta)"]
  dsf_02 [shape=tab, label="DataStoreFile (AlignmentSet)"]
  dsf_03 [shape=tab, label="DataStoreFile (VCF)"]
  dsf_04 [shape=tab, label="DataStoreFile (Report_01)"]
  dsf_05 [shape=tab, label="DataStoreFile (Report_02)"]
  dsf_06 [shape=tab, label="DataStoreFile (LOG)"]
  store -> dsf_01
  store -> dsf_02
  store -> dsf_03
  store -> dsf_04
  store -> dsf_05
  store -> dsf_06
  ep1 -> dsf_04 [style=dotted]
  dsf_02 -> dsf_05 [style=dotted]
  ep2 -> dsf_05 [style=dotted]
} 

pbsmrtpipe-job.png

pbsmrtpipe-job.svg

<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
 "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
<!-- Generated by graphviz version 2.40.1 (20161225.0304)
 -->
<!-- Title: %3 Pages: 1 -->
<svg width="898pt" height="332pt"
 viewBox="0.00 0.00 897.74 332.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 328)">
<title>%3</title>
<polygon fill="#ffffff" stroke="transparent" points="-4,4 -4,-328 893.739,-328 893.739,4 -4,4"/>
<!-- Job -->
<g id="node1" class="node">
<title>Job</title>
<polygon fill="none" stroke="#0000ff" points="423.1944,-234 391.045,-252 326.7463,-252 294.597,-234 326.7463,-216 391.045,-216 423.1944,-234"/>
<text text-anchor="middle" x="358.8957" y="-229.8" font-family="Times,serif" font-size="14.00" fill="#000000">Analysis Job</text>
</g>
<!-- store -->
<g id="node2" class="node">
<title>store</title>
<path fill="none" stroke="#000000" d="M394.8752,-176.7273C394.8752,-178.5331 378.7486,-180 358.8957,-180 339.0428,-180 322.9162,-178.5331 322.9162,-176.7273 322.9162,-176.7273 322.9162,-147.2727 322.9162,-147.2727 322.9162,-145.4669 339.0428,-144 358.8957,-144 378.7486,-144 394.8752,-145.4669 394.8752,-147.2727 394.8752,-147.2727 394.8752,-176.7273 394.8752,-176.7273"/>
<path fill="none" stroke="#000000" d="M394.8752,-176.7273C394.8752,-174.9214 378.7486,-173.4545 358.8957,-173.4545 339.0428,-173.4545 322.9162,-174.9214 322.9162,-176.7273"/>
<text text-anchor="middle" x="358.8957" y="-157.8" font-family="Times,serif" font-size="14.00" fill="#000000">DataStore</text>
</g>
<!-- Job&#45;&gt;store -->
<g id="edge3" class="edge">
<title>Job&#45;&gt;store</title>
<path fill="none" stroke="#000000" d="M358.8957,-215.8314C358.8957,-208.131 358.8957,-198.9743 358.8957,-190.4166"/>
<polygon fill="#000000" stroke="#000000" points="362.3958,-190.4132 358.8957,-180.4133 355.3958,-190.4133 362.3958,-190.4132"/>
</g>
<!-- dsf_01 -->
<g id="node5" class="node">
<title>dsf_01</title>
<polygon fill="none" stroke="#000000" points="504.0305,-108 379.7609,-108 379.7609,-112 367.7609,-112 367.7609,-72 504.0305,-72 504.0305,-108"/>
<polyline fill="none" stroke="#000000" points="367.7609,-108 379.7609,-108 "/>
<text text-anchor="middle" x="435.8957" y="-85.8" font-family="Times,serif" font-size="14.00" fill="#000000">DataStoreFile (Fasta)</text>
</g>
<!-- store&#45;&gt;dsf_01 -->
<g id="edge4" class="edge">
<title>store&#45;&gt;dsf_01</title>
<path fill="none" stroke="#000000" d="M378.326,-143.8314C387.6277,-135.1337 398.9161,-124.5783 409.0243,-115.1265"/>
<polygon fill="#000000" stroke="#000000" points="411.5578,-117.5493 416.4716,-108.1628 406.7768,-112.4363 411.5578,-117.5493"/>
</g>
<!-- dsf_02 -->
<g id="node6" class="node">
<title>dsf_02</title>
<polygon fill="none" stroke="#000000" points="706.2515,-108 533.5399,-108 533.5399,-112 521.5399,-112 521.5399,-72 706.2515,-72 706.2515,-108"/>
<polyline fill="none" stroke="#000000" points="521.5399,-108 533.5399,-108 "/>
<text text-anchor="middle" x="613.8957" y="-85.8" font-family="Times,serif" font-size="14.00" fill="#000000">DataStoreFile (AlignmentSet)</text>
</g>
<!-- store&#45;&gt;dsf_02 -->
<g id="edge5" class="edge">
<title>store&#45;&gt;dsf_02</title>
<path fill="none" stroke="#000000" d="M394.8901,-151.8369C432.5226,-141.2112 492.6338,-124.2386 540.3678,-110.7608"/>
<polygon fill="#000000" stroke="#000000" points="541.3731,-114.1139 550.0458,-108.0282 539.471,-107.3773 541.3731,-114.1139"/>
</g>
<!-- dsf_03 -->
<g id="node7" class="node">
<title>dsf_03</title>
<polygon fill="none" stroke="#000000" points="157.7132,-108 36.0782,-108 36.0782,-112 24.0782,-112 24.0782,-72 157.7132,-72 157.7132,-108"/>
<polyline fill="none" stroke="#000000" points="24.0782,-108 36.0782,-108 "/>
<text text-anchor="middle" x="90.8957" y="-85.8" font-family="Times,serif" font-size="14.00" fill="#000000">DataStoreFile (VCF)</text>
</g>
<!-- store&#45;&gt;dsf_03 -->
<g id="edge6" class="edge">
<title>store&#45;&gt;dsf_03</title>
<path fill="none" stroke="#000000" d="M322.752,-152.2898C283.1555,-141.6519 218.4933,-124.28 167.5022,-110.5809"/>
<polygon fill="#000000" stroke="#000000" points="168.3897,-107.1952 157.824,-107.9807 166.5734,-113.9555 168.3897,-107.1952"/>
</g>
<!-- dsf_04 -->
<g id="node8" class="node">
<title>dsf_04</title>
<polygon fill="none" stroke="#000000" points="889.5825,-108 736.2089,-108 736.2089,-112 724.2089,-112 724.2089,-72 889.5825,-72 889.5825,-108"/>
<polyline fill="none" stroke="#000000" points="724.2089,-108 736.2089,-108 "/>
<text text-anchor="middle" x="806.8957" y="-85.8" font-family="Times,serif" font-size="14.00" fill="#000000">DataStoreFile (Report_01)</text>
</g>
<!-- store&#45;&gt;dsf_04 -->
<g id="edge7" class="edge">
<title>store&#45;&gt;dsf_04</title>
<path fill="none" stroke="#000000" d="M395.1098,-157.0275C457.8565,-148.299 591.0671,-129.2718 714.1949,-108.0834"/>
<polygon fill="#000000" stroke="#000000" points="714.9173,-111.5106 724.1745,-106.3574 713.7243,-104.613 714.9173,-111.5106"/>
</g>
<!-- dsf_05 -->
<g id="node9" class="node">
<title>dsf_05</title>
<polygon fill="none" stroke="#000000" points="268.5825,-36 115.2089,-36 115.2089,-40 103.2089,-40 103.2089,0 268.5825,0 268.5825,-36"/>
<polyline fill="none" stroke="#000000" points="103.2089,-36 115.2089,-36 "/>
<text text-anchor="middle" x="185.8957" y="-13.8" font-family="Times,serif" font-size="14.00" fill="#000000">DataStoreFile (Report_02)</text>
</g>
<!-- store&#45;&gt;dsf_05 -->
<g id="edge8" class="edge">
<title>store&#45;&gt;dsf_05</title>
<path fill="none" stroke="#000000" d="M322.7424,-153.6719C282.3825,-143.6533 220.6494,-125.9329 204.8957,-108 190.1337,-91.1961 185.8374,-65.8391 184.9737,-46.3182"/>
<polygon fill="#000000" stroke="#000000" points="188.4707,-46.1046 184.8023,-36.1651 181.4717,-46.2228 188.4707,-46.1046"/>
</g>
<!-- dsf_06 -->
<g id="node10" class="node">
<title>dsf_06</title>
<polygon fill="none" stroke="#000000" points="349.7516,-108 226.0398,-108 226.0398,-112 214.0398,-112 214.0398,-72 349.7516,-72 349.7516,-108"/>
<polyline fill="none" stroke="#000000" points="214.0398,-108 226.0398,-108 "/>
<text text-anchor="middle" x="281.8957" y="-85.8" font-family="Times,serif" font-size="14.00" fill="#000000">DataStoreFile (LOG)</text>
</g>
<!-- store&#45;&gt;dsf_06 -->
<g id="edge9" class="edge">
<title>store&#45;&gt;dsf_06</title>
<path fill="none" stroke="#000000" d="M339.4653,-143.8314C330.1637,-135.1337 318.8753,-124.5783 308.7671,-115.1265"/>
<polygon fill="#000000" stroke="#000000" points="311.0146,-112.4363 301.3198,-108.1628 306.2336,-117.5493 311.0146,-112.4363"/>
</g>
<!-- ep1 -->
<g id="node3" class="node">
<title>ep1</title>
<polygon fill="none" stroke="#000000" points="493.8957,-324 357.0758,-306 493.8957,-288 630.7156,-306 493.8957,-324"/>
<text text-anchor="middle" x="493.8957" y="-301.8" font-family="Times,serif" font-size="14.00" fill="#000000">EntryPoint (SubreadSet)</text>
</g>
<!-- ep1&#45;&gt;Job -->
<g id="edge1" class="edge">
<title>ep1&#45;&gt;Job</title>
<path fill="none" stroke="#000000" d="M466.5764,-291.4297C447.7002,-281.3624 422.2327,-267.7797 400.8042,-256.3512"/>
<polygon fill="#000000" stroke="#000000" points="402.3694,-253.2194 391.8988,-251.6017 399.0753,-259.3958 402.3694,-253.2194"/>
</g>
<!-- ep1&#45;&gt;dsf_04 -->
<g id="edge10" class="edge">
<title>ep1&#45;&gt;dsf_04</title>
<path fill="none" stroke="#000000" stroke-dasharray="1,5" d="M515.9052,-290.8113C569.5736,-253.775 707.507,-158.5877 772.3,-113.8743"/>
<polygon fill="#000000" stroke="#000000" points="774.3281,-116.7273 780.5707,-108.1668 770.3523,-110.966 774.3281,-116.7273"/>
</g>
<!-- ep2 -->
<g id="node4" class="node">
<title>ep2</title>
<polygon fill="none" stroke="#000000" points="153.8957,-324 7.1809,-306 153.8957,-288 300.6105,-306 153.8957,-324"/>
<text text-anchor="middle" x="153.8957" y="-301.8" font-family="Times,serif" font-size="14.00" fill="#000000">EntryPoint (ReferenceSet)</text>
</g>
<!-- ep2&#45;&gt;Job -->
<g id="edge2" class="edge">
<title>ep2&#45;&gt;Job</title>
<path fill="none" stroke="#000000" d="M191.9742,-292.6261C225.1019,-280.991 273.3276,-264.0532 309.4339,-251.3719"/>
<polygon fill="#000000" stroke="#000000" points="310.9986,-254.532 319.2738,-247.916 308.679,-247.9276 310.9986,-254.532"/>
</g>
<!-- ep2&#45;&gt;dsf_05 -->
<g id="edge12" class="edge">
<title>ep2&#45;&gt;dsf_05</title>
<path fill="none" stroke="#000000" stroke-dasharray="1,5" d="M131.9672,-290.5816C80.0134,-252.1778 -42.0856,-149.749 14.8957,-72 25.115,-58.0561 58.6501,-46.0477 93.3479,-36.9197"/>
<polygon fill="#000000" stroke="#000000" points="94.321,-40.2836 103.1469,-34.4223 92.5922,-33.5004 94.321,-40.2836"/>
</g>
<!-- dsf_02&#45;&gt;dsf_05 -->
<g id="edge11" class="edge">
<title>dsf_02&#45;&gt;dsf_05</title>
<path fill="none" stroke="#000000" stroke-dasharray="1,5" d="M521.7057,-73.5203C518.7348,-73.0051 515.7921,-72.4972 512.8957,-72 433.872,-58.434 343.9911,-43.637 278.4864,-32.9686"/>
<polygon fill="#000000" stroke="#000000" points="279.0283,-29.5108 268.5959,-31.359 277.9038,-36.4199 279.0283,-29.5108"/>
</g>
</g>
</svg>

system-job-running.dot

digraph {
        postgres_db [shape=cylinder];
        WorkerA [shape=diamond, color=blue];
        WorkerB [shape=diamond, color=blue];
        WorkerC [shape=diamond, color=blue];
        WorkerD [shape=pentagon, color=green];
        WorkerE [shape=pentagon, color=orange];

        pbsmrtpipeJobA [shape=rectangle];
        pbsmrtpipeJobB [shape=rectangle]
        pbsmrtpipeJobC [shape=rectangle]

        postgres_db -> SL_Services
        SL_Services -> WorkerA
        SL_Services -> WorkerB
        SL_Services -> WorkerC
        SL_Services -> WorkerD
        SL_Services -> WorkerE

        WorkerA -> pbsmrtpipeJobA
        WorkerB -> pbsmrtpipeJobB
        WorkerC -> pbsmrtpipeJobC
        WorkerD -> backup_db
        WorkerE -> import_dataset


        pbsmrtpipeJobA -> taskA_01
        pbsmrtpipeJobA -> taskA_02
        pbsmrtpipeJobA -> taskA_03
        pbsmrtpipeJobA -> taskA_04
        pbsmrtpipeJobA -> taskA_05

        taskA_01 -> sge_sync_job_A_01
        taskA_02 -> sge_sync_job_A_02
        taskA_03 -> sge_sync_job_A_03
        taskA_04 -> sge_sync_job_A_04
        taskA_05 -> sge_sync_job_A_05


        pbsmrtpipeJobB -> taskB_01
        pbsmrtpipeJobB -> taskB_02

        taskB_01 -> sge_sync_job_B_01
        taskB_02 -> sge_sync_job_B_02

        pbsmrtpipeJobC -> taskC_01
        pbsmrtpipeJobC -> taskC_02

        taskC_01 -> sge_sync_job_C_01
        taskC_02 -> sge_sync_job_C_02


}

system-job-running.png

system-job-running.svg