jhpoelen · July 17, 2018 01:53 · jhpoelen · Jun 13, 2018
diff --git a/README b/README
 interactions.tsv.gz
    Contains pairwise interactions generated by elton 0.5.0 on 2018-06-29 . 
    Generated by dietmatrix.sh . 

 fbPredPreyOrder.tsv.gz
    Contains prey/diet items of species known to fishbase. For prey/diet items, the linked order(s) are included as well as the resolved prey/diet item terms. Calculated by dietmatrix.sh .

 majorityOrders.tsv
    Majority order were calculate by selecting the most frequently occurring order associated with a specific prey id/name.
    If different order assignments for a particular prey item have same frequency, the orders are sorted in alphabetical order and the first is selected.
    So, lets say that species X maps to only orders A and B. There's no majority order, and A is selected because it is the first name in alphabetical order.

 minorityOrders.tsv
    Minority order are the orders that occur that are not majority orders.


 fbPredPreyMajorityOrder.tsv.gz
    Contains an estimate of diet order associated with prey based on GloBI. 
    Generated using calcMajorityOrders.scala using input generated from dietmatrix.sh  

 fbPredPreyMajorityOrderCount.tsv
    Contains an estimate of diet order diversity based on GloBI.
    Calculated by taking fbPredPreyOrder.tsv.gz (generated by dietMatrix.sh), and determining majority orders.
    Then, only select majority orders for selected prey id/name, and generated a distinct list of all orders consumed by a specific predator.
    Apache Spark scala code to do this transformation can be found in calcMajorityOrders.scala .

    has columns:
    1. predator fishbase spec id
    2. predator species name
    3. number of distinct majority prey orders recorded for this predator
    4. total number of majority orders

 fbPredPreyOrderPreyFrequency.tsv
    contains a sorted frequency count of unique prey item-order pairs that occur in used intermediate fbPredPreyOrder.tsv.gz file. Used to detect suspicious prey items.
diff --git a/calcMajorityOrders.scala b/calcMajorityOrders.scala
 import spark.implicits._
 import org.apache.spark.sql.SaveMode
 val predPrey = spark.read.option("delimiter","\t").option("header", "false").csv("fbPredPreyOrder.tsv.gz")

 case class Term(id: Option[String], name: Option[String])
 case class PredPrey(pred: Term, prey: Term, order: Term)

 // get prey id,name pairs with order 
 val predPreyDs = predPrey.as[(Option[String], Option[String], Option[String], Option[String], Option[String], Option[String])].map(r => PredPrey(pred = Term(id=r._1, name=r._2), prey=Term(id=r._3,name=r._4), order=Term(id=r._5, name=r._6)))

 val predPreyWithPreyOrdersDs = predPreyDs.filter(_.order.id.isDefined).distinct

 val preyWithOrder = predPreyWithPreyOrdersDs.map(x => ((x.prey.id, x.prey.name, x.order.name), 1))

 // sometimes species match to different (hopefully similar) orders: in this case, the order with the same name is selected. If there's a split vote, the order that is first alphabetically is selected. The latter is a bit arbritary, but necessary to ensure reproducibility.

 val distinctMajorityOrders = preyWithOrder.rdd.reduceByKey( _ + _) .map( x => ((x._1._1, x._1._2),( x._2, x._1._3))).reduceByKey( (agg, v) => { if (agg._1 > v._1) agg else {if (v._1 == agg._1 && v._2.getOrElse("") > agg._2.getOrElse("")) agg else v } }).map(_._2._2).distinct

 val distinctOrders = predPreyWithPreyOrdersDs.map(_.order.name).distinct
 val distinctMinorityOrders = distinctOrders.rdd.subtract(distinctMajorityOrders)

 distinctMajorityOrders.toDS.write.option("delimiter", "\t").option("header", "false").mode(SaveMode.Overwrite).csv("majorityOrders")
 distinctMinorityOrders.toDS.write.option("delimiter", "\t").option("header", "false").mode(SaveMode.Overwrite).csv("minorityOrders")

 val majorOrders = sc.broadcast(distinctMajorityOrders.collect)

 val predPreyMajorityOrders = predPreyWithPreyOrdersDs.filter(x => majorOrders.value.contains(x.order.name))

 val predPreyMajorityOrdersOnly = predPreyMajorityOrders.map(x => (x.pred.id, x.pred.name, x.order.name)).distinct
 val predPreyMajorityOrderCount = predPreyMajorityOrdersOnly.map(x => ((x._1, x._2), 1)).rdd.reduceByKey(_ + _).map(x => (x._1._1, x._1._2, x._2, majorOrders.value.flatten.length))
 predPreyMajorityOrdersOnly.write.option("delimiter", "\t").option("header", "false").mode(SaveMode.Overwrite).csv("fbPredPreyMajorityOrder")
 predPreyMajorityOrderCount.toDS.write.option("delimiter", "\t").option("header", "false").mode(SaveMode.Overwrite).csv("fbPredPreyMajorityOrderCount")

diff --git a/dietmatix.sh b/dietmatix.sh
 #!/bin/bash
 #
 # script assumes that https://github.com/globalbioticinteractions/elton , https://github.com/globalbioticinteractions/nomer
 # and apache spark are available via elton, nomer, and spark-shell aliases respectively.
 #

 elton interactions | cut -f1,2,8,9,10 | sort | uniq | gzip > interactions.tsv.gz
 zcat interactions.tsv.gz | cut -f3 | sort | uniq > interactionLabel.tsv
 zcat interactions.tsv.gz | grep -P "(\teatenBy\t|\tpreyedUponBy\t)" | awk -F '\t' '{ print $4 "\t" $5 "\t" $1 "\t" $2 }' | gzip > interactionsPredPrey.tsv.gz 
 zcat interactions.tsv.gz | grep -P "(\teats\t|\tpreysOn\t)" | cut -f1,2,4,5 | gzip >> interactionsPredPrey.tsv.gz
 zcat interactionsPredPrey.tsv.gz | nomer append | grep SAME_AS | cut -f3,4,6,7 | gzip > interactionsPreyPred.tsv.gz

 # map to prey orders
 nomer properties | grep -v "nomer.append.schema.output" > my.properties
 echo 'nomer.append.schema.output=[{"column":0, "type":"id"}, {"column":1, "type":"name"}, {"column": 2, "type": "rank"}, {"column":3,"type":"path.order.id"},{"column":4,"type":"path.order.name"},{"column":5,"type":"path.order"}]' >> my.properties
 zcat interactionsPreyPred.tsv.gz | grep -P ".*\t.*\tFBC:FB" | nomer append --properties=my.properties | grep SAME_AS | gzip > fbPreyPredSameAsWithOrder.tsv.gz

 # remove likely homonyms
 zcat fbPreyPredSameAsWithOrder.tsv.gz | awk -F '\t' '{ print $1 "\t" $2 "\t" $6 "\t" $7 }' | sort | uniq | gzip > fbPreyMap.tsv.gz

 cat removeLikelyHomonyms.scala | spark-shell
 cat fbPreyLikelyHomonyms/*.csv | sort | uniq > fbPreyLikelyHomonyms.tsv
 cat fbPredPreySameAsWithOrderNoHomonyms/*.csv | grep -v -P "\t\t$" | sort | uniq | gzip > fbPredPreySameAsWithOrderNoHomonyms.tsv.gz

 zcat fbPredPreySameAsWithOrderNoHomonyms.tsv.gz | awk -F '\t' '{ print $1 "\t" $2 "\t" $3 "\t" $4 "\t" $5 "\t" $6 }' | sort | uniq | gzip > fbPredPreyOrderUnmapped.tsv.gz

 zcat fbPredPreyOrderUnmapped.tsv.gz | sed -f mapOrders.sed | sort | uniq | gzip > fbPredPreyOrder.tsv.gz
 zcat fbPredPreyOrder.tsv.gz | cut -f4,6 | sort | uniq -c | sort -n -r > fbPredPreyOrderPreyFrequency.tsv

 # calc majority orders

 cat calcMajorityOrders.scala | spark-shell
 cat majorityOrders/*.csv > majorityOrders.tsv
 cat minorityOrders/*.csv > minorityOrders.tsv
 cat fbPredPreyMajorityOrder/*.csv | sort | uniq > fbPredPreyMajorityOrder.tsv
 cat fbPredPreyMajorityOrderCount/*.csv | sort | uniq > fbPredPreyMajorityOrder.tsv

diff --git a/fbPredPreyMajorityOrder.tsv.gz b/fbPredPreyMajorityOrder.tsv.gz
diff --git a/fbPredPreyMajorityOrderCount.tsv b/fbPredPreyMajorityOrderCount.tsv
diff --git a/fbPredPreyOrder.tsv.gz b/fbPredPreyOrder.tsv.gz
diff --git a/fbPredPreyOrderPreyFrequency.tsv b/fbPredPreyOrderPreyFrequency.tsv
diff --git a/fbPredPreyOrderUnmapped.tsv.gz b/fbPredPreyOrderUnmapped.tsv.gz
diff --git a/fbPreyLikelyHomonyms.tsv b/fbPreyLikelyHomonyms.tsv
diff --git a/fbPreyMap.tsv.gz b/fbPreyMap.tsv.gz
diff --git a/interactions.tsv.gz b/interactions.tsv.gz
diff --git a/majorityOrders.tsv b/majorityOrders.tsv
diff --git a/mapOrders.sed b/mapOrders.sed
diff --git a/minorityOrders.tsv b/minorityOrders.tsv
diff --git a/removeLikelyHomonyms.scala b/removeLikelyHomonyms.scala
	interactions.tsv.gz
	Contains pairwise interactions generated by elton 0.5.0 on 2018-06-29 .
	Generated by dietmatrix.sh .

	fbPredPreyOrder.tsv.gz
	Contains prey/diet items of species known to fishbase. For prey/diet items, the linked order(s) are included as well as the resolved prey/diet item terms. Calculated by dietmatrix.sh .

	majorityOrders.tsv
	Majority order were calculate by selecting the most frequently occurring order associated with a specific prey id/name.
	If different order assignments for a particular prey item have same frequency, the orders are sorted in alphabetical order and the first is selected.
	So, lets say that species X maps to only orders A and B. There's no majority order, and A is selected because it is the first name in alphabetical order.

	minorityOrders.tsv
	Minority order are the orders that occur that are not majority orders.


	fbPredPreyMajorityOrder.tsv.gz
	Contains an estimate of diet order associated with prey based on GloBI.
	Generated using calcMajorityOrders.scala using input generated from dietmatrix.sh

	fbPredPreyMajorityOrderCount.tsv
	Contains an estimate of diet order diversity based on GloBI.
	Calculated by taking fbPredPreyOrder.tsv.gz (generated by dietMatrix.sh), and determining majority orders.
	Then, only select majority orders for selected prey id/name, and generated a distinct list of all orders consumed by a specific predator.
	Apache Spark scala code to do this transformation can be found in calcMajorityOrders.scala .

	has columns:
	1. predator fishbase spec id
	2. predator species name
	3. number of distinct majority prey orders recorded for this predator
	4. total number of majority orders

	fbPredPreyOrderPreyFrequency.tsv
	contains a sorted frequency count of unique prey item-order pairs that occur in used intermediate fbPredPreyOrder.tsv.gz file. Used to detect suspicious prey items.
	import spark.implicits._
	import org.apache.spark.sql.SaveMode
	val predPrey = spark.read.option("delimiter","\t").option("header", "false").csv("fbPredPreyOrder.tsv.gz")

	case class Term(id: Option[String], name: Option[String])
	case class PredPrey(pred: Term, prey: Term, order: Term)

	// get prey id,name pairs with order
	val predPreyDs = predPrey.as[(Option[String], Option[String], Option[String], Option[String], Option[String], Option[String])].map(r => PredPrey(pred = Term(id=r._1, name=r._2), prey=Term(id=r._3,name=r._4), order=Term(id=r._5, name=r._6)))

	val predPreyWithPreyOrdersDs = predPreyDs.filter(_.order.id.isDefined).distinct

	val preyWithOrder = predPreyWithPreyOrdersDs.map(x => ((x.prey.id, x.prey.name, x.order.name), 1))

	// sometimes species match to different (hopefully similar) orders: in this case, the order with the same name is selected. If there's a split vote, the order that is first alphabetically is selected. The latter is a bit arbritary, but necessary to ensure reproducibility.

	val distinctMajorityOrders = preyWithOrder.rdd.reduceByKey( _ + _) .map( x => ((x._1._1, x._1._2),( x._2, x._1._3))).reduceByKey( (agg, v) => { if (agg._1 > v._1) agg else {if (v._1 == agg._1 && v._2.getOrElse("") > agg._2.getOrElse("")) agg else v } }).map(_._2._2).distinct

	val distinctOrders = predPreyWithPreyOrdersDs.map(_.order.name).distinct
	val distinctMinorityOrders = distinctOrders.rdd.subtract(distinctMajorityOrders)

	distinctMajorityOrders.toDS.write.option("delimiter", "\t").option("header", "false").mode(SaveMode.Overwrite).csv("majorityOrders")
	distinctMinorityOrders.toDS.write.option("delimiter", "\t").option("header", "false").mode(SaveMode.Overwrite).csv("minorityOrders")

	val majorOrders = sc.broadcast(distinctMajorityOrders.collect)

	val predPreyMajorityOrders = predPreyWithPreyOrdersDs.filter(x => majorOrders.value.contains(x.order.name))

	val predPreyMajorityOrdersOnly = predPreyMajorityOrders.map(x => (x.pred.id, x.pred.name, x.order.name)).distinct
	val predPreyMajorityOrderCount = predPreyMajorityOrdersOnly.map(x => ((x._1, x._2), 1)).rdd.reduceByKey(_ + _).map(x => (x._1._1, x._1._2, x._2, majorOrders.value.flatten.length))
	predPreyMajorityOrdersOnly.write.option("delimiter", "\t").option("header", "false").mode(SaveMode.Overwrite).csv("fbPredPreyMajorityOrder")
	predPreyMajorityOrderCount.toDS.write.option("delimiter", "\t").option("header", "false").mode(SaveMode.Overwrite).csv("fbPredPreyMajorityOrderCount")
	#!/bin/bash
	#
	# script assumes that https://github.com/globalbioticinteractions/elton , https://github.com/globalbioticinteractions/nomer
	# and apache spark are available via elton, nomer, and spark-shell aliases respectively.
	#

	elton interactions \| cut -f1,2,8,9,10 \| sort \| uniq \| gzip > interactions.tsv.gz
	zcat interactions.tsv.gz \| cut -f3 \| sort \| uniq > interactionLabel.tsv
	zcat interactions.tsv.gz \| grep -P "(\teatenBy\t\|\tpreyedUponBy\t)" \| awk -F '\t' '{ print $4 "\t" $5 "\t" $1 "\t" $2 }' \| gzip > interactionsPredPrey.tsv.gz
	zcat interactions.tsv.gz \| grep -P "(\teats\t\|\tpreysOn\t)" \| cut -f1,2,4,5 \| gzip >> interactionsPredPrey.tsv.gz
	zcat interactionsPredPrey.tsv.gz \| nomer append \| grep SAME_AS \| cut -f3,4,6,7 \| gzip > interactionsPreyPred.tsv.gz

	# map to prey orders
	nomer properties \| grep -v "nomer.append.schema.output" > my.properties
	echo 'nomer.append.schema.output=[{"column":0, "type":"id"}, {"column":1, "type":"name"}, {"column": 2, "type": "rank"}, {"column":3,"type":"path.order.id"},{"column":4,"type":"path.order.name"},{"column":5,"type":"path.order"}]' >> my.properties
	zcat interactionsPreyPred.tsv.gz \| grep -P ".\t.\tFBC:FB" \| nomer append --properties=my.properties \| grep SAME_AS \| gzip > fbPreyPredSameAsWithOrder.tsv.gz

	# remove likely homonyms
	zcat fbPreyPredSameAsWithOrder.tsv.gz \| awk -F '\t' '{ print $1 "\t" $2 "\t" $6 "\t" $7 }' \| sort \| uniq \| gzip > fbPreyMap.tsv.gz

	cat removeLikelyHomonyms.scala \| spark-shell
	cat fbPreyLikelyHomonyms/*.csv \| sort \| uniq > fbPreyLikelyHomonyms.tsv
	cat fbPredPreySameAsWithOrderNoHomonyms/*.csv \| grep -v -P "\t\t$" \| sort \| uniq \| gzip > fbPredPreySameAsWithOrderNoHomonyms.tsv.gz

	zcat fbPredPreySameAsWithOrderNoHomonyms.tsv.gz \| awk -F '\t' '{ print $1 "\t" $2 "\t" $3 "\t" $4 "\t" $5 "\t" $6 }' \| sort \| uniq \| gzip > fbPredPreyOrderUnmapped.tsv.gz

	zcat fbPredPreyOrderUnmapped.tsv.gz \| sed -f mapOrders.sed \| sort \| uniq \| gzip > fbPredPreyOrder.tsv.gz
	zcat fbPredPreyOrder.tsv.gz \| cut -f4,6 \| sort \| uniq -c \| sort -n -r > fbPredPreyOrderPreyFrequency.tsv

	# calc majority orders

	cat calcMajorityOrders.scala \| spark-shell
	cat majorityOrders/*.csv > majorityOrders.tsv
	cat minorityOrders/*.csv > minorityOrders.tsv
	cat fbPredPreyMajorityOrder/*.csv \| sort \| uniq > fbPredPreyMajorityOrder.tsv
	cat fbPredPreyMajorityOrderCount/*.csv \| sort \| uniq > fbPredPreyMajorityOrder.tsv