This script fetches all the sequences links from a MycorWeb Blast result.

getMycorLinks.coffee

# getMycorLinks.coffee
# This script fetches all the sequences links from a MycorWeb Blast result.
#
# sequences can be fetched from these links with getMycorSequences.coffee:
# https://gist.github.com/4683075
# results can be splitted into multiple files to run multiple instances of
# getMycorSequences.coffee
#
# Dependencies: casperjs
# Usage: casperjs getMycorLinks.coffee numberOutputFiles URL
# Example: casperjs getMycorLinks.coffee 2 'http://mycor.nancy.inra.fr/IMGC/TuberGenome/database.php?table=gene_seq&start=0&search=&order=Name'
# Author: Bruno Vieira ([email protected])
# License: MIT

fs = require 'fs'
casper = require('casper').create()

numOutputFiles = casper.cli.args[0]
parsedUrl = casper.cli.args[1].split('/database')
rootUrl = parsedUrl[0]
firstPage = "/database#{parsedUrl[1]}"

getLinks = ->
  links = document.querySelectorAll "td>a"
  Array::map.call links, (e) -> e.getAttribute "href"

sequencesLinks = []
pages = []

casper.start "#{rootUrl}/#{firstPage}", ->
  firstPageLinks = @evaluate getLinks
  for link in firstPageLinks
    pages.push link if link.match 'database'
  pageSequencesLinks = firstPageLinks.slice(pages.length)
  sequencesLinks = sequencesLinks.concat pageSequencesLinks
  for page in pages
    @thenOpen "#{rootUrl}/#{page}", =>
      pageSequencesLinks = @evaluate(getLinks).slice(pages.length)
      sequencesLinks = sequencesLinks.concat pageSequencesLinks

casper.then ->
  numSequencesPerFile = Math.floor(sequencesLinks.length / numOutputFiles)
  for start in [0..sequencesLinks.length] by numSequencesPerFile
    end = start + numSequencesPerFile
    if end > sequencesLinks.length
      end = sequencesLinks.length
    filename = "sequencesLinks:#{start}-#{end}.txt"
    for sequenceLink in sequencesLinks.slice start, end
      fs.write filename, "#{rootUrl}/#{sequenceLink}\n", 'a'

casper.run()