extract_pubmed_abstracts.groovy

import java.util.concurrent.*
import java.util.concurrent.atomic.*
import groovyx.gpars.*
import groovy.json.*
import groovyx.gpars.GParsPool
def o = 0

def files = []
new File('/data2/pubmed_xml_baseline/ftp.ncbi.nlm.nih.gov/pubmed/baseline/').eachFile { f -> files << f }

GParsPool.withPool(10) {
files.eachParallel { f ->
  if(!(f.getName() =~ 'xml$')) { return; }
  o++
  println f.getName() + " $o/${files.size()}"
  def v = f.getName().tokenize('.')[0]
  def i = 0

  def outDir = new File('extract/'+v)
  outDir.mkdir()

  // we need to spin up a new parser each time, or it gets mad
  def parser = new XmlSlurper()
  parser.setFeature("http://apache.org/xml/features/disallow-doctype-decl", false)
  parser.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false)

  def document = parser.parse(f)
  document.depthFirst().findAll { it.name() == 'Abstract' }.each { abs ->
    def aText = abs['AbstractText'].collect { it.text() }.join('\n')
    new File(outDir, "${v}_abstract_${i}.txt").text = aText
    i++
    if(i % 5000 == 0) { println i }
  }
}
}