Deyan Ginev dginev

	//! Convert arXiv's OAI harvested XML files into a lookup table for classification labels
	// Step 0. Prerequisite: download all needed arXiv metadata via OAI, e.g.
	//```
	// $ pip install git+http://github.com/bloomonkey/oai-harvest.git#egg=oaiharvest
	// $ mkdir metadata/arxiv; cd metadata/arxiv
	// $ oai-reg add arxiv http://export.arxiv.org/oai2?verb=Identify
	// $ oai-harvest arxiv --until 2018-09-09
	//```
	// endpoint documentation at: https://arxiv.org/help/oa
	use jwalk::WalkDir;

	#!/usr/bin/env perl
	# Applies cutoffs to the very noisy 250 MB mathml_statistics.txt
	# which was generated by llamapun over arXMLiv 08.2018.
	#
	# It rewrites to a CSV file, throwing out all known erroneous markup, including:
	# - discard all SVG-associated markup (wrongly in MathML)
	# - discard all (non-math) HTML-associated markup (wrongly in MathML)
	# - discard all XMath-associated markup (wrongly in MathML)
	# - less noisy for uninteresting values (numbers with known units, hex colors, open-ended id schemes, etc)
	#

	time: 0.026; rss: 58MB parsing
	time: 0.000; rss: 58MB attributes injection
	time: 0.000; rss: 58MB garbage collect incremental cache directory
	time: 0.000; rss: 58MB recursion limit
	time: 0.000; rss: 58MB crate injection
	time: 0.000; rss: 58MB plugin loading
	time: 0.000; rss: 58MB plugin registration
	time: 0.000; rss: 58MB background load prev dep-graph
	time: 0.003; rss: 58MB pre ast expansion lint checks
	time: 1.662; rss: 237MB expand crate