Duncan Paterson duncdrum

Web Scraping with XQuery

Overview

Learning how to web scrape empowers you to apply your XQuery skills to any data residing on the web. You can fetch data from remote sites and services—for example, entire web pages or just the pieces of a page that matter to you. Once fetched, you can perform further analysis on the data, clean it up, mash it up with other data, transform it into different formats, etc.

Built-in functions for making HTTP requests

XPath-based languages like XQuery offer an standard function for accessing remote documents, the fn:doc() function. However, a limitation of this function is that it only works if the URI returns a well-formed XML document.

	xquery version "3.1";

	import module namespace hc="http://expath.org/ns/http-client";
	import module namespace util="http://exist-db.org/xquery/util";
	import module namespace xmldb="http://exist-db.org/xquery/xmldb";

	(: downloads a file from a remote HTTP server at $file-url and save it to an eXist-db $collection.
	: we try hard to recognize XML files and save them with the correct mimetype so that eXist-db can
	: efficiently index and query the files; if it doesn't appear to be XML, though, we just trust
	: the response headers :)

	// Restify Server CheatSheet.
	// More about the API: http://mcavage.me/node-restify/#server-api
	// Install restify with npm install restify


	// 1.1. Creating a Server.
	// http://mcavage.me/node-restify/#Creating-a-Server


	var restify = require('restify');

	xquery version "3.0";

	(:module namespace gitsync = "http://syriaca.org/ns/gitsync";:)

	(:~
	: XQuery endpoint to respond to Github webhook requests. Query responds only to push requests.
	: The EXPath Crypto library supplies the HMAC-SHA1 algorithm for matching Github secret.
	:
	: Secret can be stored as environmental variable.
	: Will need to be run with administrative privileges, suggest creating a git user with privileges only to relevant app.

	xquery version "3.1";

	(:~
	: An implementation of XQuery 3.1's fn:json-to-xml and fn:xml-to-json functions for eXist, which does not support them natively as of 4.3.0.
	:
	: @author Joe Wicentowski
	: @version 0.4
	: @see http://www.w3.org/TR/xpath-functions-31/#json
	:)
	module namespace jx = "http://joewiz.org/ns/xquery/json-xml";

	xquery version "3.0";

	(:~
	: POC: eXist DB module to commit files to github reop via github API
	: Prerequisites:
	: In order to run the module, you will need a github Authorization token.
	: When you create authorization token your OAuth Scope will need to include repo, allowing you to update files.
	: @see https://github.com/blog/1509-personal-api-tokens
	:
	: @Notes

	# How to uninstall Razer Synapse 2 ( https://www.razerzone.com/synapse-2 )
	# on OS X (10.11-10.13) (El Capitan, Sierra, High Sierra)
	# without using Razer's official uninstall tool.

	# Tested on OS X 10.11.5 in July 2016.
	# Edited with additional steps for later OS X versions,
	# contributed by commenters on this gist.

	# Step 1: In your terminal: stop and remove launch agents
	launchctl remove com.razer.rzupdater

	xquery version "3.1";

	import module namespace http = "http://expath.org/ns/http-client";
	(: For generating the personal access token to be used in this script, please select "public_repo \| Access public repositories". :)
	let $git-token := "<your-github-token>"
	let $user-name := "<your-github-username>"
	let $repo-name := "<your-repo-name>"
	let $file-path := "<file=path>"
	let $message := "<commit-message>"
	let $new-file-content := serialize(<a/>)

	xquery version "1.0";
	(:~
	The module contains variables for each of the unicode escape sequences at
	<a href="https://en.wikipedia.org/wiki/List_of_XML_and_HTML_character_entity_references">https://en.wikipedia.org/wiki/List_of_XML_and_HTML_character_entity_references</a>

	:)
	module namespace encchars="http://w3.org/encoded/characters";

	(: Name Escape Sequence Description :)
	declare variable $encchars:quot := """; (: quotation mark (APL quote) :)

	xquery version "3.1";

	(:~
	: An implementation of XQuery 3.1's fn:unparsed-text* functions for eXist, which does not support them natively as of 3.4.0.
	: Known limitations:
	: - Does not enforce encoding aspects of the XQuery spec beyond functionality natively supplied by the EXPath HTTP Client and eXist's internals.
	:
	: @author Joe Wicentowski
	: @version 0.2
	: @see https://www.w3.org/TR/xpath-functions-31/#func-unparsed-text