SBS for Python-Iview

sbs.diff

SBS for Python-Iview

* Overrides all the ABC Iview function, you only get SBS instead
* I hacked the internal interface for the list items in order to get the
  front end to display multiple tree branches for the one SBS XML file, but
  it should be done in a nicer way.
* Only works with the GTK version; probably breaks the CLI version
* I don’t think I ever finished the “Copy URL” button idea. But it would be
  nice to be able to quickly pipe a URL into a video player and stream it
  without worrying about downloading it.

=== modified file 'iview-gtk'
--- iview-gtk	2010-07-31 11:19:47 +0000
+++ iview-gtk	2011-11-11 03:26:40 +0000
@@ -221,44 +221,60 @@
 
 	description.set_text('')
 	download_btn.set_sensitive(False)
+	copy_btn.set_sensitive(False)
 
 	model, selected_iter = listing.get_selection().get_selected()
 	if selected_iter is None:
 		return
 	item = model.get(selected_iter, 0, 2, 3)
-	if item[1] is None or item[2] is None:
+	
+	if item[2] is not None:
+		description.set_text(item[2])
+	
+	if item[1] is None:
 		return
 
-	description.set_text(item[2])
 	download_btn.set_sensitive(True)
-
-def load_programme():
-	global programme
-
-	for series in iview.comm.get_index():
-		series_iter = programme.append(None, [series['title'], series['id'], None, None])
-		programme.append(series_iter, ['Loading...', None, None, None])
+	copy_btn.set_sensitive(True)
+
+def add_items(model, iter, items):
+	hierarchy = [iter]
+	for series in items:
+		if series is None:
+			hierarchy.pop()
+			continue
+		
+		try:
+			# key 'livestream' indicates a different story
+			url = series['url']
+		except LookupError:
+			url = None
+			id = series['id']
+		else:
+			id = None
+		series_iter = model.append(hierarchy[-1], [
+			series['title'],
+			id,
+			url,
+			series.get('description'),
+		])
+		if url is id is None:
+			hierarchy.append(series_iter)
+		if id is not None:
+			model.append(series_iter,
+				['Loading...', None, None, None])
 
 def load_series_items(widget, iter, path):
 	model = widget.get_model()
+
+	series_id = model.get(iter, 1)[0]
+	if series_id is None:
+		# We've already fetched this. Better pull out.
+		return
+
 	child = model.iter_children(iter)
-
-	if model.get(child, 2)[0] is not None:
-		# This is not a "Loading..." item, so we've already fetched this.
-		# Better pull out.
-		return
-
-	series_id = model.get(iter, 1)[0]
-	items = iview.comm.get_series_items(series_id)
-
-	for item in items:
-		model.append(iter, [
-				item['title'],
-				None,
-				item['url'],
-				item['description'],
-			])
-
+	add_items(model, iter, iview.comm.get_series_items(series_id))
+	model.set_value(iter, 1, None)
 	model.remove(child)
 
 def about(widget, data=None):
@@ -335,8 +351,12 @@
 download_btn = gtk.Button('Download')
 download_btn.set_sensitive(False)
 download_btn.connect('clicked', on_download_clicked)
+copy_btn = gtk.Button('Copy URL')
+copy_btn.set_sensitive(False)
+#~ copy_btn.connect('clicked', copy_url)
 
 bb.pack_start(about_btn)
+bb.pack_start(copy_btn)
 bb.pack_start(download_btn)
 
 vbox.pack_start(bb, expand=False)
@@ -351,7 +371,7 @@
 
 try:
 	iview.comm.get_config()
-	load_programme()
+	add_items(programme, None, iview.comm.get_index())
 except urllib2.HTTPError, error:
 	message = gtk.MessageDialog(
 		parent=window,

=== modified file 'iview/comm.py'
--- iview/comm.py	2011-03-13 03:30:22 +0000
+++ iview/comm.py	2011-11-11 04:11:38 +0000
@@ -65,14 +65,20 @@
 	global iview_config
 
 	iview_config = parser.parse_config(maybe_fetch(config.config_url))
+	
+	if True:
+		proto = config.config_url.partition('://')
+		host = proto[-1].partition('/')
+		iview_config['base'] = '{0}://{1}'.format(proto[0], host[0])
 
-def get_auth():
-	""" This function performs an authentication handshake with iView.
-		Among other things, it tells us if the connection is unmetered,
-		and gives us a one-time token we need to use to speak RTSP with
-		ABC's servers, and tells us what the RTMP URL is.
-	"""
-	return parser.parse_auth(fetch_url(iview_config['auth_url']))
+if False:
+	def get_auth():
+		""" This function performs an authentication handshake with iView.
+			Among other things, it tells us if the connection is unmetered,
+			and gives us a one-time token we need to use to speak RTSP with
+			ABC's servers, and tells us what the RTMP URL is.
+		"""
+		return parser.parse_auth(fetch_url(iview_config['auth_url']))
 
 def get_index():
 	"""	This function pulls in the index, which contains the TV series
@@ -80,7 +86,11 @@
 		must decrypt it here before passing it to the parser.
 	"""
 
-	index_data = maybe_fetch(iview_config['api_url'] + 'seriesIndex')
+	if True:
+		url = iview_config['base'] + iview_config['menu_url']
+	else:
+		url = iview_config['api_url'] + 'seriesIndex'
+	index_data = maybe_fetch(url)
 
 	return parser.parse_index(index_data)
 
@@ -89,8 +99,12 @@
 		which contain the items (i.e. the actual episodes).
 	"""
 
-	series_json = maybe_fetch(iview_config['api_url'] + 'series=%s' % series_id)
-	return parser.parse_series_items(series_json, get_meta)
+	if True:
+		url = iview_config['base'] + series_id
+	else:
+		url = iview_config['api_url'] + 'series=%s' % series_id
+	data = maybe_fetch(url)
+	return parser.parse_series_items(data, get_meta)
 
 def get_captions(url):
 	"""	This function takes a program name (e.g. news/730report_100803) and

=== modified file 'iview/config.py'
--- iview/config.py	2011-03-05 01:42:51 +0000
+++ iview/config.py	2011-11-11 04:18:15 +0000
@@ -1,7 +1,6 @@
 import os
 
-version     = '0.2'
-api_version = 374
+version     = '0.2+vadmium0'
 
 # os.uname() is not available on Windows, so we make this optional.
 try:
@@ -12,11 +11,18 @@
 
 user_agent = 'Python-iView %s%s' % (version, os_string)
 
-config_url   = 'http://www.abc.net.au/iview/xml/config.xml?r=%d' % api_version
+sites = dict(
+    iview=dict(config_url='http://www.abc.net.au/iview/xml/config.xml'),
+    sbs=dict(config_url='http://player.sbs.com.au'
+        '/playerassets/programs/standalone_settings.xml'),
+)
+globals().update(sites['sbs'])
+
 series_url   = 'http://www.abc.net.au/iview/api/series_mrss.htm?id=%s'
 captions_url = 'http://www.abc.net.au/iview/captions/%s.xml'
 
-akamai_playpath_prefix = '/flash/playback/_definst_/'
+if False:
+	akamai_playpath_prefix = '/flash/playback/_definst_/'
 
 # Used for "SWF verification", a stream obfuscation technique
 swf_hash    = '96cc76f1d5385fb5cda6e2ce5c73323a399043d0bb6c687edd807e5c73c42b37'

=== modified file 'iview/fetch.py'
--- iview/fetch.py	2010-08-07 06:50:47 +0000
+++ iview/fetch.py	2011-11-11 04:22:41 +0000
@@ -4,6 +4,11 @@
 import subprocess
 
 def get_filename(url):
+	if True:
+		url = url.rpartition(' playpath=')[-1]
+		url = url.rpartition(':')[-1]
+		return url.rpartition('/')[-1]
+	
 	return ''.join((
 		'.'.join(url.split('.')[:-1]).split('/')[-1],
 		'.flv',
@@ -17,17 +22,26 @@
 			'flvstreamer_x86',
 		)
 
-	args = [
-			None, # Name of executable; written to later.
-			'--host', rtmp_host,
-			'--app',  rtmp_app,
-			'--playpath', rtmp_playpath,
+	args = [None] # Name of executable; written to later.
+	
+	params = (
+		(rtmp_host, '--host'),
+		(rtmp_app, '--app'),
+		(rtmp_playpath, '--playpath'),
+	)
+	if any(value is None for (value, _) in params):
+		args.extend(('--rtmp', rtmp_url))
+	for (value, opt) in params:
+		if value is not None:
+			args.extend((opt, value))
+	
+	args.extend([
 			'--swfhash',  config.swf_hash,
 			'--swfsize',  config.swf_size,
 			'--swfUrl',   config.swf_url,
 		#	'-V', # verbose
 			'-o', output_filename
-		]
+		])
 
 	if resume:
 		args.append('--resume')
@@ -61,21 +75,34 @@
 	else:
 		resume = False
 
-	auth = comm.get_auth()
-
-	ext = url.split('.')[-1]
-	url = '.'.join(url.split('.')[:-1]) # strip the extension (.flv or .mp4)
-
-	url = auth['playpath_prefix'] + url
-
-	if ext == 'mp4':
-		url = ''.join(('mp4:', url))
+	if True:
+		playpath = url.partition(' playpath=')
+		if playpath[1]:
+			url = playpath[0]
+			playpath = playpath[-1]
+		else:
+			playpath = None
+		
+		host = app = None
+	
+	else:
+		auth = comm.get_auth()
+
+		ext = url.split('.')[-1]
+		playpath = '.'.join(url.split('.')[:-1]) # strip the extension (.flv or .mp4)
+
+		playpath = auth['playpath_prefix'] + playpath
+
+		if ext == 'mp4':
+			playpath = ''.join(('mp4:', playpath))
+		
+		url = auth['rtmp_url']
+		host = auth['rtmp_host']
+		app = auth['rtmp_app'] + '?auth=' + auth['token']
 
 	return rtmpdump(
-			auth['rtmp_url'],
-			auth['rtmp_host'],
-			auth['rtmp_app'] + '?auth=' + auth['token'],
 			url,
+			host, app, playpath,
 			dest_file,
 			resume,
 			execvp

=== modified file 'iview/parser.py'
--- iview/parser.py	2011-02-21 12:34:38 +0000
+++ iview/parser.py	2011-11-11 04:22:16 +0000
@@ -6,6 +6,28 @@
 except ImportError:
 	import simplejson as json
 
+entities_param = dict(convertEntities=BeautifulStoneSoup.XML_ENTITIES)
+try:
+	try:
+		BeautifulStoneSoup(' ', **entities_param)
+	except ValueError:
+		from BeautifulSoup import HTMLParserBuilder
+		class HexParser(HTMLParserBuilder):
+			def handle_charref(self, ref):
+				if ref.startswith('x'):
+					ref = int(ref[1:], 16)
+				return HTMLParserBuilder.handle_charref(self,
+					ref)
+		
+		with_builder = dict(entities_param, builder=HexParser)
+		BeautifulStoneSoup(' ', **with_builder)
+		entities_param = with_builder
+except Exception:
+	from sys import excepthook
+	from sys import exc_info
+	excepthook(*exc_info())
+	pass
+
 def parse_config(soup):
 	"""	There are lots of goodies in the config we get back from the ABC.
 		In particular, it gives us the URLs of all the other XML data we
@@ -14,8 +36,12 @@
 
 	soup = soup.replace('&', '&')
 
-	xml = BeautifulStoneSoup(soup)
+	xml = BeautifulStoneSoup(soup, **entities_param)
 
+	if True:
+		return dict(menu_url=xml.find(
+			'setting', dict(name="menuURL"))['value'])
+	
 	# should look like "rtmp://cp53909.edgefcs.net/ondemand"
 	rtmp_url = xml.find('param', attrs={'name':'server_streaming'}).get('value')
 	rtmp_chunks = rtmp_url.split('/')
@@ -29,49 +55,57 @@
 		'categories_url' : xml.find('param', attrs={'name':'categories'}).get('value'),
 	}
 
-def parse_auth(soup):
-	"""	There are lots of goodies in the auth handshake we get back,
-		but the only ones we are interested in are the RTMP URL, the auth
-		token, and whether the connection is unmetered.
-	"""
-
-	xml = BeautifulStoneSoup(soup)
-
-	# should look like "rtmp://203.18.195.10/ondemand"
-	rtmp_url = xml.find('server').string
-
-	playpath_prefix = ''
-
-	if rtmp_url is not None:
-		# Being directed to a custom streaming server (i.e. for unmetered services).
-		# Currently this includes Hostworks for all unmetered ISPs except iiNet.
-
-		rtmp_chunks = rtmp_url.split('/')
-		rtmp_host = rtmp_chunks[2]
-		rtmp_app = rtmp_chunks[3]
-	else:
-		# We are a bland generic ISP using Akamai, or we are iiNet.
-
-		if not comm.iview_config:
-			comm.get_config()
-
-		playpath_prefix = config.akamai_playpath_prefix
-
-		rtmp_url  = comm.iview_config['rtmp_url']
-		rtmp_host = comm.iview_config['rtmp_host']
-		rtmp_app  = comm.iview_config['rtmp_app']
-
-	token = xml.find("token").string
-	token = token.replace('&', '&') # work around BeautifulSoup bug
-
-	return {
-		'rtmp_url'        : rtmp_url,
-		'rtmp_host'       : rtmp_host,
-		'rtmp_app'        : rtmp_app,
-		'playpath_prefix' : playpath_prefix,
-		'token'           : token,
-		'free'            : (xml.find("free").string == "yes")
-	}
+if False:
+	def parse_auth(soup):
+		"""	There are lots of goodies in the auth handshake we get back,
+			but the only ones we are interested in are the RTMP URL, the auth
+			token, and whether the connection is unmetered.
+		"""
+
+		xml = BeautifulStoneSoup(soup)
+
+		# should look like "rtmp://203.18.195.10/ondemand"
+		rtmp_url = xml.find('server').string
+
+		playpath_prefix = ''
+
+		if rtmp_url is not None:
+			# Being directed to a custom streaming server (i.e. for unmetered services).
+			# Currently this includes Hostworks for all unmetered ISPs except iiNet.
+
+			rtmp_chunks = rtmp_url.split('/')
+			rtmp_host = rtmp_chunks[2]
+			rtmp_app = rtmp_chunks[3]
+		else:
+			# We are a bland generic ISP using Akamai, or we are iiNet.
+
+			if not comm.iview_config:
+				comm.get_config()
+
+			playpath_prefix = config.akamai_playpath_prefix
+
+			rtmp_url  = comm.iview_config['rtmp_url']
+			rtmp_host = comm.iview_config['rtmp_host']
+			rtmp_app  = comm.iview_config['rtmp_app']
+
+		token = xml.find("token").string
+		token = token.replace('&', '&') # work around BeautifulSoup bug
+
+		return {
+			'rtmp_url'        : rtmp_url,
+			'rtmp_host'       : rtmp_host,
+			'rtmp_app'        : rtmp_app,
+			'playpath_prefix' : playpath_prefix,
+			'token'           : token,
+			'free'            : (xml.find("free").string == "yes")
+		}
+
+class MenuParser(BeautifulStoneSoup):
+	# <menu> allowed inside another <menu> element
+	NESTABLE_TAGS = dict(menu=())
+	
+	# Other tags cannot implicitly these elements either
+	RESET_NESTING_TAGS = dict.fromkeys(NESTABLE_TAGS)
 
 def parse_index(soup):
 	"""	This function parses the index, which is an overall listing
@@ -79,28 +113,85 @@
 		'series' and 'items'. Series are things like 'beached az', while
 		items are things like 'beached az Episode 8'.
 	"""
+	if True:
+		xml = MenuParser(soup, **entities_param)
+		hierarchy = [iter(xml.findAll('menu', recursive=False))]
+
+		while hierarchy:
+			cursor = hierarchy[-1]
+			try:
+				item = next(cursor)
+			except StopIteration:
+				hierarchy.pop()
+			else:
+				playlist = item.find('playlist',
+					recursive=False)
+				if playlist:
+					id = playlist['xmlsrc']
+				else:
+					hierarchy.append(iter(item.findAll(
+						'menu', recursive=False)))
+					id = None
+				yield dict(
+					title=item.titleTag.string,
+					id=id,
+					description=
+						item.descriptionTag.string,
+				)
+				
+				continue
+			yield
+		
+		return
+	
 	index_json = json.loads(soup)
 	index_json.sort(key=lambda series: series['b']) # alphabetically sort by title
 
-	index_dict = []
-
 	for series in index_json:
 		# HACK: replace &amp; with & because HTML entities don't make
 		# the slightest bit of sense inside a JSON structure.
 		title = series['b'].replace('&amp;', '&')
 
-		index_dict.append({
+		yield {
 			'id'    : series['a'],
 			'title' : title,
-		})
-
-	return index_dict
+		}
 
 def parse_series_items(soup, get_meta=False):
+	if True:
+		xml = BeautifulStoneSoup(soup, **entities_param)
+		top = xml.find()
+		
+		if top.name == 'playlist':
+			for item in top.findAll('video'):
+				yield dict(
+					title=item.titleTag.string,
+					description=
+						item.descriptionTag.string,
+					id=item['src'],
+				)
+		
+		elif top.name == 'smil':
+			base = top.find('meta', dict(base=True))['base']
+			for item in top.findAll('video'):
+				src = item['src']
+				yield dict(
+					title=src,
+					description=
+						"system-bitrate={0}".format(
+						item['system-bitrate']),
+					url="{0} playpath={1}".format(
+						base, src),
+				)
+		
+		else:
+			raise ValueError("Unexpected menu element: {0}".
+				format(top.name))
+		
+		return
+	
 	series_json = json.loads(soup)
 
-	items = []
-
 	try:
 		for item in series_json[0]['f']:
 			for optional_key in ('d', 'r', 's', 'l'):
@@ -109,7 +200,7 @@
 				except KeyError:
 					item[optional_key] = ''
 
-			items.append({
+			yield {
 				'id'          : item['a'],
 				'title'       : item['b'].replace('&amp;', '&'), # HACK. See comment in parse_index()
 				'description' : item['d'].replace('&amp;', '&'),
@@ -118,7 +209,7 @@
 				'thumb'       : item['s'],
 				'date'        : item['f'],
 				'home'        : item['l'], # program website
-			})
+			}
 	except KeyError:
 		print 'An item we parsed had some missing info, so we skipped an episode. Maybe the ABC changed their API recently?'
 
@@ -128,9 +219,7 @@
 			'title' : series_json[0]['b'],
 			'thumb' : series_json[0]['d'],
 		}
-		return (items, meta)
-	else:
-		return items
+		raise StopIteration(meta)
 
 def parse_captions(soup):
 	"""	Converts custom iView captions into SRT format, usable in most