scrapeS3.py

#!/usr/bin/env python3
from urllib.request import urlopen
from urllib.parse import quote
import xml.etree.ElementTree as ET
import sys

def read_xml(f):
	it = ET.iterparse(f)
	for _, el in it:
		el.tag = el.tag.split('}', 1)[1]  # strip all namespaces
	root = it.root
	return root

key = None
page = 0
num_files = 0

if len(sys.argv) == 3:
	key = sys.argv[2]
if len(sys.argv) in (2, 3):
	base = sys.argv[1]
else:
	print('Usage: python scrapeS3.py BaseURL [StartKey]')
	sys.exit()

out_file = 'urls.txt'
with open(out_file, 'a', encoding='utf-8') as f:
	try:
		while True:
			files = list()
			page += 1
			url = f"{base}?marker={quote(key, safe='')}" if key else base
			sys.stdout.write(f'Enumerating page {page}: {url}\n')
			sys.stdout.flush()
			tree = read_xml(urlopen(url))
			key = None
			for entry in tree.findall('Contents'):
				key = entry.find('Key').text
				files.append(base + quote(key))
			num_files += len(files)
			f.write('\n'.join(files) + '\n')
			if key is None:
				break
	except KeyboardInterrupt:
		pass
	finally:
		sys.stdout.write('\n')
		sys.stdout.flush()

print(f'Wrote {num_files} URLs to {out_file}!')

No.