Put all debian directories of a repository in a subdirectory

create_one_repo.py

#!/usr/bin/python3
import urllib.request
import gzip
import sys
import io
import pprint
import os
import tarfile
import shutil
import subprocess

def read_url(spiURL):
	try:
		result = urllib.request.urlopen(spiURL)
	except urllib2.HTTPError as e:
		result = e
	ret = result.read()
	result.close()
	return ret

def gunzip(spiCompressed):
	file_obg = io.BytesIO(spiCompressed)
	gzip_file_handle = gzip.GzipFile(fileobj=file_obg)
	ret = gzip_file_handle.read()
	gzip_file_handle.close()
	return ret

def parse_sources(spiSources):
	data = []
	source = {}
	infiles = False
	file_obj = io.StringIO(str(spiSources, "UTF-8"))
	for line in file_obj.readlines():
		line = line.strip("\n\r")
		if line == "":
			data.append(source)
			infiles = False
			source = {}
		if line.startswith("Package:"):
			source["Package"] = line.split(" ")[1]
		if line.startswith("Version:"):
			source["Version"] = line.split(" ")[1]
		if line.startswith("Directory:"):
			source["Directory"] = line.split(" ")[1]
		if line.startswith("Files:"):
			infiles = True
			continue
		if line.startswith(" ") and infiles:
			parts = line.split(" ")
			md5sum = parts[1]
			filename = parts[3]
			if filename.find("orig.tar") != -1: source["tar.gz"] = filename
			if filename.endswith("diff.gz"): source["patch"] = filename
			if filename.endswith("debian.tar.gz"): source["patch"] = filename
			if filename.endswith("debian.tar.bz2"): source["patch"] = filename
			if filename.endswith("debian.tar.xz"): source["patch"] = filename
			if filename.endswith("dsc"): source["dsc"] = filename
		if infiles and line.find(":") != -1: infiles = False
	file_obj.close()
	return data

def pretty_print(spiText):
	pp = pprint.PrettyPrinter(indent=4)
	pp.pprint(spiText)

def untar(spiTarball,spiTargetDir):
	tarball_file_handle = io.BytesIO(spiTarball)
	tar = tarfile.open(fileobj=tarball_file_handle)
	tar.extractall(path=spiTargetDir)
	tar.close()
	tarball_file_handle.close()

def apply_patch(spiTargetDir, spiPatch):
	tmp_dir = os.getcwd()
	os.chdir(spiTargetDir)
	proc = subprocess.Popen(["/usr/bin/patch", "-p1"], stdin=subprocess.PIPE, stdout=open('/dev/null', 'w'))
	proc.communicate(spiPatch)
	os.chdir(tmp_dir)

def move_to_target_dir(spiSourceDir, spiTargetDir):
	for root, dirs, files in os.walk(spiSourceDir):
		for d in dirs:
			shutil.move(os.path.join(root,d),spiTargetDir)
		for f in files:
			try:
				shutil.move(os.path.join(root, f), spiTargetDir)
			except Exception as e:
				print(e,f, 'already exists in', spiTargetDir)
		break

def mkdir_p(spiPath):
	if not os.access(spiPath, os.F_OK):
		os.makedirs(spiPath)

def create_repo_dir(spiRepoDir, spiPatchURL, spiPatchName):
	mkdir_p(spiRepoDir)
	patch = read_url(spiPatchURL)
	if spiPatchName.endswith("tar.gz") or spiPatchName.endswith("tar.bz2") or spiPatchName.endswith("tar.xz"):
		print(spiPatchURL)
		untar(patch, spiRepoDir)
	elif spiPatchName.endswith("diff.gz"):
		apply_patch(spiRepoDir, gunzip(patch))
	else:
		print(spiRepoDir)
		sys.exit(1)

	debian_directory = os.path.join(spiRepoDir,"debian")
	# if there is only a debian directory (default) move it one level up.
	if os.listdir(spiRepoDir) == ["debian"]:
		move_to_target_dir(debian_directory, spiRepoDir)
		os.rmdir(debian_directory)
	else:
		print("more than debian dir in %s"%(spiRepoDir))

if __name__ == "__main__":
	useArgs = ( len(sys.argv) == 4 )
	release = sys.argv[1] if useArgs else "precise"
	component = sys.argv[2] if useArgs else "apps"
	repo_dir = sys.argv[3] if useArgs else "getdeb"
	download_root = "http://archive.getdeb.net/getdeb/ubuntu/"

	parsed_sources = parse_sources(gunzip(read_url("%(download_root)s/dists/%(release)s-getdeb/%(component)s/source/Sources.gz"%locals())))

	for package in parsed_sources:
		package_name = package["Package"]
		package_dir = os.path.join(repo_dir, package_name)
		download_path = os.path.join(download_root,package["Directory"],package["patch"])

		if not os.path.exists(package_dir):
			create_repo_dir(package_dir, download_path, package["patch"])
		else:
			print("%(package_name)s already exists"%locals())