OmniROM's roomservice is now strictly blocking Unofficial Builds if the Device Tree is not available in their Gerrit/Github Organization.
These files are patched to bypass those restrictions.
-
-
Save lopestom/c120df1a425959d96f577701d9c51437 to your computer and use it in GitHub Desktop.
Fixed OmniROM's roomservice for Unofficial builds
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env python | |
| # Copyright (C) 2013 Cybojenix <[email protected]> | |
| # Copyright (C) 2013 The OmniROM Project | |
| # | |
| # This program is free software: you can redistribute it and/or modify | |
| # it under the terms of the GNU General Public License as published by | |
| # the Free Software Foundation, either version 3 of the License, or | |
| # (at your option) any later version. | |
| # | |
| # This program is distributed in the hope that it will be useful, | |
| # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
| # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
| # GNU General Public License for more details. | |
| # | |
| # You should have received a copy of the GNU General Public License | |
| # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
| from __future__ import print_function | |
| import json | |
| import sys | |
| import os | |
| import os.path | |
| import re | |
| from xml.etree import ElementTree as ES | |
| # Use the urllib importer from the Cyanogenmod roomservice | |
| try: | |
| # For python3 | |
| import urllib.request | |
| except ImportError: | |
| # For python2 | |
| import imp | |
| import urllib2 | |
| urllib = imp.new_module('urllib') | |
| urllib.request = urllib2 | |
| # Config | |
| # set this to the default remote to use in repo | |
| default_rem = "omnirom2" | |
| # set this to the default revision to use (branch/tag name) | |
| default_rev = "android-10" | |
| # set this to the remote that you use for projects from your team repos | |
| # example fetch="https://github.com/omnirom" | |
| default_team_rem = "omnirom2" | |
| # this shouldn't change unless google makes changes | |
| local_manifest_dir = ".repo/local_manifests" | |
| # change this to your name on github (or equivalent hosting) | |
| android_team = "omnirom" | |
| # url to gerrit repository | |
| gerrit_url = "gerrit.omnirom.org" | |
| def check_repo_exists(git_data, device): | |
| re_match = "^android_device_.*_{device}$".format(device=device) | |
| matches = list(filter(lambda x: re.match(re_match, x), git_data)) | |
| if len(matches) != 1: | |
| raise Exception("{device} not found," | |
| "exiting roomservice".format(device=device)) | |
| return git_data[matches[0]] | |
| def search_gerrit_for_device(device): | |
| # TODO: In next gerrit release regex search with r= should be supported! | |
| git_search_url = "https://{gerrit_url}/projects/?m={device}".format( | |
| gerrit_url=gerrit_url, | |
| device=device | |
| ) | |
| git_req = urllib.request.Request(git_search_url) | |
| try: | |
| response = urllib.request.urlopen(git_req) | |
| except urllib.request.HTTPError: | |
| print("There was an issue connecting to gerrit." | |
| " Please try again in a minute") | |
| except urllib.request.URLError: | |
| print("WARNING: No network connection available.") | |
| else: | |
| # Skip silly gerrit "header" | |
| response.readline() | |
| git_data = json.load(response) | |
| device_data = check_repo_exists(git_data, device) | |
| print("found the {} device repo".format(device)) | |
| return device_data | |
| def parse_device_directory(device_url, device): | |
| pattern = "^android_device_(?P<vendor>.+)_{}$".format(device) | |
| match = re.match(pattern, device_url) | |
| if match is None: | |
| raise Exception("Invalid project name {}".format(device_url)) | |
| return "device/{vendor}/{device}".format( | |
| vendor=match.group('vendor'), | |
| device=device, | |
| ) | |
| # Thank you RaYmAn | |
| def iterate_manifests(): | |
| files = [] | |
| for file in os.listdir(local_manifest_dir): | |
| if file.endswith(".xml"): | |
| files.append(os.path.join(local_manifest_dir, file)) | |
| files.append('.repo/manifest.xml') | |
| for file in files: | |
| try: | |
| man = ES.parse(file) | |
| man = man.getroot() | |
| except (IOError, ES.ParseError): | |
| print("WARNING: error while parsing %s" % file) | |
| else: | |
| for project in man.findall("project"): | |
| yield project | |
| def iterate_manifests_remove_project(): | |
| files = [] | |
| for file in os.listdir(local_manifest_dir): | |
| if file.endswith(".xml"): | |
| files.append(os.path.join(local_manifest_dir, file)) | |
| files.append('.repo/manifest.xml') | |
| for file in files: | |
| try: | |
| man = ES.parse(file) | |
| man = man.getroot() | |
| except (IOError, ES.ParseError): | |
| print("WARNING: error while parsing %s" % file) | |
| else: | |
| for project in man.findall("remove-project"): | |
| yield project | |
| def check_project_exists(url, revision, path): | |
| for project in iterate_manifests(): | |
| if project.get("name") == url \ | |
| and project.get("revision") == revision \ | |
| and project.get("path") == path: | |
| return True | |
| return False | |
| def check_remove_project_exists(url): | |
| for project in iterate_manifests_remove_project(): | |
| if project.get("name") == url: | |
| return True | |
| return False | |
| def check_target_exists(directory): | |
| return os.path.isdir(directory) | |
| # Use the indent function from http://stackoverflow.com/a/4590052 | |
| def indent(elem, level=0): | |
| i = ''.join(["\n", level*" "]) | |
| if len(elem): | |
| if not elem.text or not elem.text.strip(): | |
| elem.text = ''.join([i, " "]) | |
| if not elem.tail or not elem.tail.strip(): | |
| elem.tail = i | |
| for elem in elem: | |
| indent(elem, level+1) | |
| if not elem.tail or not elem.tail.strip(): | |
| elem.tail = i | |
| else: | |
| if level and (not elem.tail or not elem.tail.strip()): | |
| elem.tail = i | |
| def create_manifest_project(url, directory, | |
| remote=default_rem, | |
| revision=default_rev): | |
| project_exists = check_project_exists(url, revision, directory) | |
| if project_exists: | |
| return None | |
| project = ES.Element("project", | |
| attrib={ | |
| "path": directory, | |
| "name": url, | |
| "remote": remote, | |
| "revision": revision | |
| }) | |
| return project | |
| def create_remove_project(url): | |
| remove_project_exists = check_remove_project_exists(url) | |
| if remove_project_exists: | |
| return None | |
| project = ES.Element("remove-project", | |
| attrib={ | |
| "name": url | |
| }) | |
| return project | |
| # Avoid adding already in manifests declared repositories in the roomservice.xml | |
| def is_in_manifest(project): | |
| files = [] | |
| for file in os.listdir(local_manifest_dir): | |
| if file.endswith(".xml") and file != "roomservice.xml": | |
| files.append(os.path.join(local_manifest_dir, file)) | |
| files.append('.repo/manifest.xml') | |
| for file in files: | |
| try: | |
| man = ES.parse(file) | |
| man = man.getroot() | |
| for project_in_manifests in man.findall("project"): | |
| if project_in_manifests.get("path") == project.get("path") or project_in_manifests.get("path") == project.get("target_path"): # path -> def append_to_manifest(project) & target_path -> def check_manifest_problems(dependencies) | |
| return True | |
| except (IOError, ES.ParseError): | |
| print("WARNING: error while parsing %s" % file) | |
| return False | |
| def append_to_manifest(project): | |
| try: | |
| lm = ES.parse('/'.join([local_manifest_dir, "roomservice.xml"])) | |
| lm = lm.getroot() | |
| except (IOError, ES.ParseError): | |
| lm = ES.Element("manifest") | |
| if is_in_manifest(project): | |
| print("HINT: The following repository is already defined in a manifest:", project.get("name")) | |
| return lm | |
| lm.append(project) | |
| return lm | |
| def write_to_manifest(manifest): | |
| indent(manifest) | |
| raw_xml = ES.tostring(manifest).decode() | |
| raw_xml = ''.join(['<?xml version="1.0" encoding="UTF-8"?>\n' | |
| '<!--Please do not manually edit this file-->\n', | |
| raw_xml]) | |
| with open('/'.join([local_manifest_dir, "roomservice.xml"]), 'w') as f: | |
| f.write(raw_xml) | |
| print("wrote the new roomservice manifest") | |
| def parse_device_from_manifest(device): | |
| for project in iterate_manifests(): | |
| name = project.get('name') | |
| if name.startswith("android_device_") and name.endswith(device): | |
| return project.get('path') | |
| return None | |
| def parse_device_from_folder(device): | |
| search = [] | |
| if not os.path.isdir("device"): | |
| os.mkdir("device") | |
| for sub_folder in os.listdir("device"): | |
| if os.path.isdir("device/%s/%s" % (sub_folder, device)): | |
| search.append("device/%s/%s" % (sub_folder, device)) | |
| if len(search) > 1: | |
| print("multiple devices under the name %s. " | |
| "defaulting to checking the manifest" % device) | |
| location = parse_device_from_manifest(device) | |
| elif len(search) == 1: | |
| location = search[0] | |
| else: | |
| print("Your device can't be found in device sources..") | |
| location = parse_device_from_manifest(device) | |
| return location | |
| def parse_dependency_file(location): | |
| dep_file = "omni.dependencies" | |
| dep_location = '/'.join([location, dep_file]) | |
| if not os.path.isfile(dep_location): | |
| print("WARNING: %s file not found" % dep_location) | |
| sys.exit() | |
| try: | |
| with open(dep_location, 'r') as f: | |
| dependencies = json.loads(f.read()) | |
| except ValueError: | |
| raise Exception("ERROR: malformed dependency file") | |
| return dependencies | |
| # if there is any conflict with existing and new | |
| # delete the roomservice.xml file and create new | |
| def check_manifest_problems(dependencies): | |
| for dependency in dependencies: | |
| if is_in_manifest(dependency): | |
| continue | |
| repository = dependency.get("repository") | |
| target_path = dependency.get("target_path") | |
| revision = dependency.get("revision", default_rev) | |
| # check for existing projects | |
| for project in iterate_manifests(): | |
| if project.get("revision") is not None \ | |
| and project.get("path") is not None \ | |
| and project.get("path") == target_path \ | |
| and project.get("revision") != revision: | |
| print("WARNING: detected conflict in revisions for repository ", | |
| repository) | |
| current_dependency = str(project.get(repository)) | |
| file = ES.parse('/'.join([local_manifest_dir, | |
| "roomservice.xml"])) | |
| file_root = file.getroot() | |
| for current_project in file_root.findall('project'): | |
| new_dependency = str(current_project.find('revision')) | |
| if new_dependency == current_dependency: | |
| file_root.remove(current_project) | |
| file.write('/'.join([local_manifest_dir, "roomservice.xml"])) | |
| return | |
| def create_dependency_manifest(dependencies): | |
| projects = [] | |
| for dependency in dependencies: | |
| repository = dependency.get("repository") | |
| target_path = dependency.get("target_path") | |
| revision = dependency.get("revision", default_rev) | |
| remote = dependency.get("remote", default_rem) | |
| override = dependency.get("override", None) | |
| remove = dependency.get("remove", None) | |
| if remove is not None: | |
| #print("found remove in ", repository) | |
| project = create_remove_project(remove) | |
| if project is not None: | |
| manifest = append_to_manifest(project) | |
| #print(ES.tostring(manifest).decode()) | |
| write_to_manifest(manifest) | |
| else: | |
| if override is not None: | |
| #print("found override in ", repository) | |
| project = create_remove_project(override) | |
| if project is not None: | |
| manifest = append_to_manifest(project) | |
| #print(ES.tostring(manifest).decode()) | |
| write_to_manifest(manifest) | |
| # not adding an organization should default to android_team | |
| # only apply this to github | |
| if remote == "github": | |
| if "/" not in repository: | |
| repository = '/'.join([android_team, repository]) | |
| project = create_manifest_project(repository, | |
| target_path, | |
| remote=remote, | |
| revision=revision) | |
| if project is not None: | |
| manifest = append_to_manifest(project) | |
| write_to_manifest(manifest) | |
| projects.append(target_path) | |
| if len(projects) > 0: | |
| os.system("repo sync -f --no-clone-bundle %s" % " ".join(projects)) | |
| def create_common_dependencies_manifest(dependencies): | |
| dep_file = "omni.dependencies" | |
| common_list = [] | |
| if dependencies is not None: | |
| for dependency in dependencies: | |
| try: | |
| index = common_list.index(dependency['target_path']) | |
| except ValueError: | |
| index = None | |
| if index is None: | |
| common_list.append(dependency['target_path']) | |
| dep_location = '/'.join([dependency['target_path'], dep_file]) | |
| if not os.path.isfile(dep_location): | |
| sys.exit() | |
| else: | |
| try: | |
| with open(dep_location, 'r') as f: | |
| common_deps = json.loads(f.read()) | |
| except ValueError: | |
| raise Exception("ERROR: malformed dependency file") | |
| if common_deps is not None: | |
| print("Looking for dependencies on: ", | |
| dependency['target_path']) | |
| check_manifest_problems(common_deps) | |
| create_dependency_manifest(common_deps) | |
| create_common_dependencies_manifest(common_deps) | |
| def fetch_dependencies(device): | |
| if device == "emulator": | |
| location = "vendor/omni/utils/emulator" | |
| else: | |
| location = parse_device_from_folder(device) | |
| if location is None or not os.path.isdir(location): | |
| raise Exception("ERROR: could not find your device " | |
| "folder location, bailing out") | |
| dependencies = parse_dependency_file(location) | |
| check_manifest_problems(dependencies) | |
| create_dependency_manifest(dependencies) | |
| create_common_dependencies_manifest(dependencies) | |
| fetch_device(device) | |
| def check_device_exists(device): | |
| location = parse_device_from_folder(device) | |
| if location is None: | |
| return False | |
| return os.path.isdir(location) | |
| def fetch_device(device): | |
| if check_device_exists(device): | |
| print("HINT: Avoid fetching the already checked out device repo:", device) | |
| return | |
| git_data = search_gerrit_for_device(device) | |
| if git_data is not None: | |
| device_url = git_data['id'] | |
| device_dir = parse_device_directory(device_url, device) | |
| project = create_manifest_project(device_url, | |
| device_dir, | |
| remote=default_team_rem) | |
| if project is not None: | |
| manifest = append_to_manifest(project) | |
| write_to_manifest(manifest) | |
| # In case a project was written to manifest, but never synced | |
| if project is not None or not check_target_exists(device_dir): | |
| print("syncing the device config") | |
| os.system('repo sync -f --no-clone-bundle %s' % device_dir) | |
| if __name__ == '__main__': | |
| if not os.path.isdir(local_manifest_dir): | |
| os.mkdir(local_manifest_dir) | |
| product = sys.argv[1] | |
| try: | |
| device = product[product.index("_") + 1:] | |
| except ValueError: | |
| device = product | |
| if len(sys.argv) > 2: | |
| deps_only = sys.argv[2] | |
| else: | |
| deps_only = False | |
| if not deps_only: | |
| fetch_device(device) | |
| fetch_dependencies(device) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env python | |
| # Copyright (C) 2013 Cybojenix <[email protected]> | |
| # Copyright (C) 2013 The OmniROM Project | |
| # | |
| # This program is free software: you can redistribute it and/or modify | |
| # it under the terms of the GNU General Public License as published by | |
| # the Free Software Foundation, either version 3 of the License, or | |
| # (at your option) any later version. | |
| # | |
| # This program is distributed in the hope that it will be useful, | |
| # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
| # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
| # GNU General Public License for more details. | |
| # | |
| # You should have received a copy of the GNU General Public License | |
| # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
| from __future__ import print_function | |
| import json | |
| import sys | |
| import os | |
| import os.path | |
| import re | |
| from xml.etree import ElementTree as ES | |
| # Use the urllib importer from the Cyanogenmod roomservice | |
| try: | |
| # For python3 | |
| import urllib.request | |
| except ImportError: | |
| # For python2 | |
| import imp | |
| import urllib2 | |
| urllib = imp.new_module('urllib') | |
| urllib.request = urllib2 | |
| # Config | |
| # set this to the default remote to use in repo | |
| default_rem = "omnirom" | |
| # set this to the default revision to use (branch/tag name) | |
| default_rev = "android-5.1" | |
| # set this to the remote that you use for projects from your team repos | |
| # example fetch="https://github.com/omnirom" | |
| default_team_rem = "omnirom" | |
| # this shouldn't change unless google makes changes | |
| local_manifest_dir = ".repo/local_manifests" | |
| # change this to your name on github (or equivalent hosting) | |
| android_team = "omnirom" | |
| # url to gerrit repository | |
| gerrit_url = "gerrit.omnirom.org" | |
| def check_repo_exists(git_data, device): | |
| re_match = "^android_device_.*_{device}$".format(device=device) | |
| matches = list(filter(lambda x: re.match(re_match, x), git_data)) | |
| if len(matches) != 1: | |
| raise Exception("{device} not found," | |
| "exiting roomservice".format(device=device)) | |
| return git_data[matches[0]] | |
| def search_gerrit_for_device(device): | |
| # TODO: In next gerrit release regex search with r= should be supported! | |
| git_search_url = "https://{gerrit_url}/projects/?m={device}".format( | |
| gerrit_url=gerrit_url, | |
| device=device | |
| ) | |
| git_req = urllib.request.Request(git_search_url) | |
| try: | |
| response = urllib.request.urlopen(git_req) | |
| except urllib.request.HTTPError: | |
| print("There was an issue connecting to gerrit." | |
| " Please try again in a minute") | |
| except urllib.request.URLError: | |
| print("WARNING: No network connection available.") | |
| else: | |
| # Skip silly gerrit "header" | |
| response.readline() | |
| git_data = json.load(response) | |
| device_data = check_repo_exists(git_data, device) | |
| print("found the {} device repo".format(device)) | |
| return device_data | |
| def parse_device_directory(device_url, device): | |
| pattern = "^android_device_(?P<vendor>.+)_{}$".format(device) | |
| match = re.match(pattern, device_url) | |
| if match is None: | |
| raise Exception("Invalid project name {}".format(device_url)) | |
| return "device/{vendor}/{device}".format( | |
| vendor=match.group('vendor'), | |
| device=device, | |
| ) | |
| # Thank you RaYmAn | |
| def iterate_manifests(): | |
| files = [] | |
| for file in os.listdir(local_manifest_dir): | |
| if file.endswith(".xml"): | |
| files.append(os.path.join(local_manifest_dir, file)) | |
| files.append('.repo/manifest.xml') | |
| for file in files: | |
| try: | |
| man = ES.parse(file) | |
| man = man.getroot() | |
| except (IOError, ES.ParseError): | |
| print("WARNING: error while parsing %s" % file) | |
| else: | |
| for project in man.findall("project"): | |
| yield project | |
| def iterate_manifests_remove_project(): | |
| files = [] | |
| for file in os.listdir(local_manifest_dir): | |
| if file.endswith(".xml"): | |
| files.append(os.path.join(local_manifest_dir, file)) | |
| files.append('.repo/manifest.xml') | |
| for file in files: | |
| try: | |
| man = ES.parse(file) | |
| man = man.getroot() | |
| except (IOError, ES.ParseError): | |
| print("WARNING: error while parsing %s" % file) | |
| else: | |
| for project in man.findall("remove-project"): | |
| yield project | |
| def check_project_exists(url, revision, path): | |
| for project in iterate_manifests(): | |
| if project.get("name") == url \ | |
| and project.get("revision") == revision \ | |
| and project.get("path") == path: | |
| return True | |
| return False | |
| def check_remove_project_exists(url): | |
| for project in iterate_manifests_remove_project(): | |
| if project.get("name") == url: | |
| return True | |
| return False | |
| def check_target_exists(directory): | |
| return os.path.isdir(directory) | |
| # Use the indent function from http://stackoverflow.com/a/4590052 | |
| def indent(elem, level=0): | |
| i = ''.join(["\n", level*" "]) | |
| if len(elem): | |
| if not elem.text or not elem.text.strip(): | |
| elem.text = ''.join([i, " "]) | |
| if not elem.tail or not elem.tail.strip(): | |
| elem.tail = i | |
| for elem in elem: | |
| indent(elem, level+1) | |
| if not elem.tail or not elem.tail.strip(): | |
| elem.tail = i | |
| else: | |
| if level and (not elem.tail or not elem.tail.strip()): | |
| elem.tail = i | |
| def create_manifest_project(url, directory, | |
| remote=default_rem, | |
| revision=default_rev): | |
| project_exists = check_project_exists(url, revision, directory) | |
| if project_exists: | |
| return None | |
| project = ES.Element("project", | |
| attrib={ | |
| "path": directory, | |
| "name": url, | |
| "remote": remote, | |
| "revision": revision | |
| }) | |
| return project | |
| def create_remove_project(url): | |
| remove_project_exists = check_remove_project_exists(url) | |
| if remove_project_exists: | |
| return None | |
| project = ES.Element("remove-project", | |
| attrib={ | |
| "name": url | |
| }) | |
| return project | |
| # Avoid adding already in manifests declared repositories in the roomservice.xml | |
| def is_in_manifest(project): | |
| files = [] | |
| for file in os.listdir(local_manifest_dir): | |
| if file.endswith(".xml") and file != "roomservice.xml": | |
| files.append(os.path.join(local_manifest_dir, file)) | |
| files.append('.repo/manifest.xml') | |
| for file in files: | |
| try: | |
| man = ES.parse(file) | |
| man = man.getroot() | |
| for project_in_manifests in man.findall("project"): | |
| if project_in_manifests.get("path") == project.get("path") or project_in_manifests.get("path") == project.get("target_path"): # path -> def append_to_manifest(project) & target_path -> def check_manifest_problems(dependencies) | |
| return True | |
| except (IOError, ES.ParseError): | |
| print("WARNING: error while parsing %s" % file) | |
| return False | |
| def append_to_manifest(project): | |
| try: | |
| lm = ES.parse('/'.join([local_manifest_dir, "roomservice.xml"])) | |
| lm = lm.getroot() | |
| except (IOError, ES.ParseError): | |
| lm = ES.Element("manifest") | |
| if is_in_manifest(project): | |
| print("HINT: The following repository is already defined in a manifest:", project.get("name")) | |
| return lm | |
| lm.append(project) | |
| return lm | |
| def write_to_manifest(manifest): | |
| indent(manifest) | |
| raw_xml = ES.tostring(manifest).decode() | |
| raw_xml = ''.join(['<?xml version="1.0" encoding="UTF-8"?>\n' | |
| '<!--Please do not manually edit this file-->\n', | |
| raw_xml]) | |
| with open('/'.join([local_manifest_dir, "roomservice.xml"]), 'w') as f: | |
| f.write(raw_xml) | |
| print("wrote the new roomservice manifest") | |
| def parse_device_from_manifest(device): | |
| for project in iterate_manifests(): | |
| name = project.get('name') | |
| if name.startswith("android_device_") and name.endswith(device): | |
| return project.get('path') | |
| return None | |
| def parse_device_from_folder(device): | |
| search = [] | |
| if not os.path.isdir("device"): | |
| os.mkdir("device") | |
| for sub_folder in os.listdir("device"): | |
| if os.path.isdir("device/%s/%s" % (sub_folder, device)): | |
| search.append("device/%s/%s" % (sub_folder, device)) | |
| if len(search) > 1: | |
| print("multiple devices under the name %s. " | |
| "defaulting to checking the manifest" % device) | |
| location = parse_device_from_manifest(device) | |
| elif len(search) == 1: | |
| location = search[0] | |
| else: | |
| print("Your device can't be found in device sources..") | |
| location = parse_device_from_manifest(device) | |
| return location | |
| def parse_dependency_file(location): | |
| dep_file = "omni.dependencies" | |
| dep_location = '/'.join([location, dep_file]) | |
| if not os.path.isfile(dep_location): | |
| print("WARNING: %s file not found" % dep_location) | |
| sys.exit() | |
| try: | |
| with open(dep_location, 'r') as f: | |
| dependencies = json.loads(f.read()) | |
| except ValueError: | |
| raise Exception("ERROR: malformed dependency file") | |
| return dependencies | |
| # if there is any conflict with existing and new | |
| # delete the roomservice.xml file and create new | |
| def check_manifest_problems(dependencies): | |
| for dependency in dependencies: | |
| if is_in_manifest(dependency): | |
| continue | |
| repository = dependency.get("repository") | |
| target_path = dependency.get("target_path") | |
| revision = dependency.get("revision", default_rev) | |
| # check for existing projects | |
| for project in iterate_manifests(): | |
| if project.get("revision") is not None \ | |
| and project.get("path") is not None \ | |
| and project.get("path") == target_path \ | |
| and project.get("revision") != revision: | |
| print("WARNING: detected conflict in revisions for repository ", | |
| repository) | |
| current_dependency = str(project.get(repository)) | |
| file = ES.parse('/'.join([local_manifest_dir, | |
| "roomservice.xml"])) | |
| file_root = file.getroot() | |
| for current_project in file_root.findall('project'): | |
| new_dependency = str(current_project.find('revision')) | |
| if new_dependency == current_dependency: | |
| file_root.remove(current_project) | |
| file.write('/'.join([local_manifest_dir, "roomservice.xml"])) | |
| return | |
| def create_dependency_manifest(dependencies): | |
| projects = [] | |
| for dependency in dependencies: | |
| repository = dependency.get("repository") | |
| target_path = dependency.get("target_path") | |
| revision = dependency.get("revision", default_rev) | |
| remote = dependency.get("remote", default_rem) | |
| override = dependency.get("override", None) | |
| if override is not None: | |
| #print("found override in ", repository) | |
| project = create_remove_project(repository) | |
| if project is not None: | |
| manifest = append_to_manifest(project) | |
| #print(ES.tostring(manifest).decode()) | |
| write_to_manifest(manifest) | |
| else: | |
| if override is not None: | |
| #print("found override in ", repository) | |
| project = create_remove_project(override) | |
| if project is not None: | |
| manifest = append_to_manifest(project) | |
| #print(ES.tostring(manifest).decode()) | |
| write_to_manifest(manifest) | |
| # not adding an organization should default to android_team | |
| # only apply this to github | |
| if remote == "github": | |
| if "/" not in repository: | |
| repository = '/'.join([android_team, repository]) | |
| project = create_manifest_project(repository, | |
| target_path, | |
| remote=remote, | |
| revision=revision) | |
| if project is not None: | |
| manifest = append_to_manifest(project) | |
| write_to_manifest(manifest) | |
| projects.append(target_path) | |
| if len(projects) > 0: | |
| os.system("repo sync -f --no-clone-bundle %s" % " ".join(projects)) | |
| def create_common_dependencies_manifest(dependencies): | |
| dep_file = "omni.dependencies" | |
| common_list = [] | |
| if dependencies is not None: | |
| for dependency in dependencies: | |
| try: | |
| index = common_list.index(dependency['target_path']) | |
| except ValueError: | |
| index = None | |
| if index is None: | |
| common_list.append(dependency['target_path']) | |
| dep_location = '/'.join([dependency['target_path'], dep_file]) | |
| if not os.path.isfile(dep_location): | |
| sys.exit() | |
| else: | |
| try: | |
| with open(dep_location, 'r') as f: | |
| common_deps = json.loads(f.read()) | |
| except ValueError: | |
| raise Exception("ERROR: malformed dependency file") | |
| if common_deps is not None: | |
| print("Looking for dependencies on: ", | |
| dependency['target_path']) | |
| check_manifest_problems(common_deps) | |
| create_dependency_manifest(common_deps) | |
| create_common_dependencies_manifest(common_deps) | |
| def fetch_dependencies(device): | |
| location = parse_device_from_folder(device) | |
| if location is None or not os.path.isdir(location): | |
| raise Exception("ERROR: could not find your device " | |
| "folder location, bailing out") | |
| dependencies = parse_dependency_file(location) | |
| check_manifest_problems(dependencies) | |
| create_dependency_manifest(dependencies) | |
| create_common_dependencies_manifest(dependencies) | |
| fetch_device(device) | |
| def check_device_exists(device): | |
| location = parse_device_from_folder(device) | |
| if location is None: | |
| return False | |
| return os.path.isdir(location) | |
| def fetch_device(device): | |
| if check_device_exists(device): | |
| print("HINT: Avoid fetching the already checked out device repo:", device) | |
| return | |
| git_data = search_gerrit_for_device(device) | |
| if git_data is not None: | |
| device_url = git_data['id'] | |
| device_dir = parse_device_directory(device_url, device) | |
| project = create_manifest_project(device_url, | |
| device_dir, | |
| remote=default_team_rem) | |
| if project is not None: | |
| manifest = append_to_manifest(project) | |
| write_to_manifest(manifest) | |
| # In case a project was written to manifest, but never synced | |
| if project is not None or not check_target_exists(device_dir): | |
| print("syncing the device config") | |
| os.system('repo sync -f --no-clone-bundle %s' % device_dir) | |
| if __name__ == '__main__': | |
| if not os.path.isdir(local_manifest_dir): | |
| os.mkdir(local_manifest_dir) | |
| product = sys.argv[1] | |
| try: | |
| device = product[product.index("_") + 1:] | |
| except ValueError: | |
| device = product | |
| if len(sys.argv) > 2: | |
| deps_only = sys.argv[2] | |
| else: | |
| deps_only = False | |
| if not deps_only: | |
| fetch_device(device) | |
| fetch_dependencies(device) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env python | |
| # Copyright (C) 2013 Cybojenix <[email protected]> | |
| # Copyright (C) 2013 The OmniROM Project | |
| # | |
| # This program is free software: you can redistribute it and/or modify | |
| # it under the terms of the GNU General Public License as published by | |
| # the Free Software Foundation, either version 3 of the License, or | |
| # (at your option) any later version. | |
| # | |
| # This program is distributed in the hope that it will be useful, | |
| # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
| # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
| # GNU General Public License for more details. | |
| # | |
| # You should have received a copy of the GNU General Public License | |
| # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
| from __future__ import print_function | |
| import json | |
| import sys | |
| import os | |
| import os.path | |
| import re | |
| from xml.etree import ElementTree as ES | |
| # Use the urllib importer from the Cyanogenmod roomservice | |
| try: | |
| # For python3 | |
| import urllib.request | |
| except ImportError: | |
| # For python2 | |
| import imp | |
| import urllib2 | |
| urllib = imp.new_module('urllib') | |
| urllib.request = urllib2 | |
| # Config | |
| # set this to the default remote to use in repo | |
| default_rem = "omnirom" | |
| # set this to the default revision to use (branch/tag name) | |
| default_rev = "android-6.0" | |
| # set this to the remote that you use for projects from your team repos | |
| # example fetch="https://github.com/omnirom" | |
| default_team_rem = "omnirom" | |
| # this shouldn't change unless google makes changes | |
| local_manifest_dir = ".repo/local_manifests" | |
| # change this to your name on github (or equivalent hosting) | |
| android_team = "omnirom" | |
| # url to gerrit repository | |
| gerrit_url = "gerrit.omnirom.org" | |
| def check_repo_exists(git_data, device): | |
| re_match = "^android_device_.*_{device}$".format(device=device) | |
| matches = list(filter(lambda x: re.match(re_match, x), git_data)) | |
| if len(matches) != 1: | |
| raise Exception("{device} not found," | |
| "exiting roomservice".format(device=device)) | |
| return git_data[matches[0]] | |
| def search_gerrit_for_device(device): | |
| # TODO: In next gerrit release regex search with r= should be supported! | |
| git_search_url = "https://{gerrit_url}/projects/?m={device}".format( | |
| gerrit_url=gerrit_url, | |
| device=device | |
| ) | |
| git_req = urllib.request.Request(git_search_url) | |
| try: | |
| response = urllib.request.urlopen(git_req) | |
| except urllib.request.HTTPError: | |
| print("There was an issue connecting to gerrit." | |
| " Please try again in a minute") | |
| except urllib.request.URLError: | |
| print("WARNING: No network connection available.") | |
| else: | |
| # Skip silly gerrit "header" | |
| response.readline() | |
| git_data = json.load(response) | |
| device_data = check_repo_exists(git_data, device) | |
| print("found the {} device repo".format(device)) | |
| return device_data | |
| def parse_device_directory(device_url, device): | |
| pattern = "^android_device_(?P<vendor>.+)_{}$".format(device) | |
| match = re.match(pattern, device_url) | |
| if match is None: | |
| raise Exception("Invalid project name {}".format(device_url)) | |
| return "device/{vendor}/{device}".format( | |
| vendor=match.group('vendor'), | |
| device=device, | |
| ) | |
| # Thank you RaYmAn | |
| def iterate_manifests(): | |
| files = [] | |
| for file in os.listdir(local_manifest_dir): | |
| if file.endswith(".xml"): | |
| files.append(os.path.join(local_manifest_dir, file)) | |
| files.append('.repo/manifest.xml') | |
| for file in files: | |
| try: | |
| man = ES.parse(file) | |
| man = man.getroot() | |
| except (IOError, ES.ParseError): | |
| print("WARNING: error while parsing %s" % file) | |
| else: | |
| for project in man.findall("project"): | |
| yield project | |
| def iterate_manifests_remove_project(): | |
| files = [] | |
| for file in os.listdir(local_manifest_dir): | |
| if file.endswith(".xml"): | |
| files.append(os.path.join(local_manifest_dir, file)) | |
| files.append('.repo/manifest.xml') | |
| for file in files: | |
| try: | |
| man = ES.parse(file) | |
| man = man.getroot() | |
| except (IOError, ES.ParseError): | |
| print("WARNING: error while parsing %s" % file) | |
| else: | |
| for project in man.findall("remove-project"): | |
| yield project | |
| def check_project_exists(url, revision, path): | |
| for project in iterate_manifests(): | |
| if project.get("name") == url \ | |
| and project.get("revision") == revision \ | |
| and project.get("path") == path: | |
| return True | |
| return False | |
| def check_remove_project_exists(url): | |
| for project in iterate_manifests_remove_project(): | |
| if project.get("name") == url: | |
| return True | |
| return False | |
| def check_target_exists(directory): | |
| return os.path.isdir(directory) | |
| # Use the indent function from http://stackoverflow.com/a/4590052 | |
| def indent(elem, level=0): | |
| i = ''.join(["\n", level*" "]) | |
| if len(elem): | |
| if not elem.text or not elem.text.strip(): | |
| elem.text = ''.join([i, " "]) | |
| if not elem.tail or not elem.tail.strip(): | |
| elem.tail = i | |
| for elem in elem: | |
| indent(elem, level+1) | |
| if not elem.tail or not elem.tail.strip(): | |
| elem.tail = i | |
| else: | |
| if level and (not elem.tail or not elem.tail.strip()): | |
| elem.tail = i | |
| def create_manifest_project(url, directory, | |
| remote=default_rem, | |
| revision=default_rev): | |
| project_exists = check_project_exists(url, revision, directory) | |
| if project_exists: | |
| return None | |
| project = ES.Element("project", | |
| attrib={ | |
| "path": directory, | |
| "name": url, | |
| "remote": remote, | |
| "revision": revision | |
| }) | |
| return project | |
| def create_remove_project(url): | |
| remove_project_exists = check_remove_project_exists(url) | |
| if remove_project_exists: | |
| return None | |
| project = ES.Element("remove-project", | |
| attrib={ | |
| "name": url | |
| }) | |
| return project | |
| # Avoid adding already in manifests declared repositories in the roomservice.xml | |
| def is_in_manifest(project): | |
| files = [] | |
| for file in os.listdir(local_manifest_dir): | |
| if file.endswith(".xml") and file != "roomservice.xml": | |
| files.append(os.path.join(local_manifest_dir, file)) | |
| files.append('.repo/manifest.xml') | |
| for file in files: | |
| try: | |
| man = ES.parse(file) | |
| man = man.getroot() | |
| for project_in_manifests in man.findall("project"): | |
| if project_in_manifests.get("path") == project.get("path") or project_in_manifests.get("path") == project.get("target_path"): # path -> def append_to_manifest(project) & target_path -> def check_manifest_problems(dependencies) | |
| return True | |
| except (IOError, ES.ParseError): | |
| print("WARNING: error while parsing %s" % file) | |
| return False | |
| def append_to_manifest(project): | |
| try: | |
| lm = ES.parse('/'.join([local_manifest_dir, "roomservice.xml"])) | |
| lm = lm.getroot() | |
| except (IOError, ES.ParseError): | |
| lm = ES.Element("manifest") | |
| if is_in_manifest(project): | |
| print("HINT: The following repository is already defined in a manifest:", project.get("name")) | |
| return lm | |
| lm.append(project) | |
| return lm | |
| def write_to_manifest(manifest): | |
| indent(manifest) | |
| raw_xml = ES.tostring(manifest).decode() | |
| raw_xml = ''.join(['<?xml version="1.0" encoding="UTF-8"?>\n' | |
| '<!--Please do not manually edit this file-->\n', | |
| raw_xml]) | |
| with open('/'.join([local_manifest_dir, "roomservice.xml"]), 'w') as f: | |
| f.write(raw_xml) | |
| print("wrote the new roomservice manifest") | |
| def parse_device_from_manifest(device): | |
| for project in iterate_manifests(): | |
| name = project.get('name') | |
| if name.startswith("android_device_") and name.endswith(device): | |
| return project.get('path') | |
| return None | |
| def parse_device_from_folder(device): | |
| search = [] | |
| if not os.path.isdir("device"): | |
| os.mkdir("device") | |
| for sub_folder in os.listdir("device"): | |
| if os.path.isdir("device/%s/%s" % (sub_folder, device)): | |
| search.append("device/%s/%s" % (sub_folder, device)) | |
| if len(search) > 1: | |
| print("multiple devices under the name %s. " | |
| "defaulting to checking the manifest" % device) | |
| location = parse_device_from_manifest(device) | |
| elif len(search) == 1: | |
| location = search[0] | |
| else: | |
| print("Your device can't be found in device sources..") | |
| location = parse_device_from_manifest(device) | |
| return location | |
| def parse_dependency_file(location): | |
| dep_file = "omni.dependencies" | |
| dep_location = '/'.join([location, dep_file]) | |
| if not os.path.isfile(dep_location): | |
| print("WARNING: %s file not found" % dep_location) | |
| sys.exit() | |
| try: | |
| with open(dep_location, 'r') as f: | |
| dependencies = json.loads(f.read()) | |
| except ValueError: | |
| raise Exception("ERROR: malformed dependency file") | |
| return dependencies | |
| # if there is any conflict with existing and new | |
| # delete the roomservice.xml file and create new | |
| def check_manifest_problems(dependencies): | |
| for dependency in dependencies: | |
| if is_in_manifest(dependency): | |
| continue | |
| repository = dependency.get("repository") | |
| target_path = dependency.get("target_path") | |
| revision = dependency.get("revision", default_rev) | |
| # check for existing projects | |
| for project in iterate_manifests(): | |
| if project.get("revision") is not None \ | |
| and project.get("path") is not None \ | |
| and project.get("path") == target_path \ | |
| and project.get("revision") != revision: | |
| print("WARNING: detected conflict in revisions for repository ", | |
| repository) | |
| current_dependency = str(project.get(repository)) | |
| file = ES.parse('/'.join([local_manifest_dir, | |
| "roomservice.xml"])) | |
| file_root = file.getroot() | |
| for current_project in file_root.findall('project'): | |
| new_dependency = str(current_project.find('revision')) | |
| if new_dependency == current_dependency: | |
| file_root.remove(current_project) | |
| file.write('/'.join([local_manifest_dir, "roomservice.xml"])) | |
| return | |
| def create_dependency_manifest(dependencies): | |
| projects = [] | |
| for dependency in dependencies: | |
| repository = dependency.get("repository") | |
| target_path = dependency.get("target_path") | |
| revision = dependency.get("revision", default_rev) | |
| remote = dependency.get("remote", default_rem) | |
| override = dependency.get("override", None) | |
| if override is not None: | |
| #print("found override in ", repository) | |
| project = create_remove_project(repository) | |
| if project is not None: | |
| manifest = append_to_manifest(project) | |
| #print(ES.tostring(manifest).decode()) | |
| write_to_manifest(manifest) | |
| else: | |
| if override is not None: | |
| #print("found override in ", repository) | |
| project = create_remove_project(override) | |
| if project is not None: | |
| manifest = append_to_manifest(project) | |
| #print(ES.tostring(manifest).decode()) | |
| write_to_manifest(manifest) | |
| # not adding an organization should default to android_team | |
| # only apply this to github | |
| if remote == "github": | |
| if "/" not in repository: | |
| repository = '/'.join([android_team, repository]) | |
| project = create_manifest_project(repository, | |
| target_path, | |
| remote=remote, | |
| revision=revision) | |
| if project is not None: | |
| manifest = append_to_manifest(project) | |
| write_to_manifest(manifest) | |
| projects.append(target_path) | |
| if len(projects) > 0: | |
| os.system("repo sync -f --no-clone-bundle %s" % " ".join(projects)) | |
| def create_common_dependencies_manifest(dependencies): | |
| dep_file = "omni.dependencies" | |
| common_list = [] | |
| if dependencies is not None: | |
| for dependency in dependencies: | |
| try: | |
| index = common_list.index(dependency['target_path']) | |
| except ValueError: | |
| index = None | |
| if index is None: | |
| common_list.append(dependency['target_path']) | |
| dep_location = '/'.join([dependency['target_path'], dep_file]) | |
| if not os.path.isfile(dep_location): | |
| sys.exit() | |
| else: | |
| try: | |
| with open(dep_location, 'r') as f: | |
| common_deps = json.loads(f.read()) | |
| except ValueError: | |
| raise Exception("ERROR: malformed dependency file") | |
| if common_deps is not None: | |
| print("Looking for dependencies on: ", | |
| dependency['target_path']) | |
| check_manifest_problems(common_deps) | |
| create_dependency_manifest(common_deps) | |
| create_common_dependencies_manifest(common_deps) | |
| def fetch_dependencies(device): | |
| location = parse_device_from_folder(device) | |
| if location is None or not os.path.isdir(location): | |
| raise Exception("ERROR: could not find your device " | |
| "folder location, bailing out") | |
| dependencies = parse_dependency_file(location) | |
| check_manifest_problems(dependencies) | |
| create_dependency_manifest(dependencies) | |
| create_common_dependencies_manifest(dependencies) | |
| fetch_device(device) | |
| def check_device_exists(device): | |
| location = parse_device_from_folder(device) | |
| if location is None: | |
| return False | |
| return os.path.isdir(location) | |
| def fetch_device(device): | |
| if check_device_exists(device): | |
| print("HINT: Avoid fetching the already checked out device repo:", device) | |
| return | |
| git_data = search_gerrit_for_device(device) | |
| if git_data is not None: | |
| device_url = git_data['id'] | |
| device_dir = parse_device_directory(device_url, device) | |
| project = create_manifest_project(device_url, | |
| device_dir, | |
| remote=default_team_rem) | |
| if project is not None: | |
| manifest = append_to_manifest(project) | |
| write_to_manifest(manifest) | |
| # In case a project was written to manifest, but never synced | |
| if project is not None or not check_target_exists(device_dir): | |
| print("syncing the device config") | |
| os.system('repo sync -f --no-clone-bundle %s' % device_dir) | |
| if __name__ == '__main__': | |
| if not os.path.isdir(local_manifest_dir): | |
| os.mkdir(local_manifest_dir) | |
| product = sys.argv[1] | |
| try: | |
| device = product[product.index("_") + 1:] | |
| except ValueError: | |
| device = product | |
| if len(sys.argv) > 2: | |
| deps_only = sys.argv[2] | |
| else: | |
| deps_only = False | |
| if not deps_only: | |
| fetch_device(device) | |
| fetch_dependencies(device) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env python | |
| # Copyright (C) 2013 Cybojenix <[email protected]> | |
| # Copyright (C) 2013 The OmniROM Project | |
| # | |
| # This program is free software: you can redistribute it and/or modify | |
| # it under the terms of the GNU General Public License as published by | |
| # the Free Software Foundation, either version 3 of the License, or | |
| # (at your option) any later version. | |
| # | |
| # This program is distributed in the hope that it will be useful, | |
| # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
| # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
| # GNU General Public License for more details. | |
| # | |
| # You should have received a copy of the GNU General Public License | |
| # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
| from __future__ import print_function | |
| import json | |
| import sys | |
| import os | |
| import os.path | |
| import re | |
| from xml.etree import ElementTree as ES | |
| # Use the urllib importer from the Cyanogenmod roomservice | |
| try: | |
| # For python3 | |
| import urllib.request | |
| except ImportError: | |
| # For python2 | |
| import imp | |
| import urllib2 | |
| urllib = imp.new_module('urllib') | |
| urllib.request = urllib2 | |
| # Config | |
| # set this to the default remote to use in repo | |
| default_rem = "omnirom" | |
| # set this to the default revision to use (branch/tag name) | |
| default_rev = "android-7.1" | |
| # set this to the remote that you use for projects from your team repos | |
| # example fetch="https://github.com/omnirom" | |
| default_team_rem = "omnirom" | |
| # this shouldn't change unless google makes changes | |
| local_manifest_dir = ".repo/local_manifests" | |
| # change this to your name on github (or equivalent hosting) | |
| android_team = "omnirom" | |
| # url to gerrit repository | |
| gerrit_url = "gerrit.omnirom.org" | |
| def check_repo_exists(git_data, device): | |
| re_match = "^android_device_.*_{device}$".format(device=device) | |
| matches = list(filter(lambda x: re.match(re_match, x), git_data)) | |
| if len(matches) != 1: | |
| raise Exception("{device} not found," | |
| "exiting roomservice".format(device=device)) | |
| return git_data[matches[0]] | |
| def search_gerrit_for_device(device): | |
| # TODO: In next gerrit release regex search with r= should be supported! | |
| git_search_url = "https://{gerrit_url}/projects/?m={device}".format( | |
| gerrit_url=gerrit_url, | |
| device=device | |
| ) | |
| git_req = urllib.request.Request(git_search_url) | |
| try: | |
| response = urllib.request.urlopen(git_req) | |
| except urllib.request.HTTPError: | |
| print("There was an issue connecting to gerrit." | |
| " Please try again in a minute") | |
| except urllib.request.URLError: | |
| print("WARNING: No network connection available.") | |
| else: | |
| # Skip silly gerrit "header" | |
| response.readline() | |
| git_data = json.load(response) | |
| device_data = check_repo_exists(git_data, device) | |
| print("found the {} device repo".format(device)) | |
| return device_data | |
| def parse_device_directory(device_url, device): | |
| pattern = "^android_device_(?P<vendor>.+)_{}$".format(device) | |
| match = re.match(pattern, device_url) | |
| if match is None: | |
| raise Exception("Invalid project name {}".format(device_url)) | |
| return "device/{vendor}/{device}".format( | |
| vendor=match.group('vendor'), | |
| device=device, | |
| ) | |
| # Thank you RaYmAn | |
| def iterate_manifests(): | |
| files = [] | |
| for file in os.listdir(local_manifest_dir): | |
| if file.endswith(".xml"): | |
| files.append(os.path.join(local_manifest_dir, file)) | |
| files.append('.repo/manifest.xml') | |
| for file in files: | |
| try: | |
| man = ES.parse(file) | |
| man = man.getroot() | |
| except (IOError, ES.ParseError): | |
| print("WARNING: error while parsing %s" % file) | |
| else: | |
| for project in man.findall("project"): | |
| yield project | |
| def iterate_manifests_remove_project(): | |
| files = [] | |
| for file in os.listdir(local_manifest_dir): | |
| if file.endswith(".xml"): | |
| files.append(os.path.join(local_manifest_dir, file)) | |
| files.append('.repo/manifest.xml') | |
| for file in files: | |
| try: | |
| man = ES.parse(file) | |
| man = man.getroot() | |
| except (IOError, ES.ParseError): | |
| print("WARNING: error while parsing %s" % file) | |
| else: | |
| for project in man.findall("remove-project"): | |
| yield project | |
| def check_project_exists(url, revision, path): | |
| for project in iterate_manifests(): | |
| if project.get("name") == url \ | |
| and project.get("revision") == revision \ | |
| and project.get("path") == path: | |
| return True | |
| return False | |
| def check_remove_project_exists(url): | |
| for project in iterate_manifests_remove_project(): | |
| if project.get("name") == url: | |
| return True | |
| return False | |
| def check_target_exists(directory): | |
| return os.path.isdir(directory) | |
| # Use the indent function from http://stackoverflow.com/a/4590052 | |
| def indent(elem, level=0): | |
| i = ''.join(["\n", level*" "]) | |
| if len(elem): | |
| if not elem.text or not elem.text.strip(): | |
| elem.text = ''.join([i, " "]) | |
| if not elem.tail or not elem.tail.strip(): | |
| elem.tail = i | |
| for elem in elem: | |
| indent(elem, level+1) | |
| if not elem.tail or not elem.tail.strip(): | |
| elem.tail = i | |
| else: | |
| if level and (not elem.tail or not elem.tail.strip()): | |
| elem.tail = i | |
| def create_manifest_project(url, directory, | |
| remote=default_rem, | |
| revision=default_rev): | |
| project_exists = check_project_exists(url, revision, directory) | |
| if project_exists: | |
| return None | |
| project = ES.Element("project", | |
| attrib={ | |
| "path": directory, | |
| "name": url, | |
| "remote": remote, | |
| "revision": revision | |
| }) | |
| return project | |
| def create_remove_project(url): | |
| remove_project_exists = check_remove_project_exists(url) | |
| if remove_project_exists: | |
| return None | |
| project = ES.Element("remove-project", | |
| attrib={ | |
| "name": url | |
| }) | |
| return project | |
| # Avoid adding already in manifests declared repositories in the roomservice.xml | |
| def is_in_manifest(project): | |
| files = [] | |
| for file in os.listdir(local_manifest_dir): | |
| if file.endswith(".xml") and file != "roomservice.xml": | |
| files.append(os.path.join(local_manifest_dir, file)) | |
| files.append('.repo/manifest.xml') | |
| for file in files: | |
| try: | |
| man = ES.parse(file) | |
| man = man.getroot() | |
| for project_in_manifests in man.findall("project"): | |
| if project_in_manifests.get("path") == project.get("path") or project_in_manifests.get("path") == project.get("target_path"): # path -> def append_to_manifest(project) & target_path -> def check_manifest_problems(dependencies) | |
| return True | |
| except (IOError, ES.ParseError): | |
| print("WARNING: error while parsing %s" % file) | |
| return False | |
| def append_to_manifest(project): | |
| try: | |
| lm = ES.parse('/'.join([local_manifest_dir, "roomservice.xml"])) | |
| lm = lm.getroot() | |
| except (IOError, ES.ParseError): | |
| lm = ES.Element("manifest") | |
| if is_in_manifest(project): | |
| print("HINT: The following repository is already defined in a manifest:", project.get("name")) | |
| return lm | |
| lm.append(project) | |
| return lm | |
| def write_to_manifest(manifest): | |
| indent(manifest) | |
| raw_xml = ES.tostring(manifest).decode() | |
| raw_xml = ''.join(['<?xml version="1.0" encoding="UTF-8"?>\n' | |
| '<!--Please do not manually edit this file-->\n', | |
| raw_xml]) | |
| with open('/'.join([local_manifest_dir, "roomservice.xml"]), 'w') as f: | |
| f.write(raw_xml) | |
| print("wrote the new roomservice manifest") | |
| def parse_device_from_manifest(device): | |
| for project in iterate_manifests(): | |
| name = project.get('name') | |
| if name.startswith("android_device_") and name.endswith(device): | |
| return project.get('path') | |
| return None | |
| def parse_device_from_folder(device): | |
| search = [] | |
| if not os.path.isdir("device"): | |
| os.mkdir("device") | |
| for sub_folder in os.listdir("device"): | |
| if os.path.isdir("device/%s/%s" % (sub_folder, device)): | |
| search.append("device/%s/%s" % (sub_folder, device)) | |
| if len(search) > 1: | |
| print("multiple devices under the name %s. " | |
| "defaulting to checking the manifest" % device) | |
| location = parse_device_from_manifest(device) | |
| elif len(search) == 1: | |
| location = search[0] | |
| else: | |
| print("Your device can't be found in device sources..") | |
| location = parse_device_from_manifest(device) | |
| return location | |
| def parse_dependency_file(location): | |
| dep_file = "omni.dependencies" | |
| dep_location = '/'.join([location, dep_file]) | |
| if not os.path.isfile(dep_location): | |
| print("WARNING: %s file not found" % dep_location) | |
| sys.exit() | |
| try: | |
| with open(dep_location, 'r') as f: | |
| dependencies = json.loads(f.read()) | |
| except ValueError: | |
| raise Exception("ERROR: malformed dependency file") | |
| return dependencies | |
| # if there is any conflict with existing and new | |
| # delete the roomservice.xml file and create new | |
| def check_manifest_problems(dependencies): | |
| for dependency in dependencies: | |
| if is_in_manifest(dependency): | |
| continue | |
| repository = dependency.get("repository") | |
| target_path = dependency.get("target_path") | |
| revision = dependency.get("revision", default_rev) | |
| # check for existing projects | |
| for project in iterate_manifests(): | |
| if project.get("revision") is not None \ | |
| and project.get("path") is not None \ | |
| and project.get("path") == target_path \ | |
| and project.get("revision") != revision: | |
| print("WARNING: detected conflict in revisions for repository ", | |
| repository) | |
| current_dependency = str(project.get(repository)) | |
| file = ES.parse('/'.join([local_manifest_dir, | |
| "roomservice.xml"])) | |
| file_root = file.getroot() | |
| for current_project in file_root.findall('project'): | |
| new_dependency = str(current_project.find('revision')) | |
| if new_dependency == current_dependency: | |
| file_root.remove(current_project) | |
| file.write('/'.join([local_manifest_dir, "roomservice.xml"])) | |
| return | |
| def create_dependency_manifest(dependencies): | |
| projects = [] | |
| for dependency in dependencies: | |
| repository = dependency.get("repository") | |
| target_path = dependency.get("target_path") | |
| revision = dependency.get("revision", default_rev) | |
| remote = dependency.get("remote", default_rem) | |
| override = dependency.get("override", None) | |
| if override is not None: | |
| #print("found override in ", repository) | |
| project = create_remove_project(repository) | |
| if project is not None: | |
| manifest = append_to_manifest(project) | |
| #print(ES.tostring(manifest).decode()) | |
| write_to_manifest(manifest) | |
| else: | |
| if override is not None: | |
| #print("found override in ", repository) | |
| project = create_remove_project(override) | |
| if project is not None: | |
| manifest = append_to_manifest(project) | |
| #print(ES.tostring(manifest).decode()) | |
| write_to_manifest(manifest) | |
| # not adding an organization should default to android_team | |
| # only apply this to github | |
| if remote == "github": | |
| if "/" not in repository: | |
| repository = '/'.join([android_team, repository]) | |
| project = create_manifest_project(repository, | |
| target_path, | |
| remote=remote, | |
| revision=revision) | |
| if project is not None: | |
| manifest = append_to_manifest(project) | |
| write_to_manifest(manifest) | |
| projects.append(target_path) | |
| if len(projects) > 0: | |
| os.system("repo sync -f --no-clone-bundle %s" % " ".join(projects)) | |
| def create_common_dependencies_manifest(dependencies): | |
| dep_file = "omni.dependencies" | |
| common_list = [] | |
| if dependencies is not None: | |
| for dependency in dependencies: | |
| try: | |
| index = common_list.index(dependency['target_path']) | |
| except ValueError: | |
| index = None | |
| if index is None: | |
| common_list.append(dependency['target_path']) | |
| dep_location = '/'.join([dependency['target_path'], dep_file]) | |
| if not os.path.isfile(dep_location): | |
| sys.exit() | |
| else: | |
| try: | |
| with open(dep_location, 'r') as f: | |
| common_deps = json.loads(f.read()) | |
| except ValueError: | |
| raise Exception("ERROR: malformed dependency file") | |
| if common_deps is not None: | |
| print("Looking for dependencies on: ", | |
| dependency['target_path']) | |
| check_manifest_problems(common_deps) | |
| create_dependency_manifest(common_deps) | |
| create_common_dependencies_manifest(common_deps) | |
| def fetch_dependencies(device): | |
| location = parse_device_from_folder(device) | |
| if location is None or not os.path.isdir(location): | |
| raise Exception("ERROR: could not find your device " | |
| "folder location, bailing out") | |
| dependencies = parse_dependency_file(location) | |
| check_manifest_problems(dependencies) | |
| create_dependency_manifest(dependencies) | |
| create_common_dependencies_manifest(dependencies) | |
| fetch_device(device) | |
| def check_device_exists(device): | |
| location = parse_device_from_folder(device) | |
| if location is None: | |
| return False | |
| return os.path.isdir(location) | |
| def fetch_device(device): | |
| if check_device_exists(device): | |
| print("HINT: Avoid fetching the already checked out device repo:", device) | |
| return | |
| git_data = search_gerrit_for_device(device) | |
| if git_data is not None: | |
| device_url = git_data['id'] | |
| device_dir = parse_device_directory(device_url, device) | |
| project = create_manifest_project(device_url, | |
| device_dir, | |
| remote=default_team_rem) | |
| if project is not None: | |
| manifest = append_to_manifest(project) | |
| write_to_manifest(manifest) | |
| # In case a project was written to manifest, but never synced | |
| if project is not None or not check_target_exists(device_dir): | |
| print("syncing the device config") | |
| os.system('repo sync -f --no-clone-bundle %s' % device_dir) | |
| if __name__ == '__main__': | |
| if not os.path.isdir(local_manifest_dir): | |
| os.mkdir(local_manifest_dir) | |
| product = sys.argv[1] | |
| try: | |
| device = product[product.index("_") + 1:] | |
| except ValueError: | |
| device = product | |
| if len(sys.argv) > 2: | |
| deps_only = sys.argv[2] | |
| else: | |
| deps_only = False | |
| if not deps_only: | |
| fetch_device(device) | |
| fetch_dependencies(device) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env python | |
| # Copyright (C) 2013 Cybojenix <[email protected]> | |
| # Copyright (C) 2013 The OmniROM Project | |
| # | |
| # This program is free software: you can redistribute it and/or modify | |
| # it under the terms of the GNU General Public License as published by | |
| # the Free Software Foundation, either version 3 of the License, or | |
| # (at your option) any later version. | |
| # | |
| # This program is distributed in the hope that it will be useful, | |
| # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
| # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
| # GNU General Public License for more details. | |
| # | |
| # You should have received a copy of the GNU General Public License | |
| # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
| from __future__ import print_function | |
| import json | |
| import sys | |
| import os | |
| import os.path | |
| import re | |
| from xml.etree import ElementTree as ES | |
| # Use the urllib importer from the Cyanogenmod roomservice | |
| try: | |
| # For python3 | |
| import urllib.request | |
| except ImportError: | |
| # For python2 | |
| import imp | |
| import urllib2 | |
| urllib = imp.new_module('urllib') | |
| urllib.request = urllib2 | |
| # Config | |
| # set this to the default remote to use in repo | |
| default_rem = "omnirom" | |
| # set this to the default revision to use (branch/tag name) | |
| default_rev = "android-8.1" | |
| # set this to the remote that you use for projects from your team repos | |
| # example fetch="https://github.com/omnirom" | |
| default_team_rem = "omnirom" | |
| # this shouldn't change unless google makes changes | |
| local_manifest_dir = ".repo/local_manifests" | |
| # change this to your name on github (or equivalent hosting) | |
| android_team = "omnirom" | |
| # url to gerrit repository | |
| gerrit_url = "gerrit.omnirom.org" | |
| def check_repo_exists(git_data, device): | |
| re_match = "^android_device_.*_{device}$".format(device=device) | |
| matches = list(filter(lambda x: re.match(re_match, x), git_data)) | |
| if len(matches) != 1: | |
| raise Exception("{device} not found," | |
| "exiting roomservice".format(device=device)) | |
| return git_data[matches[0]] | |
| def search_gerrit_for_device(device): | |
| # TODO: In next gerrit release regex search with r= should be supported! | |
| git_search_url = "https://{gerrit_url}/projects/?m={device}".format( | |
| gerrit_url=gerrit_url, | |
| device=device | |
| ) | |
| git_req = urllib.request.Request(git_search_url) | |
| try: | |
| response = urllib.request.urlopen(git_req) | |
| except urllib.request.HTTPError: | |
| print("There was an issue connecting to gerrit." | |
| " Please try again in a minute") | |
| except urllib.request.URLError: | |
| print("WARNING: No network connection available.") | |
| else: | |
| # Skip silly gerrit "header" | |
| response.readline() | |
| git_data = json.load(response) | |
| device_data = check_repo_exists(git_data, device) | |
| print("found the {} device repo".format(device)) | |
| return device_data | |
| def parse_device_directory(device_url, device): | |
| pattern = "^android_device_(?P<vendor>.+)_{}$".format(device) | |
| match = re.match(pattern, device_url) | |
| if match is None: | |
| raise Exception("Invalid project name {}".format(device_url)) | |
| return "device/{vendor}/{device}".format( | |
| vendor=match.group('vendor'), | |
| device=device, | |
| ) | |
| # Thank you RaYmAn | |
| def iterate_manifests(): | |
| files = [] | |
| for file in os.listdir(local_manifest_dir): | |
| if file.endswith(".xml"): | |
| files.append(os.path.join(local_manifest_dir, file)) | |
| files.append('.repo/manifest.xml') | |
| for file in files: | |
| try: | |
| man = ES.parse(file) | |
| man = man.getroot() | |
| except (IOError, ES.ParseError): | |
| print("WARNING: error while parsing %s" % file) | |
| else: | |
| for project in man.findall("project"): | |
| yield project | |
| def iterate_manifests_remove_project(): | |
| files = [] | |
| for file in os.listdir(local_manifest_dir): | |
| if file.endswith(".xml"): | |
| files.append(os.path.join(local_manifest_dir, file)) | |
| files.append('.repo/manifest.xml') | |
| for file in files: | |
| try: | |
| man = ES.parse(file) | |
| man = man.getroot() | |
| except (IOError, ES.ParseError): | |
| print("WARNING: error while parsing %s" % file) | |
| else: | |
| for project in man.findall("remove-project"): | |
| yield project | |
| def check_project_exists(url, revision, path): | |
| for project in iterate_manifests(): | |
| if project.get("name") == url \ | |
| and project.get("revision") == revision \ | |
| and project.get("path") == path: | |
| return True | |
| return False | |
| def check_remove_project_exists(url): | |
| for project in iterate_manifests_remove_project(): | |
| if project.get("name") == url: | |
| return True | |
| return False | |
| def check_target_exists(directory): | |
| return os.path.isdir(directory) | |
| # Use the indent function from http://stackoverflow.com/a/4590052 | |
| def indent(elem, level=0): | |
| i = ''.join(["\n", level*" "]) | |
| if len(elem): | |
| if not elem.text or not elem.text.strip(): | |
| elem.text = ''.join([i, " "]) | |
| if not elem.tail or not elem.tail.strip(): | |
| elem.tail = i | |
| for elem in elem: | |
| indent(elem, level+1) | |
| if not elem.tail or not elem.tail.strip(): | |
| elem.tail = i | |
| else: | |
| if level and (not elem.tail or not elem.tail.strip()): | |
| elem.tail = i | |
| def create_manifest_project(url, directory, | |
| remote=default_rem, | |
| revision=default_rev): | |
| project_exists = check_project_exists(url, revision, directory) | |
| if project_exists: | |
| return None | |
| project = ES.Element("project", | |
| attrib={ | |
| "path": directory, | |
| "name": url, | |
| "remote": remote, | |
| "revision": revision | |
| }) | |
| return project | |
| def create_remove_project(url): | |
| remove_project_exists = check_remove_project_exists(url) | |
| if remove_project_exists: | |
| return None | |
| project = ES.Element("remove-project", | |
| attrib={ | |
| "name": url | |
| }) | |
| return project | |
| # Avoid adding already in manifests declared repositories in the roomservice.xml | |
| def is_in_manifest(project): | |
| files = [] | |
| for file in os.listdir(local_manifest_dir): | |
| if file.endswith(".xml") and file != "roomservice.xml": | |
| files.append(os.path.join(local_manifest_dir, file)) | |
| files.append('.repo/manifest.xml') | |
| for file in files: | |
| try: | |
| man = ES.parse(file) | |
| man = man.getroot() | |
| for project_in_manifests in man.findall("project"): | |
| if project_in_manifests.get("path") == project.get("path") or project_in_manifests.get("path") == project.get("target_path"): # path -> def append_to_manifest(project) & target_path -> def check_manifest_problems(dependencies) | |
| return True | |
| except (IOError, ES.ParseError): | |
| print("WARNING: error while parsing %s" % file) | |
| return False | |
| def append_to_manifest(project): | |
| try: | |
| lm = ES.parse('/'.join([local_manifest_dir, "roomservice.xml"])) | |
| lm = lm.getroot() | |
| except (IOError, ES.ParseError): | |
| lm = ES.Element("manifest") | |
| if is_in_manifest(project): | |
| print("HINT: The following repository is already defined in a manifest:", project.get("name")) | |
| return lm | |
| lm.append(project) | |
| return lm | |
| def write_to_manifest(manifest): | |
| indent(manifest) | |
| raw_xml = ES.tostring(manifest).decode() | |
| raw_xml = ''.join(['<?xml version="1.0" encoding="UTF-8"?>\n' | |
| '<!--Please do not manually edit this file-->\n', | |
| raw_xml]) | |
| with open('/'.join([local_manifest_dir, "roomservice.xml"]), 'w') as f: | |
| f.write(raw_xml) | |
| print("wrote the new roomservice manifest") | |
| def parse_device_from_manifest(device): | |
| for project in iterate_manifests(): | |
| name = project.get('name') | |
| if name.startswith("android_device_") and name.endswith(device): | |
| return project.get('path') | |
| return None | |
| def parse_device_from_folder(device): | |
| search = [] | |
| if not os.path.isdir("device"): | |
| os.mkdir("device") | |
| for sub_folder in os.listdir("device"): | |
| if os.path.isdir("device/%s/%s" % (sub_folder, device)): | |
| search.append("device/%s/%s" % (sub_folder, device)) | |
| if len(search) > 1: | |
| print("multiple devices under the name %s. " | |
| "defaulting to checking the manifest" % device) | |
| location = parse_device_from_manifest(device) | |
| elif len(search) == 1: | |
| location = search[0] | |
| else: | |
| print("Your device can't be found in device sources..") | |
| location = parse_device_from_manifest(device) | |
| return location | |
| def parse_dependency_file(location): | |
| dep_file = "omni.dependencies" | |
| dep_location = '/'.join([location, dep_file]) | |
| if not os.path.isfile(dep_location): | |
| print("WARNING: %s file not found" % dep_location) | |
| sys.exit() | |
| try: | |
| with open(dep_location, 'r') as f: | |
| dependencies = json.loads(f.read()) | |
| except ValueError: | |
| raise Exception("ERROR: malformed dependency file") | |
| return dependencies | |
| # if there is any conflict with existing and new | |
| # delete the roomservice.xml file and create new | |
| def check_manifest_problems(dependencies): | |
| for dependency in dependencies: | |
| if is_in_manifest(dependency): | |
| continue | |
| repository = dependency.get("repository") | |
| target_path = dependency.get("target_path") | |
| revision = dependency.get("revision", default_rev) | |
| # check for existing projects | |
| for project in iterate_manifests(): | |
| if project.get("revision") is not None \ | |
| and project.get("path") is not None \ | |
| and project.get("path") == target_path \ | |
| and project.get("revision") != revision: | |
| print("WARNING: detected conflict in revisions for repository ", | |
| repository) | |
| current_dependency = str(project.get(repository)) | |
| file = ES.parse('/'.join([local_manifest_dir, | |
| "roomservice.xml"])) | |
| file_root = file.getroot() | |
| for current_project in file_root.findall('project'): | |
| new_dependency = str(current_project.find('revision')) | |
| if new_dependency == current_dependency: | |
| file_root.remove(current_project) | |
| file.write('/'.join([local_manifest_dir, "roomservice.xml"])) | |
| return | |
| def create_dependency_manifest(dependencies): | |
| projects = [] | |
| for dependency in dependencies: | |
| repository = dependency.get("repository") | |
| target_path = dependency.get("target_path") | |
| revision = dependency.get("revision", default_rev) | |
| remote = dependency.get("remote", default_rem) | |
| override = dependency.get("override", None) | |
| if override is not None: | |
| #print("found override in ", repository) | |
| project = create_remove_project(repository) | |
| if project is not None: | |
| manifest = append_to_manifest(project) | |
| #print(ES.tostring(manifest).decode()) | |
| write_to_manifest(manifest) | |
| else: | |
| if override is not None: | |
| #print("found override in ", repository) | |
| project = create_remove_project(override) | |
| if project is not None: | |
| manifest = append_to_manifest(project) | |
| #print(ES.tostring(manifest).decode()) | |
| write_to_manifest(manifest) | |
| # not adding an organization should default to android_team | |
| # only apply this to github | |
| if remote == "github": | |
| if "/" not in repository: | |
| repository = '/'.join([android_team, repository]) | |
| project = create_manifest_project(repository, | |
| target_path, | |
| remote=remote, | |
| revision=revision) | |
| if project is not None: | |
| manifest = append_to_manifest(project) | |
| write_to_manifest(manifest) | |
| projects.append(target_path) | |
| if len(projects) > 0: | |
| os.system("repo sync -f --no-clone-bundle %s" % " ".join(projects)) | |
| def create_common_dependencies_manifest(dependencies): | |
| dep_file = "omni.dependencies" | |
| common_list = [] | |
| if dependencies is not None: | |
| for dependency in dependencies: | |
| try: | |
| index = common_list.index(dependency['target_path']) | |
| except ValueError: | |
| index = None | |
| if index is None: | |
| common_list.append(dependency['target_path']) | |
| dep_location = '/'.join([dependency['target_path'], dep_file]) | |
| if not os.path.isfile(dep_location): | |
| sys.exit() | |
| else: | |
| try: | |
| with open(dep_location, 'r') as f: | |
| common_deps = json.loads(f.read()) | |
| except ValueError: | |
| raise Exception("ERROR: malformed dependency file") | |
| if common_deps is not None: | |
| print("Looking for dependencies on: ", | |
| dependency['target_path']) | |
| check_manifest_problems(common_deps) | |
| create_dependency_manifest(common_deps) | |
| create_common_dependencies_manifest(common_deps) | |
| def fetch_dependencies(device): | |
| location = parse_device_from_folder(device) | |
| if location is None or not os.path.isdir(location): | |
| raise Exception("ERROR: could not find your device " | |
| "folder location, bailing out") | |
| dependencies = parse_dependency_file(location) | |
| check_manifest_problems(dependencies) | |
| create_dependency_manifest(dependencies) | |
| create_common_dependencies_manifest(dependencies) | |
| fetch_device(device) | |
| def check_device_exists(device): | |
| location = parse_device_from_folder(device) | |
| if location is None: | |
| return False | |
| return os.path.isdir(location) | |
| def fetch_device(device): | |
| if check_device_exists(device): | |
| print("HINT: Avoid fetching the already checked out device repo:", device) | |
| return | |
| git_data = search_gerrit_for_device(device) | |
| if git_data is not None: | |
| device_url = git_data['id'] | |
| device_dir = parse_device_directory(device_url, device) | |
| project = create_manifest_project(device_url, | |
| device_dir, | |
| remote=default_team_rem) | |
| if project is not None: | |
| manifest = append_to_manifest(project) | |
| write_to_manifest(manifest) | |
| # In case a project was written to manifest, but never synced | |
| if project is not None or not check_target_exists(device_dir): | |
| print("syncing the device config") | |
| os.system('repo sync -f --no-clone-bundle %s' % device_dir) | |
| if __name__ == '__main__': | |
| if not os.path.isdir(local_manifest_dir): | |
| os.mkdir(local_manifest_dir) | |
| product = sys.argv[1] | |
| try: | |
| device = product[product.index("_") + 1:] | |
| except ValueError: | |
| device = product | |
| if len(sys.argv) > 2: | |
| deps_only = sys.argv[2] | |
| else: | |
| deps_only = False | |
| if not deps_only: | |
| fetch_device(device) | |
| fetch_dependencies(device) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env python | |
| # Copyright (C) 2013 Cybojenix <[email protected]> | |
| # Copyright (C) 2013 The OmniROM Project | |
| # | |
| # This program is free software: you can redistribute it and/or modify | |
| # it under the terms of the GNU General Public License as published by | |
| # the Free Software Foundation, either version 3 of the License, or | |
| # (at your option) any later version. | |
| # | |
| # This program is distributed in the hope that it will be useful, | |
| # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
| # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
| # GNU General Public License for more details. | |
| # | |
| # You should have received a copy of the GNU General Public License | |
| # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
| from __future__ import print_function | |
| import json | |
| import sys | |
| import os | |
| import os.path | |
| import re | |
| from xml.etree import ElementTree as ES | |
| # Use the urllib importer from the Cyanogenmod roomservice | |
| try: | |
| # For python3 | |
| import urllib.request | |
| except ImportError: | |
| # For python2 | |
| import imp | |
| import urllib2 | |
| urllib = imp.new_module('urllib') | |
| urllib.request = urllib2 | |
| # Config | |
| # set this to the default remote to use in repo | |
| default_rem = "omnirom" | |
| # set this to the default revision to use (branch/tag name) | |
| default_rev = "android-9.0" | |
| # set this to the remote that you use for projects from your team repos | |
| # example fetch="https://github.com/omnirom" | |
| default_team_rem = "omnirom" | |
| # this shouldn't change unless google makes changes | |
| local_manifest_dir = ".repo/local_manifests" | |
| # change this to your name on github (or equivalent hosting) | |
| android_team = "omnirom" | |
| # url to gerrit repository | |
| gerrit_url = "gerrit.omnirom.org" | |
| def check_repo_exists(git_data, device): | |
| re_match = "^android_device_.*_{device}$".format(device=device) | |
| matches = list(filter(lambda x: re.match(re_match, x), git_data)) | |
| if len(matches) != 1: | |
| raise Exception("{device} not found," | |
| "exiting roomservice".format(device=device)) | |
| return git_data[matches[0]] | |
| def search_gerrit_for_device(device): | |
| # TODO: In next gerrit release regex search with r= should be supported! | |
| git_search_url = "https://{gerrit_url}/projects/?m={device}".format( | |
| gerrit_url=gerrit_url, | |
| device=device | |
| ) | |
| git_req = urllib.request.Request(git_search_url) | |
| try: | |
| response = urllib.request.urlopen(git_req) | |
| except urllib.request.HTTPError: | |
| print("There was an issue connecting to gerrit." | |
| " Please try again in a minute") | |
| except urllib.request.URLError: | |
| print("WARNING: No network connection available.") | |
| else: | |
| # Skip silly gerrit "header" | |
| response.readline() | |
| git_data = json.load(response) | |
| device_data = check_repo_exists(git_data, device) | |
| print("found the {} device repo".format(device)) | |
| return device_data | |
| def parse_device_directory(device_url, device): | |
| pattern = "^android_device_(?P<vendor>.+)_{}$".format(device) | |
| match = re.match(pattern, device_url) | |
| if match is None: | |
| raise Exception("Invalid project name {}".format(device_url)) | |
| return "device/{vendor}/{device}".format( | |
| vendor=match.group('vendor'), | |
| device=device, | |
| ) | |
| # Thank you RaYmAn | |
| def iterate_manifests(): | |
| files = [] | |
| for file in os.listdir(local_manifest_dir): | |
| if file.endswith(".xml"): | |
| files.append(os.path.join(local_manifest_dir, file)) | |
| files.append('.repo/manifest.xml') | |
| for file in files: | |
| try: | |
| man = ES.parse(file) | |
| man = man.getroot() | |
| except (IOError, ES.ParseError): | |
| print("WARNING: error while parsing %s" % file) | |
| else: | |
| for project in man.findall("project"): | |
| yield project | |
| def iterate_manifests_remove_project(): | |
| files = [] | |
| for file in os.listdir(local_manifest_dir): | |
| if file.endswith(".xml"): | |
| files.append(os.path.join(local_manifest_dir, file)) | |
| files.append('.repo/manifest.xml') | |
| for file in files: | |
| try: | |
| man = ES.parse(file) | |
| man = man.getroot() | |
| except (IOError, ES.ParseError): | |
| print("WARNING: error while parsing %s" % file) | |
| else: | |
| for project in man.findall("remove-project"): | |
| yield project | |
| def check_project_exists(url, revision, path): | |
| for project in iterate_manifests(): | |
| if project.get("name") == url \ | |
| and project.get("revision") == revision \ | |
| and project.get("path") == path: | |
| return True | |
| return False | |
| def check_remove_project_exists(url): | |
| for project in iterate_manifests_remove_project(): | |
| if project.get("name") == url: | |
| return True | |
| return False | |
| def check_target_exists(directory): | |
| return os.path.isdir(directory) | |
| # Use the indent function from http://stackoverflow.com/a/4590052 | |
| def indent(elem, level=0): | |
| i = ''.join(["\n", level*" "]) | |
| if len(elem): | |
| if not elem.text or not elem.text.strip(): | |
| elem.text = ''.join([i, " "]) | |
| if not elem.tail or not elem.tail.strip(): | |
| elem.tail = i | |
| for elem in elem: | |
| indent(elem, level+1) | |
| if not elem.tail or not elem.tail.strip(): | |
| elem.tail = i | |
| else: | |
| if level and (not elem.tail or not elem.tail.strip()): | |
| elem.tail = i | |
| def create_manifest_project(url, directory, | |
| remote=default_rem, | |
| revision=default_rev): | |
| project_exists = check_project_exists(url, revision, directory) | |
| if project_exists: | |
| return None | |
| project = ES.Element("project", | |
| attrib={ | |
| "path": directory, | |
| "name": url, | |
| "remote": remote, | |
| "revision": revision | |
| }) | |
| return project | |
| def create_remove_project(url): | |
| remove_project_exists = check_remove_project_exists(url) | |
| if remove_project_exists: | |
| return None | |
| project = ES.Element("remove-project", | |
| attrib={ | |
| "name": url | |
| }) | |
| return project | |
| # Avoid adding already in manifests declared repositories in the roomservice.xml | |
| def is_in_manifest(project): | |
| files = [] | |
| for file in os.listdir(local_manifest_dir): | |
| if file.endswith(".xml") and file != "roomservice.xml": | |
| files.append(os.path.join(local_manifest_dir, file)) | |
| files.append('.repo/manifest.xml') | |
| for file in files: | |
| try: | |
| man = ES.parse(file) | |
| man = man.getroot() | |
| for project_in_manifests in man.findall("project"): | |
| if project_in_manifests.get("path") == project.get("path") or project_in_manifests.get("path") == project.get("target_path"): # path -> def append_to_manifest(project) & target_path -> def check_manifest_problems(dependencies) | |
| return True | |
| except (IOError, ES.ParseError): | |
| print("WARNING: error while parsing %s" % file) | |
| return False | |
| def append_to_manifest(project): | |
| try: | |
| lm = ES.parse('/'.join([local_manifest_dir, "roomservice.xml"])) | |
| lm = lm.getroot() | |
| except (IOError, ES.ParseError): | |
| lm = ES.Element("manifest") | |
| if is_in_manifest(project): | |
| print("HINT: The following repository is already defined in a manifest:", project.get("name")) | |
| return lm | |
| lm.append(project) | |
| return lm | |
| def write_to_manifest(manifest): | |
| indent(manifest) | |
| raw_xml = ES.tostring(manifest).decode() | |
| raw_xml = ''.join(['<?xml version="1.0" encoding="UTF-8"?>\n' | |
| '<!--Please do not manually edit this file-->\n', | |
| raw_xml]) | |
| with open('/'.join([local_manifest_dir, "roomservice.xml"]), 'w') as f: | |
| f.write(raw_xml) | |
| print("wrote the new roomservice manifest") | |
| def parse_device_from_manifest(device): | |
| for project in iterate_manifests(): | |
| name = project.get('name') | |
| if name.startswith("android_device_") and name.endswith(device): | |
| return project.get('path') | |
| return None | |
| def parse_device_from_folder(device): | |
| search = [] | |
| if not os.path.isdir("device"): | |
| os.mkdir("device") | |
| for sub_folder in os.listdir("device"): | |
| if os.path.isdir("device/%s/%s" % (sub_folder, device)): | |
| search.append("device/%s/%s" % (sub_folder, device)) | |
| if len(search) > 1: | |
| print("multiple devices under the name %s. " | |
| "defaulting to checking the manifest" % device) | |
| location = parse_device_from_manifest(device) | |
| elif len(search) == 1: | |
| location = search[0] | |
| else: | |
| print("Your device can't be found in device sources..") | |
| location = parse_device_from_manifest(device) | |
| return location | |
| def parse_dependency_file(location): | |
| dep_file = "omni.dependencies" | |
| dep_location = '/'.join([location, dep_file]) | |
| if not os.path.isfile(dep_location): | |
| print("WARNING: %s file not found" % dep_location) | |
| sys.exit() | |
| try: | |
| with open(dep_location, 'r') as f: | |
| dependencies = json.loads(f.read()) | |
| except ValueError: | |
| raise Exception("ERROR: malformed dependency file") | |
| return dependencies | |
| # if there is any conflict with existing and new | |
| # delete the roomservice.xml file and create new | |
| def check_manifest_problems(dependencies): | |
| for dependency in dependencies: | |
| if is_in_manifest(dependency): | |
| continue | |
| repository = dependency.get("repository") | |
| target_path = dependency.get("target_path") | |
| revision = dependency.get("revision", default_rev) | |
| # check for existing projects | |
| for project in iterate_manifests(): | |
| if project.get("revision") is not None \ | |
| and project.get("path") is not None \ | |
| and project.get("path") == target_path \ | |
| and project.get("revision") != revision: | |
| print("WARNING: detected conflict in revisions for repository ", | |
| repository) | |
| current_dependency = str(project.get(repository)) | |
| file = ES.parse('/'.join([local_manifest_dir, | |
| "roomservice.xml"])) | |
| file_root = file.getroot() | |
| for current_project in file_root.findall('project'): | |
| new_dependency = str(current_project.find('revision')) | |
| if new_dependency == current_dependency: | |
| file_root.remove(current_project) | |
| file.write('/'.join([local_manifest_dir, "roomservice.xml"])) | |
| return | |
| def create_dependency_manifest(dependencies): | |
| projects = [] | |
| for dependency in dependencies: | |
| repository = dependency.get("repository") | |
| target_path = dependency.get("target_path") | |
| revision = dependency.get("revision", default_rev) | |
| remote = dependency.get("remote", default_rem) | |
| override = dependency.get("override", None) | |
| remove = dependency.get("remove", None) | |
| if remove is not None: | |
| #print("found remove in ", repository) | |
| project = create_remove_project(remove) | |
| if project is not None: | |
| manifest = append_to_manifest(project) | |
| #print(ES.tostring(manifest).decode()) | |
| write_to_manifest(manifest) | |
| else: | |
| if override is not None: | |
| #print("found override in ", repository) | |
| project = create_remove_project(override) | |
| if project is not None: | |
| manifest = append_to_manifest(project) | |
| #print(ES.tostring(manifest).decode()) | |
| write_to_manifest(manifest) | |
| # not adding an organization should default to android_team | |
| # only apply this to github | |
| if remote == "github": | |
| if "/" not in repository: | |
| repository = '/'.join([android_team, repository]) | |
| project = create_manifest_project(repository, | |
| target_path, | |
| remote=remote, | |
| revision=revision) | |
| if project is not None: | |
| manifest = append_to_manifest(project) | |
| write_to_manifest(manifest) | |
| projects.append(target_path) | |
| if len(projects) > 0: | |
| os.system("repo sync -f --no-clone-bundle %s" % " ".join(projects)) | |
| def create_common_dependencies_manifest(dependencies): | |
| dep_file = "omni.dependencies" | |
| common_list = [] | |
| if dependencies is not None: | |
| for dependency in dependencies: | |
| try: | |
| index = common_list.index(dependency['target_path']) | |
| except ValueError: | |
| index = None | |
| if index is None: | |
| common_list.append(dependency['target_path']) | |
| dep_location = '/'.join([dependency['target_path'], dep_file]) | |
| if not os.path.isfile(dep_location): | |
| sys.exit() | |
| else: | |
| try: | |
| with open(dep_location, 'r') as f: | |
| common_deps = json.loads(f.read()) | |
| except ValueError: | |
| raise Exception("ERROR: malformed dependency file") | |
| if common_deps is not None: | |
| print("Looking for dependencies on: ", | |
| dependency['target_path']) | |
| check_manifest_problems(common_deps) | |
| create_dependency_manifest(common_deps) | |
| create_common_dependencies_manifest(common_deps) | |
| def fetch_dependencies(device): | |
| if device == "emulator": | |
| location = "vendor/omni/utils/emulator" | |
| else: | |
| location = parse_device_from_folder(device) | |
| if location is None or not os.path.isdir(location): | |
| raise Exception("ERROR: could not find your device " | |
| "folder location, bailing out") | |
| dependencies = parse_dependency_file(location) | |
| check_manifest_problems(dependencies) | |
| create_dependency_manifest(dependencies) | |
| create_common_dependencies_manifest(dependencies) | |
| fetch_device(device) | |
| def check_device_exists(device): | |
| location = parse_device_from_folder(device) | |
| if location is None: | |
| return False | |
| return os.path.isdir(location) | |
| def fetch_device(device): | |
| if check_device_exists(device): | |
| print("HINT: Avoid fetching the already checked out device repo:", device) | |
| return | |
| git_data = search_gerrit_for_device(device) | |
| if git_data is not None: | |
| device_url = git_data['id'] | |
| device_dir = parse_device_directory(device_url, device) | |
| project = create_manifest_project(device_url, | |
| device_dir, | |
| remote=default_team_rem) | |
| if project is not None: | |
| manifest = append_to_manifest(project) | |
| write_to_manifest(manifest) | |
| # In case a project was written to manifest, but never synced | |
| if project is not None or not check_target_exists(device_dir): | |
| print("syncing the device config") | |
| os.system('repo sync -f --no-clone-bundle %s' % device_dir) | |
| if __name__ == '__main__': | |
| if not os.path.isdir(local_manifest_dir): | |
| os.mkdir(local_manifest_dir) | |
| product = sys.argv[1] | |
| try: | |
| device = product[product.index("_") + 1:] | |
| except ValueError: | |
| device = product | |
| if len(sys.argv) > 2: | |
| deps_only = sys.argv[2] | |
| else: | |
| deps_only = False | |
| if not deps_only: | |
| fetch_device(device) | |
| fetch_dependencies(device) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment