-
Notifications
You must be signed in to change notification settings - Fork 0
/
repo.py
210 lines (170 loc) · 9.08 KB
/
repo.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
import urllib.request
import os
import xml.etree.ElementTree as ET
from urllib.parse import urlparse
aosp_manifest_url_base = 'https://raw.githubusercontent.com/android/platform_manifest/{}/default.xml'
aosp_git_base = 'https://android.googlesource.com/'
def read_file(path):
with open(path) as file:
return file.read()
def download_url(url):
print(url)
response = urllib.request.urlopen(url)
data = response.read()
text = data.decode('utf-8')
return text
def normalize_manifest_fetch_url(current_url, url):
if url[:2] == '..':
parsed_uri = urlparse(current_url)
domain = '{uri.scheme}://{uri.netloc}/'.format(uri=parsed_uri)
return domain + url[2:]
if url[-1:] != '/':
url = url + '/'
return url
def get_target_repositories(manifest_text, target_remote=None, fetch_includes=True, manifest_url=None):
target_repositories = dict()
if manifest_text is None or manifest_text == '':
manifest_text = download_url(manifest_url)
root = ET.fromstring(manifest_text)
if fetch_includes and root.find('include') is not None:
for include in root.findall('include'):
included_manifest_url = manifest_url[:manifest_url.rfind('/') + 1] + include.get('name')
included_repos = get_target_repositories(None, target_remote, True, included_manifest_url)
target_repositories = {**target_repositories, **included_repos}
for repository in root.findall('project'):
if target_remote is None and 'remote' in repository.keys():
continue
if target_remote is not None:
if 'remote' not in repository.keys() or repository.get('remote') != target_remote:
continue
target_repositories[repository.get('path')] = dict()
for attr in repository.keys():
if attr != 'path':
target_repositories[repository.get('path')][attr] = repository.get(attr)
return target_repositories
def get_version_manifest(manifest_url, project_git_host, aosp_remote_name=None, proprietary_remote_name=None):
version_manifest = dict()
manifest_text = download_url(manifest_url)
root = ET.fromstring(manifest_text)
remote_default = root.find('default')
all_remotes = dict()
for remote in root.findall('remote'):
all_remotes[remote.get('name')] = remote
if aosp_remote_name is None:
aosp_remote = remote_default
else:
aosp_remote = root.find('.//remote[@name="{}"]'.format(aosp_remote_name))
if proprietary_remote_name is None:
proprietary_remote = remote_default
else:
proprietary_remote = root.find('.//remote[@name="{}"]'.format(proprietary_remote_name))
aosp_branch = aosp_remote.get('revision')
if aosp_branch is None:
proprietary_aosp_repos = get_target_repositories(manifest_text, aosp_remote_name, False)
aosp_branch = proprietary_aosp_repos[list(proprietary_aosp_repos.keys())[0]]['revision']
# Strip branch name.
aosp_branch = aosp_branch[aosp_branch.rfind('/') + 1:]
aosp_manifest_url = aosp_manifest_url_base.format(aosp_branch)
aosp_repos = get_target_repositories(None, None, True, aosp_manifest_url)
proprietary_branch = proprietary_remote.get('revision')
if proprietary_branch is None:
proprietary_repos = get_target_repositories(manifest_text, proprietary_remote_name, False)
proprietary_branch = proprietary_repos[list(proprietary_repos.keys())[0]]['revision']
# Strip branch name.
proprietary_branch = proprietary_branch[proprietary_branch.rfind('/') + 1:]
proprietary_base_git_url = proprietary_remote.get('fetch')
while proprietary_base_git_url is None:
proprietary_remote = all_remotes[proprietary_remote.get('remote')]
proprietary_base_git_url = proprietary_remote.get('fetch')
proprietary_base_git_url = normalize_manifest_fetch_url(project_git_host, proprietary_base_git_url)
proprietary_repos = get_target_repositories(manifest_text, proprietary_remote_name, True, manifest_url)
version_manifest['aosp_branch'] = aosp_branch
version_manifest['aosp_repos'] = aosp_repos
version_manifest['proprietary_branch'] = proprietary_branch
version_manifest['proprietary_base_git_url'] = proprietary_base_git_url
version_manifest['proprietary_repos'] = proprietary_repos
return version_manifest
def parse_project_configs(config_path):
config_text = read_file(config_path)
config_root = ET.fromstring(config_text)
project_configs = list()
for project in config_root.findall('project'):
project_dict = dict()
project_dict['name'] = project.get('name')
project_dict['git_host'] = project.get('git_host')
project_versions = list()
for version in project.findall('version'):
version_config = dict()
version_config['branch_name'] = version.get('branch_name')
version_config['manifest_url'] = version.get('manifest_url')
version_config['aosp_remote_name'] = version.get('aosp_remote_name')
if version.get('aosp_remote_name').strip() == '':
version_config['aosp_remote_name'] = None
version_config['proprietary_remote_name'] = version.get('proprietary_remote_name')
if version.get('proprietary_remote_name').strip() == '':
version_config['proprietary_remote_name'] = None
project_versions.append(version_config)
project_dict['versions'] = project_versions
project_configs.append(project_dict)
return project_configs
def get_project_manifests(config_path):
project_configs = parse_project_configs(config_path)
project_manifests = dict()
for project_config in project_configs:
project_name = project_config['name']
project_git_host = project_config['git_host']
project_manifest = dict()
for version_config in project_config['versions']:
version_manifest = get_version_manifest(version_config['manifest_url'], project_git_host,
version_config['aosp_remote_name'],
version_config['proprietary_remote_name'])
project_manifest[version_config['branch_name']] = version_manifest
project_manifests[project_name] = project_manifest
return project_configs, project_manifests
def extract_mutual_repos_single_version(version_manifest):
mutual_repos = set()
for proprietary_repo in version_manifest['proprietary_repos']:
if proprietary_repo in version_manifest['aosp_repos']:
mutual_repos.add(proprietary_repo)
return mutual_repos
def extract_mutual_repos_all_versions(project_manifest):
mutual_repos_single_version_list = list()
for version_name, version_manifest in project_manifest.items():
mutual_repos_single_version_list.append(extract_mutual_repos_single_version(version_manifest))
return set.intersection(*mutual_repos_single_version_list)
def get_first_release_for_aosp_branch(aosp_branch):
return aosp_branch[:aosp_branch.rfind('_')] + '_r1'
def write_file(content, path):
if '/' in path:
directory = path[:path.rfind('/')]
if not os.path.exists(directory):
os.makedirs(directory)
with open(path, 'w+') as file:
file.write(content)
def run(config_path):
project_configs, project_manifests = get_project_manifests(config_path)
for project_config in project_configs:
project_name = project_config['name']
project_manifest = project_manifests[project_name]
for version_index in range(len(project_config['versions'])):
output = ''
version_name = project_config['versions'][version_index]['branch_name']
version_manifest = project_manifest[version_name]
aosp_branch = version_manifest['aosp_branch']
next_aosp_branch = 'PLEASE_REPLACE_ME_WITH_ANDROID_NEW_VERSION_FOR_THIS_COMPARISON_SCENARIO'
if version_index < len(project_config['versions']) - 1:
next_version_name = project_config['versions'][version_index + 1]['branch_name']
next_version_manifest = project_manifest[next_version_name]
next_aosp_branch = get_first_release_for_aosp_branch(next_version_manifest['aosp_branch'])
output += 'versions:{},{},{}'.format(aosp_branch, next_aosp_branch, version_name)
output += '\n'
mutual_repos = extract_mutual_repos_single_version(version_manifest)
for mutual_repo in mutual_repos:
aosp_git_url = aosp_git_base + version_manifest['aosp_repos'][mutual_repo]['name']
proprietary_git_url = version_manifest['proprietary_base_git_url'] + \
version_manifest['proprietary_repos'][mutual_repo]['name']
repo_name = mutual_repo.replace('/', '_')
output += '{},{},{}'.format(repo_name, aosp_git_url, proprietary_git_url)
output += '\n'
write_file(output, '{}/{}_{}_{}.csv'.format(project_name, aosp_branch, next_aosp_branch, version_name))
run('repos_config.xml')