TeamCity-aggregator

comparison.py

#!/bin/python3
import os
import re
import collections

configs = set()
buildNumbers = set()
totalTimeTable = collections.defaultdict(dict)
buildTxsTable = collections.defaultdict(dict)
txsPerSecTable = collections.defaultdict(dict)
commentsTable = collections.defaultdict(dict)

for root, dirs, files in os.walk('data'):
	for i, file in enumerate(files):
		if file[-8:] == '.comment':
			delim = file.find('.')
			buildNumber = file[:delim]
			with open('data/' + file, 'r') as f:
				commentsTable[int(buildNumber)] = f.read()
		if file[-4:] != '.log':
			continue
		delim = file.find('-')
		buildNumber = file[:delim]
		config = file[delim+1:-4]
		print(buildNumber + ' ' + config)
		
		with open('data/' + file, 'r') as f:
			content = f.read()
			x = re.search('Total elapsed time: ([0-9\.]*) s, processed ([0-9\.]*) blocks \(~ ([0-9\.]*) Tx/s\)', content)
			if x == None:
				print('unrecognized total time')
				continue
			print(x.group(0))
			totalTime = x.group(1)
			txsCount = x.group(2)
			txsPerSec = x.group(3)
			configs.add(config)
			buildNumbers.add(int(buildNumber))
			totalTimeTable[config][int(buildNumber)] = totalTime
			buildTxsTable[int(buildNumber)] = txsCount
			txsPerSecTable[config][int(buildNumber)] = txsPerSec

buildNumbersSorted = sorted(buildNumbers)
configsSorted = sorted(configs)

with open('totalTime.csv', 'w') as f:
	f.write("build;txsCount;comment;" + ";".join(configsSorted) + "\n")
	for buildNumber in buildNumbersSorted:
		f.write(str(buildNumber) + ";" + buildTxsTable[buildNumber] + ";")
		if buildNumber in commentsTable:
			f.write(commentsTable[buildNumber])
		for config in configsSorted:
			try:
				f.write(";" + totalTimeTable[config][buildNumber])
			except KeyError:
				f.write(";")
		f.write("\n")

with open('txsPerSec.csv', 'w') as f:
	f.write("build;txsCount;comment;" + ";".join(configsSorted) + "\n")
	for buildNumber in buildNumbersSorted:
		f.write(str(buildNumber) + ";" + buildTxsTable[buildNumber] + ";")
		if buildNumber in commentsTable:
			f.write(commentsTable[buildNumber])
		for config in configsSorted:
			try:
				f.write(";" + txsPerSecTable[config][buildNumber])
			except KeyError:
				f.write(";")
		f.write("\n")

download.py

#!/bin/python3
import requests
import os
import xmltodict

# Downloads all artifacts from the build configuration in TeamCity

# Fill your Access Token (Profile - Access Tokens) here: (this is only demo token)
headers = {
    'Authorization': 'Bearer eyJ0eXAiOiAiVENWMiJ9.NVZpX3NSZENTZ2ZGYUFVVUxfcm5GWkhaV1JR.OTZkYWYzNDctZDg0Yi00MzZhLWI0MmMtZDRmOWY3YjZkZWI4',
    'Content-Type': 'application/xml'
}

response = requests.get('https://team.fantom.network/app/rest/builds/?locator=buildType:Aida_RunVmComparison&fields=build(id,number,status,running,startDate,comment,artifacts(file(name,content)))', headers=headers)

if response.status_code != 200:
	print(response.content)
	quit(1)

dict_data = xmltodict.parse(response.content)

for i, build in enumerate(dict_data['builds']['build']):
	print(build['@number'])
	
	if 'comment' in build:
		print(build['comment']['text'])
		with open('data/' + build['@number'] + '.comment', 'w') as f:
			f.write(build['comment']['text'])
	
	for j, artifact in enumerate(build['artifacts']['file']):
		name = build['@number'] + '-' + artifact['@name']
		print(' * ' + name + ': ' + artifact['content']['@href'])
		
		if os.path.exists('data/' + name):
			continue
		
		response = requests.get('https://team.fantom.network' + artifact['content']['@href'], headers=headers)
		with open('data/' + name, 'wb') as f:
			f.write(response.content)

print('Downloading complete')