Created
March 9, 2020 16:29
-
-
Save i-b1/131c6f65f0255ca7202207e25f24f164 to your computer and use it in GitHub Desktop.
Azure DevOps API - Repositories Backup
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
using System; | |
using System.Collections.Generic; | |
using System.IO; | |
using System.IO.Compression; | |
using System.Linq; | |
using System.Threading.Tasks; | |
using Microsoft.Azure.WebJobs; | |
using Microsoft.Extensions.Logging; | |
using Microsoft.WindowsAzure.Storage; | |
using Newtonsoft.Json; | |
using RestSharp; | |
namespace AzureDevopsBackupFunction | |
{ | |
public static class BackupFunction | |
{ | |
private const string version = "api-version=5.1"; | |
[FunctionName("BackupFunction")] | |
public static async Task Run([TimerTrigger("0 0 20 * * *")]TimerInfo myTimer, ILogger log) //, RunOnStartup = true | |
{ | |
log.LogInformation($"DevOps BackupFunction function starting execution at: {DateTime.Now}"); | |
// configure connections | |
string storageAccountKey = Environment.GetEnvironmentVariable("storageAccountKey", EnvironmentVariableTarget.Process); | |
string storageName = Environment.GetEnvironmentVariable("storageName", EnvironmentVariableTarget.Process); | |
string token = Environment.GetEnvironmentVariable("token", EnvironmentVariableTarget.Process); | |
string organization = Environment.GetEnvironmentVariable("organization", EnvironmentVariableTarget.Process); | |
string storageConnection = $"DefaultEndpointsProtocol=https;AccountName={storageName};AccountKey={storageAccountKey};EndpointSuffix=core.windows.net"; | |
string devopsURL = $"https://dev.azure.com/{organization}/"; | |
// make API request to get all projects | |
string auth = "Basic " + Convert.ToBase64String(System.Text.Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", token))); | |
var clientProjects = new RestClient($"{devopsURL}_apis/projects?{version}"); | |
var requestProjects = new RestRequest(Method.GET); | |
requestProjects.AddHeader("Authorization", auth); | |
var responseProjects = clientProjects.Execute(requestProjects); | |
if(responseProjects.StatusCode != System.Net.HttpStatusCode.OK) | |
{ | |
throw new Exception("API Request failed: " + responseProjects.StatusCode + " " + responseProjects.ErrorMessage); | |
} | |
Projects projects = JsonConvert.DeserializeObject<Projects>(responseProjects.Content); | |
// connect to Azure Storage | |
var storageAccount = CloudStorageAccount.Parse(storageConnection); | |
var client = storageAccount.CreateCloudBlobClient(); | |
var container = client.GetContainerReference("devopsbackup"); | |
await container.CreateIfNotExistsAsync(); | |
foreach (Project project in projects.value) | |
{ | |
log.LogInformation(project.name); | |
// get repositories | |
var clientRepos = new RestClient($"{devopsURL}{project.name}/_apis/git/repositories?{version}"); | |
var requestRepos = new RestRequest(Method.GET); | |
requestRepos.AddHeader("Authorization", auth); | |
var responseRepos = clientRepos.Execute(requestRepos); | |
Repos repos = JsonConvert.DeserializeObject<Repos>(responseRepos.Content); | |
foreach (Repo repo in repos.value) | |
{ | |
log.LogInformation("Repo: " + repo.name); | |
// get file mapping | |
var clientItems = new RestClient($"{devopsURL}_apis/git/repositories/{repo.id}/items?recursionlevel=full&{version}"); | |
var requestItems = new RestRequest(Method.GET); | |
requestItems.AddHeader("Authorization", auth); | |
var responseItems = clientItems.Execute(requestItems); | |
Items items = JsonConvert.DeserializeObject<Items>(responseItems.Content); | |
log.LogInformation("Items count: " + items.count); | |
if (items.count > 0) | |
{ | |
// get files as zip | |
var clientBlob = new RestClient($"{devopsURL}_apis/git/repositories/{repo.id}/blobs?{version}"); | |
var requestBlob = new RestRequest(Method.POST); | |
requestBlob.AddJsonBody(items.value.Where(itm => itm.gitObjectType == "blob").Select(itm => itm.objectId).ToList()); | |
requestBlob.AddHeader("Authorization", auth); | |
requestBlob.AddHeader("Accept", "application/zip"); | |
var zipfile = clientBlob.DownloadData(requestBlob); | |
// upload blobs to Azure Storage | |
string name = $"{project.name}_{repo.name}_blob.zip"; | |
var blob = container.GetBlockBlobReference(name); | |
await blob.DeleteIfExistsAsync(); | |
await blob.UploadFromByteArrayAsync(zipfile, 0, zipfile.Length); | |
// upload file mapping | |
string namejson = $"{project.name}_{repo.name}_tree.json"; | |
var blobjson = container.GetBlockBlobReference(name); | |
await blobjson.DeleteIfExistsAsync(); | |
blobjson.Properties.ContentType = "application/json"; | |
await blobjson.UploadTextAsync(responseItems.Content); | |
/* TODO: | |
* File mapping defines relationship between blob IDs and file names/paths. | |
* To reproduce a full file structure | |
* 1. Recreate all folders for <item.isFolder> | |
* 2. Extract all other items to <item.path> | |
*/ | |
} | |
} | |
} | |
log.LogInformation($"DevOps BackupFunction function finished at: {DateTime.Now}"); | |
} | |
} | |
struct Project | |
{ | |
public string name; | |
} | |
struct Projects | |
{ | |
public List<Project> value; | |
} | |
struct Repo | |
{ | |
public string id; | |
public string name; | |
} | |
struct Repos | |
{ | |
public List<Repo> value; | |
} | |
struct Item | |
{ | |
public string objectId; | |
public string gitObjectType; | |
public string commitId; | |
public string path; | |
public bool isFolder; | |
public string url; | |
} | |
struct Items | |
{ | |
public int count; | |
public List<Item> value; | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment