Skip to content

Instantly share code, notes, and snippets.

@i-b1
Created March 9, 2020 16:29
Show Gist options
  • Save i-b1/131c6f65f0255ca7202207e25f24f164 to your computer and use it in GitHub Desktop.
Save i-b1/131c6f65f0255ca7202207e25f24f164 to your computer and use it in GitHub Desktop.
Azure DevOps API - Repositories Backup
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.Azure.WebJobs;
using Microsoft.Extensions.Logging;
using Microsoft.WindowsAzure.Storage;
using Newtonsoft.Json;
using RestSharp;
namespace AzureDevopsBackupFunction
{
public static class BackupFunction
{
private const string version = "api-version=5.1";
[FunctionName("BackupFunction")]
public static async Task Run([TimerTrigger("0 0 20 * * *")]TimerInfo myTimer, ILogger log) //, RunOnStartup = true
{
log.LogInformation($"DevOps BackupFunction function starting execution at: {DateTime.Now}");
// configure connections
string storageAccountKey = Environment.GetEnvironmentVariable("storageAccountKey", EnvironmentVariableTarget.Process);
string storageName = Environment.GetEnvironmentVariable("storageName", EnvironmentVariableTarget.Process);
string token = Environment.GetEnvironmentVariable("token", EnvironmentVariableTarget.Process);
string organization = Environment.GetEnvironmentVariable("organization", EnvironmentVariableTarget.Process);
string storageConnection = $"DefaultEndpointsProtocol=https;AccountName={storageName};AccountKey={storageAccountKey};EndpointSuffix=core.windows.net";
string devopsURL = $"https://dev.azure.com/{organization}/";
// make API request to get all projects
string auth = "Basic " + Convert.ToBase64String(System.Text.Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", token)));
var clientProjects = new RestClient($"{devopsURL}_apis/projects?{version}");
var requestProjects = new RestRequest(Method.GET);
requestProjects.AddHeader("Authorization", auth);
var responseProjects = clientProjects.Execute(requestProjects);
if(responseProjects.StatusCode != System.Net.HttpStatusCode.OK)
{
throw new Exception("API Request failed: " + responseProjects.StatusCode + " " + responseProjects.ErrorMessage);
}
Projects projects = JsonConvert.DeserializeObject<Projects>(responseProjects.Content);
// connect to Azure Storage
var storageAccount = CloudStorageAccount.Parse(storageConnection);
var client = storageAccount.CreateCloudBlobClient();
var container = client.GetContainerReference("devopsbackup");
await container.CreateIfNotExistsAsync();
foreach (Project project in projects.value)
{
log.LogInformation(project.name);
// get repositories
var clientRepos = new RestClient($"{devopsURL}{project.name}/_apis/git/repositories?{version}");
var requestRepos = new RestRequest(Method.GET);
requestRepos.AddHeader("Authorization", auth);
var responseRepos = clientRepos.Execute(requestRepos);
Repos repos = JsonConvert.DeserializeObject<Repos>(responseRepos.Content);
foreach (Repo repo in repos.value)
{
log.LogInformation("Repo: " + repo.name);
// get file mapping
var clientItems = new RestClient($"{devopsURL}_apis/git/repositories/{repo.id}/items?recursionlevel=full&{version}");
var requestItems = new RestRequest(Method.GET);
requestItems.AddHeader("Authorization", auth);
var responseItems = clientItems.Execute(requestItems);
Items items = JsonConvert.DeserializeObject<Items>(responseItems.Content);
log.LogInformation("Items count: " + items.count);
if (items.count > 0)
{
// get files as zip
var clientBlob = new RestClient($"{devopsURL}_apis/git/repositories/{repo.id}/blobs?{version}");
var requestBlob = new RestRequest(Method.POST);
requestBlob.AddJsonBody(items.value.Where(itm => itm.gitObjectType == "blob").Select(itm => itm.objectId).ToList());
requestBlob.AddHeader("Authorization", auth);
requestBlob.AddHeader("Accept", "application/zip");
var zipfile = clientBlob.DownloadData(requestBlob);
// upload blobs to Azure Storage
string name = $"{project.name}_{repo.name}_blob.zip";
var blob = container.GetBlockBlobReference(name);
await blob.DeleteIfExistsAsync();
await blob.UploadFromByteArrayAsync(zipfile, 0, zipfile.Length);
// upload file mapping
string namejson = $"{project.name}_{repo.name}_tree.json";
var blobjson = container.GetBlockBlobReference(name);
await blobjson.DeleteIfExistsAsync();
blobjson.Properties.ContentType = "application/json";
await blobjson.UploadTextAsync(responseItems.Content);
/* TODO:
* File mapping defines relationship between blob IDs and file names/paths.
* To reproduce a full file structure
* 1. Recreate all folders for <item.isFolder>
* 2. Extract all other items to <item.path>
*/
}
}
}
log.LogInformation($"DevOps BackupFunction function finished at: {DateTime.Now}");
}
}
struct Project
{
public string name;
}
struct Projects
{
public List<Project> value;
}
struct Repo
{
public string id;
public string name;
}
struct Repos
{
public List<Repo> value;
}
struct Item
{
public string objectId;
public string gitObjectType;
public string commitId;
public string path;
public bool isFolder;
public string url;
}
struct Items
{
public int count;
public List<Item> value;
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment