Skip to content

Instantly share code, notes, and snippets.

@hodzanassredin
Created March 19, 2021 08:05
Show Gist options
  • Save hodzanassredin/c13a2ea3f914a261c9bd3174805abb22 to your computer and use it in GitHub Desktop.
Save hodzanassredin/c13a2ea3f914a261c9bd3174805abb22 to your computer and use it in GitHub Desktop.
console tool to upload big file to blob
using Azure.Storage;
using Azure.Storage.Files.DataLake;
using Azure.Storage.Files.DataLake.Models;
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading.Tasks;
namespace CopyToBlob
{
class Program
{
public static IEnumerable<string> GetDirFiles(string path) {
foreach (string file in Directory.GetFiles(path, "*", SearchOption.AllDirectories))
{
yield return file;
}
}
public static DataLakeServiceClient GetDataLakeServiceClient(string accountName, string accountKey)
{
StorageSharedKeyCredential sharedKeyCredential =
new StorageSharedKeyCredential(accountName, accountKey);
string dfsUri = "https://" + accountName + ".dfs.core.windows.net";
return new DataLakeServiceClient
(new Uri(dfsUri), sharedKeyCredential);
}
public static async Task UploadFileBulk(FileStream fileStream, string dirName, string fileName, DataLakeFileSystemClient fileSystemClient)
{
DataLakeDirectoryClient directoryClient =
fileSystemClient.GetDirectoryClient(dirName);
var p = new Progress(fileStream.Length);
var opts = new DataLakeFileUploadOptions {
ProgressHandler = p,
TransferOptions = new StorageTransferOptions {
MaximumTransferSize = 100003838
}
};
DataLakeFileClient fileClient = directoryClient.GetFileClient(fileName);
await fileClient.UploadAsync(fileStream, opts);
}
static async Task Main(string[] args)
{
var ds = GetDataLakeServiceClient("", "");
var fs = ds.GetFileSystemClient("backups");
var dir = args[0];
var files = GetDirFiles(dir);
foreach (var file in files)
{
var localPath = file.Substring(dir.Length+1);
var remotePath = Path.GetRelativePath(dir, file);
var remoteName = Path.GetFileName(remotePath);
var remoteDir = "cs/" + remotePath.Replace(remoteName, "");
Console.WriteLine("Uploading " + localPath);
var s = File.OpenRead(file);
if (s.Length == 0) {
Console.WriteLine("Skip zero length file");
continue;
}
await UploadFileBulk(s, remoteDir.Replace('\\','/'), remoteName, fs);
}
}
}
public class Progress : IProgress<long>
{
private readonly long size;
public Progress(long size)
{
this.size = size + 1;
}
public void Report(long value)
{
Console.WriteLine($"{DateTime.UtcNow.ToShortTimeString()}:{size}-{value} = {value * 100 / size }%");
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment