Created
March 19, 2021 08:05
-
-
Save hodzanassredin/c13a2ea3f914a261c9bd3174805abb22 to your computer and use it in GitHub Desktop.
console tool to upload big file to blob
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
using Azure.Storage; | |
using Azure.Storage.Files.DataLake; | |
using Azure.Storage.Files.DataLake.Models; | |
using System; | |
using System.Collections.Generic; | |
using System.IO; | |
using System.Threading.Tasks; | |
namespace CopyToBlob | |
{ | |
class Program | |
{ | |
public static IEnumerable<string> GetDirFiles(string path) { | |
foreach (string file in Directory.GetFiles(path, "*", SearchOption.AllDirectories)) | |
{ | |
yield return file; | |
} | |
} | |
public static DataLakeServiceClient GetDataLakeServiceClient(string accountName, string accountKey) | |
{ | |
StorageSharedKeyCredential sharedKeyCredential = | |
new StorageSharedKeyCredential(accountName, accountKey); | |
string dfsUri = "https://" + accountName + ".dfs.core.windows.net"; | |
return new DataLakeServiceClient | |
(new Uri(dfsUri), sharedKeyCredential); | |
} | |
public static async Task UploadFileBulk(FileStream fileStream, string dirName, string fileName, DataLakeFileSystemClient fileSystemClient) | |
{ | |
DataLakeDirectoryClient directoryClient = | |
fileSystemClient.GetDirectoryClient(dirName); | |
var p = new Progress(fileStream.Length); | |
var opts = new DataLakeFileUploadOptions { | |
ProgressHandler = p, | |
TransferOptions = new StorageTransferOptions { | |
MaximumTransferSize = 100003838 | |
} | |
}; | |
DataLakeFileClient fileClient = directoryClient.GetFileClient(fileName); | |
await fileClient.UploadAsync(fileStream, opts); | |
} | |
static async Task Main(string[] args) | |
{ | |
var ds = GetDataLakeServiceClient("", ""); | |
var fs = ds.GetFileSystemClient("backups"); | |
var dir = args[0]; | |
var files = GetDirFiles(dir); | |
foreach (var file in files) | |
{ | |
var localPath = file.Substring(dir.Length+1); | |
var remotePath = Path.GetRelativePath(dir, file); | |
var remoteName = Path.GetFileName(remotePath); | |
var remoteDir = "cs/" + remotePath.Replace(remoteName, ""); | |
Console.WriteLine("Uploading " + localPath); | |
var s = File.OpenRead(file); | |
if (s.Length == 0) { | |
Console.WriteLine("Skip zero length file"); | |
continue; | |
} | |
await UploadFileBulk(s, remoteDir.Replace('\\','/'), remoteName, fs); | |
} | |
} | |
} | |
public class Progress : IProgress<long> | |
{ | |
private readonly long size; | |
public Progress(long size) | |
{ | |
this.size = size + 1; | |
} | |
public void Report(long value) | |
{ | |
Console.WriteLine($"{DateTime.UtcNow.ToShortTimeString()}:{size}-{value} = {value * 100 / size }%"); | |
} | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment