Skip to content

Instantly share code, notes, and snippets.

@beckyconning
Created April 25, 2019 16:27
Show Gist options
  • Save beckyconning/c025dd073f3b20ff7bc8f6e1494e5fa5 to your computer and use it in GitHub Desktop.
Save beckyconning/c025dd073f3b20ff7bc8f6e1494e5fa5 to your computer and use it in GitHub Desktop.
using CsvHelper;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Data;
using System.Data.SqlClient;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Microsoft.SqlServer.Management.Smo;
using Microsoft.SqlServer.Server;
using Microsoft.SqlServer.Management.Common;
using Microsoft.SqlServer;
using System.Globalization;
namespace HTTPCSV
{
class ReformTable
{
[JsonProperty("columns")]
public ReformColumn[] Columns { get; set; }
[JsonProperty("name")]
public String Name { get; set; }
}
class ReformColumn
{
[JsonProperty("column")]
public String Name { get; set; }
[JsonProperty("type")]
public String Type { get; set; }
}
class Program
{
static DataType SqlType(String reformType)
{
switch (reformType)
{
case "offsetdatetime":
return DataType.DateTime;
case "number":
return DataType.Money;
case "string":
return DataType.Text;
case "boolean":
return DataType.Bit;
}
return DataType.Text;
}
static Type CSType(String reformType)
{
switch (reformType)
{
case "offsetdatetime":
return typeof(DateTime);
case "number":
return typeof(Decimal);
case "string":
return typeof(String);
case "boolean":
return typeof(Boolean);
}
return typeof(String);
}
static String SqlName(String reformName)
{
return $"{Regex.Replace(reformName, @"[^A-Za-z0-9]","_")}";
}
static void Main(string[] args)
{
using (WebClient client = new WebClient())
{
using (Stream reformTableStream = client.OpenRead(args[0]))
using (StreamReader reformTableStreamReader = new StreamReader(reformTableStream))
{
String encodedTable = reformTableStreamReader.ReadToEnd();
ReformTable table = JsonConvert.DeserializeObject<ReformTable>(encodedTable);
Server server = new Server(new ServerConnection(args[1]));
Database database = server.Databases[args[2]];
Table newTable = new Table(database, SqlName(table.Name));
Table oldTable = database.Tables[SqlName(table.Name)];
foreach (ReformColumn column in table.Columns)
{
Column newColumn = new Column(newTable, SqlName(column.Name), SqlType(column.Type));
newColumn.Nullable = true;
newTable.Columns.Add(newColumn);
}
oldTable?.Drop();
newTable.Create();
using (Stream stream = client.OpenRead($"{args[0]}/live/dataset"))
using (StreamReader streamReader = new StreamReader(stream))
using (var csv = new CsvReader(streamReader))
using (var dataReader = new CsvDataReader(csv))
{
csv.Configuration.TypeConverterOptionsCache.GetOptions<string>().NullValues.Add("");
using (SqlBulkCopy bulkCopy = new SqlBulkCopy("Data Source=EC2AMAZ-PFM7ROF;Integrated Security=True;Persist Security Info=False;Pooling=False;MultipleActiveResultSets=False;Encrypt=False;TrustServerCertificate=False;"))
{
bulkCopy.DestinationTableName = $"{args[2]}.dbo.[{SqlName(table.Name)}]";
bulkCopy.EnableStreaming = true;
bulkCopy.BulkCopyTimeout = 0;
bulkCopy.WriteToServer(dataReader);
}
}
}
}
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment