Created
May 6, 2024 14:44
-
-
Save dontpaniclabsgists/64e5da978d1378e7bf06b0bc9eb94432 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
class AmazonReview | |
{ | |
public string Id { get; set; } | |
public string ProductId { get; set; } | |
public string ProfileName { get; set; } | |
public string HelpfulnessNumerator { get; set; } | |
public string HelpfulnessDenominator { get; set; } | |
public int Score { get; set; } | |
public int Time { get; set; } | |
public string Summary { get; set; } | |
public string Text { get; set; } | |
// These won't be in the CSV. | |
// We'll created "Combined" with a custom mapping, and then | |
// we'll calculate "Embeddings" using OpenAI API. | |
public string Combined { get; set; } | |
public float[] Embeddings { get; set; } | |
} | |
class AmazonReviewMap : ClassMap<AmazonReview> | |
{ | |
public AmazonReviewMap() | |
{ | |
AutoMap(CultureInfo.InvariantCulture); | |
Map(m => m.Embeddings).Ignore(); | |
Map(m => m.Combined).Convert(args => | |
{ | |
var combined = "Title: " | |
+ args.Row.GetField<string>("Summary").Trim() | |
+ " Content: " | |
+ args.Row.GetField<string>("Text").Trim(); | |
return combined; | |
}); | |
} | |
} | |
AmazonReview[] reviews; | |
using (var reader = new StreamReader("./amazon_reviews_1000.csv")) | |
using (var csv = new CsvHelper.CsvReader(reader, new CsvHelper.Configuration.CsvConfiguration(CultureInfo.InvariantCulture) | |
{ | |
HasHeaderRecord = true, | |
HeaderValidated = null, | |
MissingFieldFound = null | |
}) | |
) | |
{ | |
csv.Context.RegisterClassMap<AmazonReviewMap>(); | |
reviews = csv.GetRecords<AmazonReview>().ToArray(); | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment