Last active
June 14, 2021 19:37
-
-
Save YannickRe/936d9d1a620421940ef99f69dc66c658 to your computer and use it in GitHub Desktop.
Code accompanying post at https://blog.yannickreekmans.be/index-your-ghost-blog-in-azure-search-with-an-azure-function/
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<Project Sdk="Microsoft.NET.Sdk"> | |
<PropertyGroup> | |
<TargetFramework>netstandard2.0</TargetFramework> | |
<AzureFunctionsVersion>v2</AzureFunctionsVersion> | |
</PropertyGroup> | |
<ItemGroup> | |
<PackageReference Include="Microsoft.Azure.Search" Version="5.0.2" /> | |
<PackageReference Include="morelinq" Version="3.0.0" /> | |
</ItemGroup> | |
</Project> |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#r "Newtonsoft.Json" | |
#r "Microsoft.WindowsAzure.Storage" | |
using System.Net; | |
using Microsoft.AspNetCore.Mvc; | |
using Microsoft.Extensions.Primitives; | |
using Newtonsoft.Json; | |
using Microsoft.WindowsAzure.Storage.Blob; | |
using Microsoft.Azure.Search; | |
using Microsoft.Azure.Search.Models; | |
using MoreLinq.Extensions; | |
private static HttpClient client; | |
private static readonly string BlogUrl = ""; | |
private static readonly string ContentApiKey = ""; | |
private static readonly string SearchServiceName = ""; | |
private static readonly string SearchAdminApiKey = ""; | |
public static async Task<IActionResult> Run(HttpRequest req, ILogger log) | |
{ | |
log.LogInformation("Loading posts from blog"); | |
var client = GetClient(); | |
var response = await client.GetAsync($"{BlogUrl}/ghost/api/v2/content/posts/?include=tags,authors&formats=plaintext&limit=15&key={ContentApiKey}"); | |
var json = await response.Content.ReadAsStringAsync(); | |
dynamic data = JsonConvert.DeserializeObject(json); | |
List<dynamic> postList = data.posts.ToObject<List<dynamic>>(); | |
log.LogInformation("Converting posts into the correct format"); | |
var postsArray = postList.ConvertAll((p) => { | |
List<dynamic> tags = p.tags.ToObject<List<dynamic>>(); | |
return new Post() { | |
Id = p.id, | |
Uuid = p.uuid, | |
Title = p.title, | |
Slug = p.slug, | |
PlainText = p.plaintext, | |
FeatureImage = p.feature_image, | |
CustomExcerpt = p.custom_excerpt, | |
Page = p.page, | |
Url = p.url, | |
CreatedAt = p.created_at, | |
UpdatedAt = p.updated_at, | |
PublishedAt = p.published_at, | |
Tags = tags.Select(t => t.slug.ToString()).Cast<string>().ToArray() | |
}; | |
}); | |
log.LogInformation("Creating search service client"); | |
var searchClient = new SearchServiceClient(SearchServiceName, new SearchCredentials(SearchAdminApiKey)); | |
log.LogInformation("Deleting Index"); | |
await searchClient.Indexes.DeleteAsync("posts"); | |
log.LogInformation("Creating Index"); | |
var definition = new Index() | |
{ | |
Name = "posts", | |
Fields = FieldBuilder.BuildForType<Post>(), | |
CorsOptions = new CorsOptions() | |
{ | |
AllowedOrigins = new List<string>() { BlogUrl }, | |
MaxAgeInSeconds = 300 | |
} | |
}; | |
await searchClient.Indexes.CreateAsync(definition); | |
log.LogInformation("BatchInsert Posts into Index"); | |
var indexClient = searchClient.Indexes.GetClient("posts"); | |
foreach(var objectBatch in postsArray.Batch(1000)) | |
{ | |
try | |
{ | |
IndexBatch<Post> batch = IndexBatch.MergeOrUpload(objectBatch); | |
await indexClient.Documents.IndexAsync(batch); | |
} | |
catch (IndexBatchException e) | |
{ | |
log.LogError(e, $"Failed to index some of the groups: {string.Join(", ", e.IndexingResults.Where(r => !r.Succeeded).Select(r => r.Key))}"); | |
} | |
} | |
return(ActionResult)new OkResult(); | |
} | |
private static HttpClient GetClient() | |
{ | |
client = client ?? new HttpClient(); | |
return client; | |
} | |
[SerializePropertyNamesAsCamelCase] | |
public class Post | |
{ | |
[System.ComponentModel.DataAnnotations.Key] | |
[IsRetrievable(true)] | |
public string Id { get; set; } | |
[IsSearchable, IsRetrievable(true)] | |
public string Uuid { get; set; } | |
[IsSearchable, IsRetrievable(true)] | |
public string Title { get; set; } | |
[IsSearchable, IsRetrievable(true)] | |
public string Slug { get; set; } | |
[IsSearchable, IsRetrievable(true)] | |
public string PlainText { get; set; } | |
[IsSearchable, IsRetrievable(true)] | |
public string FeatureImage { get; set; } | |
[IsSearchable, IsRetrievable(true)] | |
public string CustomExcerpt { get; set; } | |
[IsSearchable, IsRetrievable(true)] | |
public string[] Tags { get; set; } | |
[IsRetrievable(true)] | |
public string Url { get; set; } | |
[IsFilterable, IsRetrievable(true)] | |
public bool? Page { get; set; } | |
[IsRetrievable(true)] | |
public DateTime? CreatedAt { get; set; } | |
[IsRetrievable(true)] | |
public DateTime? UpdatedAt { get; set; } | |
[IsRetrievable(true)] | |
public DateTime? PublishedAt { get; set; } | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Updated the function script to use the newer Azure.Cognitive.Search client V11, which replaces the above V10 Azure.Search client
https://gist.github.com/SimonDarksideJ/6c4099900f1c66383a2ac00a1a322dfe