UpdateNotionFeeds.fs

module Main

open FSharp.Data
open FSharp.Data.JsonExtensions
open Google.Cloud.Functions.Framework
open System
open System.Globalization
open System.Net
open System.Text.RegularExpressions

module Notion =
    let private token = "secret_"
    let private feedsTableId = ""
    let private feedItemsTableId = ""

    let private http httpMethod data path = async {
        let data = match data with
                   | Some data -> Some <| TextRequest data
                   | None -> None

        let! contents =
            async {
                try
                    return! Http.AsyncRequestString(
                                sprintf "https://api.notion.com/v1/%s" path,
                                httpMethod=httpMethod,
                                headers=[
                                    "Authorization"  , sprintf "Bearer %s" token
                                    "Notion-Version" , "2021-05-13"
                                    "Content-Type"   , "application/json"
                                ],
                                ?body=data)
                with
                | :? WebException as e when e.Status = WebExceptionStatus.ProtocolError && (e.Response :?> HttpWebResponse).StatusCode = HttpStatusCode.Conflict ->
                    return "{}"
            }

        return JsonValue.Parse(contents)
    }

    let private get             = http "GET"   None
    let private post path data  = http "POST"  (Some data) path
    let private patch path data = http "PATCH" (Some data) path

    type Feed =
        { Id            : string
          Url           : string
          ReleaseWindow : DateTimeOffset * DateTimeOffset
          IntervalDays  : int
          RegExp        : (Regex * string) option
          ItemsCount    : int }


    let private reRe = Regex @"^/(.+)/(.+?)/$"
    let private dateFormat = "yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fffK"
    let private toDateFromNotion s = DateTimeOffset.ParseExact(s, dateFormat, CultureInfo.InvariantCulture)
    let private toDate s = DateTimeOffset.Parse(s, CultureInfo.InvariantCulture)
    let private toString (d : DateTimeOffset) = d.ToString(dateFormat)

    let getFeeds () = async {
        let data =
            sprintf """{
                "filter": {
                    "property": "Release window",
                    "date": {
                        "on_or_before": "%s"
                    }
                }
            }""" (DateTime.Now.ToString("yyyy-MM-dd"))
        let! json = post (sprintf "databases/%s/query" feedsTableId) data

        return [
            for result in json.["results"] do
                let properties = result.["properties"]
                let startDate =
                    properties.["Release window"].["date"].["start"].AsString() |> toDateFromNotion
                let endDate =
                    match properties.["Release window"].["date"].["end"] with
                    | JsonValue.String endDate -> toDateFromNotion endDate
                    | _ -> startDate

                yield {
                    Id = result.["id"].AsString()
                    Url = properties.["URL"].["url"].AsString()
                    ReleaseWindow = (startDate, endDate)
                    IntervalDays = properties.["Interval (days)"].["number"].AsInteger()
                    RegExp = [|
                        for rt in properties.["RegExp"].["rich_text"].AsArray() ->
                            rt.["text"].["content"].AsString()
                    |] |> String.concat ""
                       |> reRe.Match
                       |> function
                          | m when m.Success -> Some (Regex m.Groups.[1].Value, m.Groups.[2].Value)
                          | _ -> None
                    ItemsCount = 0
                }
        ]
    }

    let updateFeeds feeds =
        feeds
        |> Seq.filter (fun feed ->
               feed.ItemsCount > 0 || snd feed.ReleaseWindow < DateTimeOffset.Now
           )
        |> Seq.map (fun feed ->
               let mutable feed = feed

               while fst feed.ReleaseWindow < DateTimeOffset.Now do
                   let s, e = feed.ReleaseWindow
                   let s, e = s.AddDays(float feed.IntervalDays), e.AddDays(float feed.IntervalDays)

                   feed <- { feed with ReleaseWindow = (s, e) }

               let startDate = "\"" + toString (fst feed.ReleaseWindow) + "\""
               let endDate =
                   if fst feed.ReleaseWindow = snd feed.ReleaseWindow then
                       "null"
                   else
                       "\"" + toString (snd feed.ReleaseWindow) + "\""

               patch (sprintf "pages/%s" feed.Id) (sprintf """{
                   "properties": {
                       "Release window": {
                           "id": "cNh^",
                           "type": "date",
                           "date": {
                               "start": %s,
                               "end": %s
                           }
                       }
                   }
               }""" startDate endDate)
           )
        |> Async.Parallel
        |> Async.Ignore

    type FeedItem =
        { Feed : Feed
          Text : string
          Link : string
          Date : DateTimeOffset }

    let private regex pattern = Regex(pattern, RegexOptions.ECMAScript)
    let private atomRe =
        regex @"<entry>[\s\S]*?<title>(.+?)</title>[\s\S]*?<link.+?href=""(.+?)"".+?/>[\s\S]*?<updated>(.+?)</updated>"
    let private rssRe =
        regex @"<item>[\s\S]*?<title>(.+?)</title>[\s\S]*?<link>(.+?)</link>[\s\S]*?<pubDate>(.+?)</pubDate>"

    let addFeedItems feeds =
        feeds
        |> List.filter (fun feed -> fst feed.ReleaseWindow <= DateTimeOffset.Now)
        |> List.map (fun feed -> async {
               let! doc = Http.AsyncRequestString feed.Url
               printfn "Fetched document for feed %A" feed

               let feedOffset = (fst feed.ReleaseWindow).Offset
               let lastEnd = (snd feed.ReleaseWindow).AddDays(-float feed.IntervalDays)
               let re = if doc.StartsWith("<feed") then atomRe else rssRe
               let items = seq {
                   for m in re.Matches(doc) do
                       let text = m.Groups.[1].Value
                       let link = m.Groups.[2].Value
                       let date = (toDate m.Groups.[3].Value).ToOffset(feedOffset)

                       if date > lastEnd then
                           yield { Feed = feed; Text = text; Link = link; Date = date }
               }
               let items =
                   match feed.RegExp with
                   | Some (re, repl) ->
                       seq {
                           for item in items do
                               let repl = re.Replace(item.Text, repl)

                               if repl <> item.Text then
                                   yield { item with Text = repl }
                       }
                   | None -> items
               let escape (s: string) = s.Replace("\"", "\\\"").Replace("\n", "\\n")

               let! results =
                   items
                   |> Seq.map (fun item ->
                          printfn "Add item %A" item
                          post "pages" (sprintf """{
                              "parent": { "database_id": "%s" },
                              "properties": {
                                  "Title": { "title": [{ "text": { "content": "%s" } }] },
                                  "URL": { "url": "%s" },
                                  "Feed": { "relation": [{ "id": "%s" }] },
                                  "Release time": { "date": { "start": "%s" } }
                              }
                          }""" feedItemsTableId (escape item.Text) (escape item.Link) item.Feed.Id (toString item.Date))
                      )
                   |> Async.Parallel

               return { feed with ItemsCount = Seq.length results }
           })
        |> Async.Parallel

    let updateAllFeeds () = async {
        let! feeds = getFeeds()
        do printfn "Fetched %d feeds." feeds.Length
        let! feeds = addFeedItems feeds
        do printfn "Added feed items."

        do! updateFeeds feeds
        do printfn "Updated feeds."
    }

type Function() =
    interface ICloudEventFunction with
        member this.HandleAsync(_, _) = Notion.updateAllFeeds() |> Async.StartAsTask :> _

UpdateNotionFeeds.fsproj

<Project Sdk="Microsoft.NET.Sdk">
  <PropertyGroup>
    <OutputType>Exe</OutputType>
    <TargetFramework>netcoreapp3.1</TargetFramework>
  </PropertyGroup>

  <ItemGroup>
    <Compile Include="UpdateNotionFeeds.fs" />
  </ItemGroup>

  <ItemGroup>
    <PackageReference Include="Google.Cloud.Functions.Hosting" Version="1.0.0" />
    <PackageReference Include="FSharp.Data" Version="4.2.0" />
  </ItemGroup>
</Project>