enrich.rb

# GO TO START HERE ---------------------------------------------------------------------------

require "JSON"
require "securerandom"
require "date"

filename = ARGV[0]
string = File.read(filename)
json = JSON.parse(string)

$storage = {}

def filter(node, value, key="name", exact=true, force=false)
    if (exact && node[key] == value) || (!exact && node[key].include?(value))
        if force || node["children"] == nil  || node["children"].count == 0
            puts "[FILTER] Filtered \"#{node[key]}\""
            return nil
        end
        return node
    elsif node["children"]
        # Continue looking in child nodes
        node["children"] = node["children"].map{ |n| filter(n, value, key, exact, force) }.compact
    end

    return node
end

def create_supertag(supertag)
    return { "uid" => SecureRandom.hex, "name" => supertag}
end

def assign_supertag(node, supertag, value, key="name", direct_parent_supertag=nil, parent=nil)
    if (!direct_parent_supertag && node[key] == value) || (direct_parent_supertag && parent && parent["supertags"] && parent["supertags"].include?(direct_parent_supertag["uid"]) && node[key] == value)
        node["supertags"] ||= []
        node["supertags"] += [supertag["uid"]]
        node["supertags"] = node["supertags"].uniq

        $storage[supertag["uid"]] ||= []
        $storage[supertag["uid"]] += [node["uid"]]

        if direct_parent_supertag
            puts "[MATCH UNDER PARENT] Assigned \"#{value}\" → #{supertag["name"]}"
        else
            puts "[MATCH] Assigned \"#{value}\" → #{supertag["name"]}"
        end
    end

    if node["children"]
        # Continue looking in child nodes
        node["children"] = node["children"].map{ |n| assign_supertag(n, supertag, value, key, direct_parent_supertag, node) }
    end

    return node
end

def assign_supertag_to_children_of(node, supertag, value, key="name", nested=false)
    if node[key] == value
        if node["children"]
            node["children"].each do |child|
                if child["name"] != ""
                    child["supertags"] ||= []
                    child["supertags"] += [supertag["uid"]]
                    child["supertags"] = child["supertags"].uniq
                    
                    $storage[supertag["uid"]] ||= []
                    $storage[supertag["uid"]] += [child["uid"]]

                    puts "[CHILD OF] Assigned \"#{child["name"]}\" → #{supertag["name"]} (child of #{value})"
                end
            end
        end

        # Stop if we do not expect nested children to be tagged (default)
        return node unless nested
    end

    if node["children"]
        # Continue looking in child nodes
        node["children"] = node["children"].map{ |n| assign_supertag_to_children_of(n, supertag, value, key, nested) }
    end

    return node
end

def assign_supertag_by_prefix(node, supertag, prefix, replace=false, direct_parent_supertag=nil, parent=nil)
    if (!direct_parent_supertag && node["name"][0,prefix.length] == prefix) || (direct_parent_supertag && parent && parent["supertags"] && parent["supertags"].include?(direct_parent_supertag["uid"]) && node["name"][0,prefix.length] == prefix)
        node["supertags"] ||= []
        node["supertags"] += [supertag["uid"]]
        node["supertags"] = node["supertags"].uniq

        $storage[supertag["uid"]] ||= []
        $storage[supertag["uid"]] += [node["uid"]]

        if replace
            node["name"] = node["name"].sub(prefix,"")
        end

        puts "[PREFIX] Assigned \"#{node["name"]}\" → #{supertag["name"]}"
    end

    if node["children"]
        # Continue looking in child nodes
        node["children"] = node["children"].map{ |n| assign_supertag_by_prefix(n, supertag, prefix, replace, direct_parent_supertag, node) }
    end

    return node
end

def assign_supertag_by_postfix(node, supertag, postfix, replace=false, direct_parent_supertag=nil, parent=nil)
    if (!direct_parent_supertag && node["name"][-postfix.length,node["name"].length] == postfix) || (direct_parent_supertag && parent && parent["supertags"] && parent["supertags"].include?(direct_parent_supertag["uid"]) && node["name"][-postfix.length,node["name"].length] == postfix)
        node["supertags"] ||= []
        node["supertags"] += [supertag["uid"]]
        node["supertags"] = node["supertags"].uniq

        $storage[supertag["uid"]] ||= []
        $storage[supertag["uid"]] += [node["uid"]]

        if replace
            node["name"] = node["name"].sub(prefix,"")
        end

        puts "[POSTFIX] Assigned \"#{node["name"]}\" → #{supertag["name"]}"
    end

    if node["children"]
        # Continue looking in child nodes
        node["children"] = node["children"].map{ |n| assign_supertag_by_postfix(n, supertag, postfix, replace, direct_parent_supertag, node) }
    end

    return node
end

def assign_supertag_to_matches(node, new_supertag, old_supertag, term, replace=true)
    if node["name"].include?(term) && node["supertags"] && node["supertags"].find{ |t| t == old_supertag["uid"] }        
        node["supertags"] += [new_supertag["uid"]]
        node["supertags"] = node["supertags"].uniq

        $storage[new_supertag["uid"]] ||= []
        $storage[new_supertag["uid"]] += [node["uid"]]

        if replace
            node["supertags"] -= [old_supertag["uid"]]
            
            $storage[old_supertag["uid"]] -= [node["uid"]]

            puts "[SUPERTAG MATCH] Reassigned \"#{node["name"]}\" #{old_supertag["name"]} → #{new_supertag["name"]}"
        else            
            puts "[SUPERTAG MATCH] Assigned \"#{node["name"]}\" → #{new_supertag["name"]}"
        end
    end

    if node["children"]
        # Continue looking in child nodes
        node["children"] = node["children"].map{ |n| assign_supertag_to_matches(n, new_supertag, old_supertag, term) }
    end

    return node
end

def assign_supertags(json, supertag, direct_parent_supertag=nil, list=nil)
    list ||= File.read("./#{supertag}.txt").split("\n")

    json["supertags"] = json["supertags"] || []
    json["supertags"] += [tag = create_supertag(supertag)] unless tag = json["supertags"].find{ |t| t["name"] == supertag }
    json["supertags"] += [direct_parent_tag = create_supertag(direct_parent_supertag)] unless direct_parent_tag = json["supertags"].find{ |t| t["name"] == direct_parent_supertag } if direct_parent_supertag

    list.each{ |term| json["nodes"] = json["nodes"].map{ |n| assign_supertag(n, tag, term, "name", direct_parent_tag) }.compact }
    return json
end

def assign_supertags_by_type(json, supertag, type)
    json["supertags"] = json["supertags"] || []
    json["supertags"] += [tag = create_supertag(supertag)] unless tag = json["supertags"].find{ |t| t["name"] == supertag }
    json["nodes"] = json["nodes"].map{ |n| assign_supertag(n, tag, type, "type") }
    return json
end

def assign_supertags_to_children_of(json, supertag, title)
    json["supertags"] = json["supertags"] || []
    json["supertags"] += [tag = create_supertag(supertag)] unless tag = json["supertags"].find{ |t| t["name"] == supertag }
    json["nodes"] = json["nodes"].map{ |n| assign_supertag_to_children_of(n, tag, title) }.compact
    return json
end

def assign_supertags_by_prefix(json, supertag, prefix, replace=false, direct_parent_supertag=nil)
    json["supertags"] = json["supertags"] || []
    json["supertags"] += [tag = create_supertag(supertag)] unless tag = json["supertags"].find{ |t| t["name"] == supertag }
    json["supertags"] += [direct_parent_tag = create_supertag(direct_parent_supertag)] unless direct_parent_tag = json["supertags"].find{ |t| t["name"] == direct_parent_supertag } if direct_parent_supertag

    json["nodes"] = json["nodes"].map{ |n| assign_supertag_by_prefix(n, tag, prefix, replace, direct_parent_tag) }
    return json
end

def assign_supertags_by_postfix(json, supertag, postfix, replace=false, direct_parent_supertag=nil)
    json["supertags"] = json["supertags"] || []
    json["supertags"] += [tag = create_supertag(supertag)] unless tag = json["supertags"].find{ |t| t["name"] == supertag }
    json["supertags"] += [direct_parent_tag = create_supertag(direct_parent_supertag)] unless direct_parent_tag = json["supertags"].find{ |t| t["name"] == direct_parent_supertag } if direct_parent_supertag

    json["nodes"] = json["nodes"].map{ |n| assign_supertag_by_postfix(n, tag, postfix, replace, direct_parent_tag) }
    return json
end

def assign_supertags_to_matches(json, new_supertag, old_supertag, term)
    json["supertags"] = json["supertags"] || []
    json["supertags"] += [new_tag = create_supertag(new_supertag)] unless new_tag = json["supertags"].find{ |t| t["name"] == new_supertag }

    old_tag = json["supertags"].find{ |t| t["name"] == old_supertag }
    raise "Old supertag not found" unless old_tag

    json["nodes"] = json["nodes"].map{ |n| assign_supertag_to_matches(n, new_tag, old_tag, term) }
    return json
end

# Field magic

def remove_field(node, name)        
    if node["type"] == "field" && node["name"] == name
        puts "[REMOVE FIELD] Removed field \"#{name}\""

        return nil
    end

    if node["children"]
        # Continue looking in child nodes
        node["children"] = node["children"].map{ |n| remove_field(n, name) }.compact
    end

    return node
end

def remove_attribute_and_fields(json, name)
    puts "[REMOVE ATTRIBUTE] Removed attribute \"#{name}\""

    # Filter the attribute
    json["attributes"] = json["attributes"].select do |attribute|
        attribute["name"] != name
    end

    json["nodes"] = json["nodes"].map{ |n| remove_field(n, name) }.compact
    return json
end

def enrich_supertag_with_fields(node, mapping)
    # If the node has a supertag that matches one of the keys in the mapping
    if node["supertags"] && (node["supertags"] - mapping.keys.map{|s| s["uid"]}).length != node["supertags"].length
        # Get the refs from the name, classified by supertag
        refs = classified_refs_from_string(node["name"])

        # For each supertag in the mapper
        mapping.each do |node_supertag, field_mapping|
            # That this node has
            if node["supertags"].include?(node_supertag["uid"])
                # Go through the fields
                field_mapping.each do |field_name, field_supertags|
                    relevant_refs = []
                    field_supertags.each do |field_supertag|
                        relevant_refs += refs[field_supertag["uid"]] || []
                    end
                    
                    # If the name has any references tagged as field_supertag
                    if relevant_refs.length > 0
                        # Find or create the field                        
                        if field = node["children"].find{ |c| c["name"] == field_name && c["type"] == "field" }
                            node["children"] = node["children"].select{ |c| c["uid"] != field["uid"] }
                        else
                            # Create a new field
                            field = child_node_for(node, field_name, "field")
                        end
                        
                        # With the refs from the node's name that match the field_supertag
                        field["children"] = relevant_refs.map do |ref|
                            child_node_for(field, "[[#{ref}]]")
                        end
                        
                        # Add the field to the node's children
                        node["children"] ||= []
                        node["children"] = [field] + node["children"]

                        puts "[FIELD] Added \"#{field_name}\" field to \"#{node["name"]}\" tagged as \"#{node_supertag["name"]}\" with refs: #{relevant_refs}"
                    else
                        # puts "[FIELD] No relevant references in \"#{node["name"]}\" tagged as \"#{node_supertag["name"]}\""
                    end
                end
            end            
        end
    end

    if node["children"]
        # Continue looking in child nodes
        node["children"] = node["children"].map{ |n| enrich_supertag_with_fields(n, mapping) }
    end

    return node
end

def enrich_supertags_with_fields(json, mapping)
    json["supertags"] = json["supertags"] || [] 
    json["attributes"] = json["attributes"] || []
    
    # Convert the names in the mapping to actual supertags from the JSON
    converted_mapping = {}
    mapping.each do |node_supertag_name, field_mapping|
        json["supertags"] += [node_supertag = create_supertag(node_supertag_name)] unless node_supertag = json["supertags"].find{ |t| t["name"] == node_supertag_name }
        converted_mapping[node_supertag] = {}

        field_mapping.each do |field_name, field_supertag_names|

            unless json["attributes"].find{ |a| a["name"] == field_name }
                json["attributes"] += [{
                    "name" => field_name,
                    "values" => [],
                    "count" => 1
                }]
            end

            converted_mapping[node_supertag][field_name] = []
            field_supertag_names.each do |field_supertag_name|
                json["supertags"] += [field_supertag = create_supertag(field_supertag_name)] unless field_supertag = json["supertags"].find{ |t| t["name"] == field_supertag_name }
                converted_mapping[node_supertag][field_name] += [field_supertag]
            end
        end
    end

    json["nodes"] = json["nodes"].map{ |n| enrich_supertag_with_fields(n, converted_mapping) }

    return json
end

def refs_from_string string
    string.scan(/\[\[([^\]]*)\]\]/).flatten
end

def classified_refs_from_string string
    refs = refs_from_string(string)
    classified_refs = {}

    refs.each do |ref|
        $storage.each do |supertag_uid, supertag_refs|
            if supertag_refs.include?(ref)
                classified_refs[supertag_uid] ||= []
                classified_refs[supertag_uid] += [ref]
            end
        end
    end

    return classified_refs
end

def child_node_for node, name, type="node"
    {
        "uid" => SecureRandom.hex,
        "name" => name,
        "children" => [],
        "refs" => refs_from_string(name),
        "type" => type,
        "createdAt" => node ? node["createdAt"] : DateTime.now.strftime('%Q').to_i,
        "editedAt" => node ? node["editedAt"] : DateTime.now.strftime('%Q').to_i
    }
end

# Find and replace

def replace_node(node, old_value, new_value)
    if node["name"] == old_value        
        node["name"] = new_value
        node["refs"] = refs_from_string(new_value)
        
        puts "[REPLACE] Replaced \"#{old_value}\" with #{new_value}"
    end

    if node["children"]
        # Continue looking in child nodes
        node["children"] = node["children"].map{ |n| replace_node(n, old_value, new_value) }
    end

    return node
end

def replace_nodes(json, old_value, new_value)
    json["nodes"] = json["nodes"].map{ |n| replace_node(n, old_value, new_value) }
    return json
end

def replace_nodes_with_reference(json, name, italic=false)
    new_node = child_node_for(nil, italic ? "<i>#{name}</i>" : name)
    
    # Replace the old name with the reference to the new node
    json["nodes"] = json["nodes"].map{ |n| replace_node(n, name, "[[#{new_node["uid"]}]]") }
    
    # Add the replacement reference
    json["nodes"] += [new_node]

    return json
end


# START HERE ---------------------------------------------------------------------------

# Filter every node without children that exact matches the lines in filter_exact.txt (top to bottom)
# Filter every node without children that contains a lines in filter_contains.txt (top to bottom)
# If you want to remove more complex (but empty) structures, add the lowest nodes first:
# 
# Origin:
# - Personal
#   - Notes
#   - Tasks
#  
# filter.txt:
# Tasks
# Notes
# Personal
File.read("./filter_exact.txt").split("\n").each{ |term| json["nodes"] = json["nodes"].map{ |n| filter(n, term, "name", true) }.compact }
File.read("./filter_contains.txt").split("\n").each{ |term| json["nodes"] = json["nodes"].map{ |n| filter(n, term, "name", false) }.compact }

# Destroy every node from kill.txt (no matter the content) (only for dead nodes like javascript/css/etc)
File.read("./kill.txt").split("\n").each{ |term| json["nodes"] = json["nodes"].map{ |n| filter(n, term, "name", true, true) }.compact }

# Filter dates without nodes (this might break references)
json["nodes"] = json["nodes"].map{ |n| filter(n, "date", "type") }.compact

# Assign the day supertag to all dates
assign_supertags_by_type(json, "day", "date")

# Assign a supertag based on prefix/postfix with optional filtering of the prefix/postfix
# in the new title
json = assign_supertags_by_prefix(json, "article", "Article/", true)
json = assign_supertags_by_prefix(json, "book", "Book/", true)
json = assign_supertags_by_prefix(json, "podcast", "Podcast/", true)
json = assign_supertags_by_prefix(json, "video", "Video/", true)
json = assign_supertags_by_prefix(json, "movie", "Movie/", true)
json = assign_supertags_by_prefix(json, "tv-show", "TV Show/", true)
json = assign_supertags_by_prefix(json, "tweet", "Tweet/", true)
json = assign_supertags_by_prefix(json, "stoic-discipline", "Discipline/", true)
json = assign_supertags_by_prefix(json, "stoic-passion", "Passion/", true)
json = assign_supertags_by_prefix(json, "mental-model", "Mental Model/", true)
json = assign_supertags_by_prefix(json, "cognitive-distortion", "Cognitive Distortion/", true)
json = assign_supertags_by_prefix(json, "cognitive-bias", "Cognitive Bias/", true)
json = assign_supertags_by_prefix(json, "informal-fallacy", "Informal Fallacy/", true)
json = assign_supertags_by_prefix(json, "virtue", "Virtue/", true)
json = assign_supertags_by_prefix(json, "zettel", "Z: ", true)
json = assign_supertags_by_prefix(json, "project", "Project/", true)
json = assign_supertags_by_prefix(json, "weekly-plan", "Weekly Plan", false)
json = assign_supertags_by_prefix(json, "yearly-plan", "Yearly Plan", false)
json = assign_supertags_by_prefix(json, "meeting", "Meeting", false, "day")

json = assign_supertags_by_postfix(json, "meeting", "[[MijI43YZl]]", false, "day") # ends with [[1-1]]
json = assign_supertags_by_postfix(json, "meeting", "[[ICDIEhoeL]]", false, "day") # ends with [[3P]]
json = assign_supertags_by_postfix(json, "meeting", "Meeting", false, "day")

# Assign a supertag to every node with a name from SUPERTAG.txt
# i.e. json = assign_supertags(json, "todo") # reads todo.txt and adds #todo to each node with those names
json = assign_supertags(json, "co-worker")
json = assign_supertags(json, "person")
json = assign_supertags(json, "team")
json = assign_supertags(json, "tribe")
json = assign_supertags(json, "product")
json = assign_supertags(json, "organisation")
json = assign_supertags(json, "place")
json = assign_supertags(json, "meeting-type")
json = assign_supertags(json, "section")

# Assign meeting to to nodes that match meeting.txt, directly under a node with the day supertag
json = assign_supertags(json, "meeting", "day")

# Assign a supertag to each child of a node with a name (we skip blank nodes)
json = assign_supertags_to_children_of(json, "meeting", "Meetings")

# Assign a supertag to any node that already has the given supertag and contains the search term
json = assign_supertags_to_matches(json, "1-1", "meeting", "[[MijI43YZl]]")
json = assign_supertags_to_matches(json, "3P", "meeting", "[[ICDIEhoeL]]")

# Enrich nodes with specific supertag from other supertags in their title. 
# It knows all earlier assigned supertags.
#
# Original:
# - Andre Foeken #1-1
#
# Enriched:
# - Andre Foeken #1-1
#   - Attendees:: Andre Foeken
json = enrich_supertags_with_fields(json, {
    "1-1" => {
        "Attendees" => ["person", "co-worker"],
        "Type" => ["meeting-type"]
    },
    "3P" => {
        "Attendees" => ["person", "co-worker"],
        "Type" => ["meeting-type"]
    },
    "meeting" => {
        "Attendees" => ["person", "co-worker"],
        "Team" => ["team"]
    }
})

File.read("filter_fields.txt").split("\n").each do |field|
    json = remove_attribute_and_fields(json, field)
end

# Replace prompts with refs and assign supertag
File.read("prompt.txt").split("\n").each do |prompt|
    json = replace_nodes_with_reference(json, prompt, true)
end
italic_list = File.read("prompt.txt").split("\n").map{ |prompt| "<i>#{prompt}</i>" }
json = assign_supertags(json, "prompt", nil, italic_list)

json = replace_nodes(json, "[#中文]([[FhGxpMYAD]])", "#[[FhGxpMYAD]]")
json = replace_nodes(json, "[#Personal]([[T2nEKE4-K]])", "#[[T2nEKE4-K]]")
json = replace_nodes(json, "[#Journal]([[5euaGktOi]])", "#[[5euaGktOi]]")
json = replace_nodes(json, "[#Inbox]([[ivDvrOPOo]])", "#[[ivDvrOPOo]]")
json = replace_nodes(json, "[#Nedap]([[CtBBNqRJ1]])", "#[[CtBBNqRJ1]]")

File.write(ARGV[1], JSON.dump(json))