collaborative

workedonmymachine.mjs

import { select, drag, local } from 'd3';
import slugify from "slugify";
import { v4 as uuidv4 } from 'uuid';
import hljs from 'highlight.js';
import markdownit from "markdown-it";
import footnote_plugin from "markdown-it-footnote"
import deflist_plugin from "markdown-it-deflist"
import { basicSetup } from "codemirror"
import { EditorView, keymap } from "@codemirror/view"
import { indentWithTab } from "@codemirror/commands"
import * as Y from 'yjs'
import { yCollab } from 'y-codemirror.next'
import { WebrtcProvider } from 'y-webrtc'

const HEADINGS = "table.cells h1,h2,h3,h4,h5,h6,[role=heading]";

// class is not an idref but we class elements based on their parent document
const IDREF_SELECTORS = ["id", "aria-labelledby", "aria-describedby", "aria-controls", "aria-owns", "form", "name", "class"];
const IDREF_SELECTOR = IDREF_SELECTORS.map(x => `[${x}]`).join(",");

const CELL_TYPES = ["code", "markdown", "raw"]
const OUTPUT_TYPES = ["display_data", "execute_result"]
const LITERATE = /^\s*%{2}\s+/m;

class Templates {
    constructor(document) {
        this.document = document.node();
        this.cells_body = this.document.getElementById("tpl:cell-row").content.querySelector("tbody");
        this.docs_row = this.document.getElementById("tpl:doc.meta").content.querySelector("tr");
        this.cell_row = this.document.getElementById("tpl:cell-row").content.querySelector("tr");
        this.cell_output = this.document.getElementById("tpl:output").content.querySelector("table");
        this.cell_output_row = this.document.getElementById("tpl:output-row").content.querySelector("tr");
        this.toc_row = this.document.getElementById("tpl:toc").content.querySelector("tr");
        this.toc_body = this.document.getElementById("tpl:toc").content.querySelector("tbody");
    }
    new_cells_body() { return this.cells_body.cloneNode() }
    new_cell_row() { return this.cell_row.cloneNode(true) }
    new_docs_row() { return this.docs_row.cloneNode(true) }
    new_cell_output() { return this.cell_output.cloneNode(true) }
    new_cell_output_row() { return this.cell_output_row.cloneNode(true) }
    new_toc_row() { return this.toc_row.cloneNode(true) }
    new_toc_body() { return this.toc_row.cloneNode() }
    static new_markdown_cell(source = "", cell = {}) {
        return {
            cell_type: "markdown",
            id: uuidv4(),
            metadata: {},
            source: source.split("\n"),
            ...cell
        }
    }
    static new_code_cell(source = "", cell = {}) {
        return { ...this.new_markdown_cell(source), cell_type: "code", outputs: [], ...cell }
    }
    static new_notebook(cells = [], metadata = {}, nb) {
        return { cells: cells, metadata: metadata }
    }
}

export default class Notebook {
    constructor(document) {
        document = document.node === undefined ? select(document) : document;
        this.document = document

        let section = this.document.select("section");
        let nb = { cells: [] }
        if (section.datum() === undefined) {
            let nbformat = section.select("script.nb").text();
            if (nbformat) {
                nb = JSON.parse(nbformat);
                section.datum(nb)
            }
        }
        this.templates = new Templates(document)
        // an array holding the order of documents

        this.order = new Array()
        this.cells = new Map()
        this.md_env = {}

        this.appendix = {
            references: Templates.new_markdown_cell("", { id: "references" }),
            equations: Templates.new_markdown_cell("", { id: "equations" }),
            supplementary: Templates.new_markdown_cell("", { id: "supplementary" }),
        }

    }

    // update the class attributes then build the interface from the worked values.

    // uses arguments to build the document
    updateDocumentSummary(selection) {
        if (!selection.size()) { return }
        let d = selection.datum()
        selection.attr("id", d.id);
        selection.select(".id").text(d.id)
        selection.select(".path").text(d.metadata?.path)
        selection.select(".all.cells>output").text(d.cells.length)
        selection.select(".markdown.cells>output").text(d.cells.filter(d => d.cell_type == "markdown").length)
        selection.select(".code.cells>output").text(d.cells.filter(d => d.cell_type == "code").length)
    }
    fromDom() {
        docs = Array();
        for (const body of document.querySelectorAll("table.cells>tbody")) {
            let doc = Templates.new_notebook();
            docs.push(doc)
            for (const row of body.querySelectorAll("&>tr.cell")) {
                let cell = Notebook.fromDomCell(row);
                doc.cells.push(cell)
            }
        }
    }

    static fromDomCell(row) {
        let cell = Object.fromEntries(new FormData(row.node().querySelector("form")));
        cell.source = cell.source.split("\n").map(s => s + "\n")
        if (cell.cell_type == "code") {
            cell.outputs = Array()
            for (const output_body of row.querySelectorAll("table.outputs>tbody")) {
                for (const entry of output_body.querySelectorAll("&>tr")) {

                }
            }
        }
        return cell
    }
    update() {
        let self = this;
        let section = this.document.select("section");
        let documents = Object.values(arguments).map((d) => { return { id: uuidv4(), ...d } });
        let order = Array()
        let args = Array()
        if (!arguments) {
            args = this.document.node().querySelectorAll("table#docs>tbody").map(
                select
            ).map(s => s.datum())
        }
        console.error(args)
        for (const nb of arguments || args) {
            let local_order = Array();
            order.push(local_order)
            for (const cell of nb.cells) {
                cell.id = cell.id || uuidv4();
                self.cells.set(cell.id, cell);
                local_order.push(cell.id)
            }
        }



        self.order.length = order.length
        for (const [i, d] of order.entries()) {
            self.order[i] = d
        }
        // everything should be a notebook by the time it reaches here.
        // lets assume we are getting the whole document in the arguments
        // the command line tool would provide that.
        // when the cli runs we need to boostrap the document data by inferring a document format from the html
        section.select("table#docs>tbody").selectChildren("tr").data(
            documents
        ).join(
            enter => enter.append((d, i) => {
                let doc = select(self.templates.new_docs_row()).datum(d);
                self.updateDocumentSummary(doc)
                return doc.node()
            }),
            update => self.updateDocumentSummary(update)
        )


        // iterate over each document and build all the cells
        section.select("table.cells").selectChildren("tbody").data(
            self.order
        ).join(
            enter => self.enterDocument(enter),
            update => self.updateDocument(update),
            exit => exit.remove()
        )

        let foot = section.select("table#docs>tfoot");
        foot.select(".all.cells>output").text(this.document.selectAll("tbody.cells>tr.cell").size())
        foot.select(".markdown.cells>output").text(this.document.selectAll("tbody.cells>tr.cell.markdown:not(.code)").size())
        foot.select(".code.cells>output").text(this.document.selectAll("tbody.cells>tr.cell.code").size())

        this.exitDocuments()
        return this
    }
    enterDocument(selection) {
        let self = this;
        selection.insert((d, i) => {
            let selection = select(self.templates.new_cells_body()).datum(d);
            self.updateDocument(selection)
            return selection.node()
        }, "tfoot")
    }

    updateDocument(selection) {
        let self = this;
        if (!selection.size()) { return }
        selection.selectChildren("tr").data(
            selection.datum().map(self.cells.get.bind(self.cells)).map(
                d => { return { doc: selection.datum().id, ...d } }
            ), this.cellsKey
        ).join(
            enter => self.enterCells(enter),
            update => self.updateCell(update),
            exit => exit.remove()
        )
    }

    enterDocumentMeta(selection) {
        let self = this;
        selection.append((d, i) => {
            let document = select(self.templates.new_docs_row()).datum(d);
        })
    }

    exitDocuments() {
        let document = this.document;
        let self = this;

        document.select("table.cells>tfoot.cells").selectAll("tr").data(
            Object.values(this.appendix)
        ).join(
            enter => self.enterCells(enter),
            update => self.updateCell(update),
            exit => exit.remove()
        )

        Array.from(document.selectAll(HEADINGS)).forEach(node => headingToId(select(node), document.node()))
        this.enterToc()
        this.enterSummary()
        document.select("a.perma").attr("id", "h1")
    }

    enterCells(selection) {
        let self = this;
        selection.append(
            (cell, i) => {
                let selection = select(self.templates.new_cell_row())
                    .datum(Object.assign({}, cell, { position: i }));
                self.enterCell(selection);
                return selection.node()
            }
        );
    }

    enterCell(cell_row) {
        let cell = cell_row.datum();

        cell_row.classed(cell.doc, true)
        cell_row.select("th.id>input").attr("value", cell.id);

        this.enterCellIds(cell_row)
        this.updateCell(cell_row)

        if (cell.cell_type == "markdown") {
            cell.outputs = [{ output_type: "display_data", data: { "text/markdown": cell.source } }]
        }
        cell.outputs?.length > 0 && this.enterOutputs(cell_row)
    }

    updateCell(cell_row) {
        console.error(22, cell_row.node())
        if (!cell_row.node()) { return }
        let cell = cell_row.datum();
        let i = cell.position;

        cell_row.attr("aria-posinset", i + 1)
        cell_row.select("th.doc").select("a").text(cell.doc).attr("href", `#${cell.doc}`);
        cell_row.select("th.pos").select("a").text(i + 1).attr("href", `#c${i + 1}`).attr("id", `c${i + 1}`);

        if (!cell_row.classed(cell.cell_type)) {
            CELL_TYPES.forEach(t => cell_row.classed(t, cell.cell_type == t));
            cell_row.select("td.cell_type>label").text(cell.cell_type)
            cell_row.select(`option[value="${cell.cell_type}"]`).attr("selected", "")
        }
        cell.execution_count && cell_row.select("td.execution_count>label>output").text(cell.execution_count);
        let source = cell.source.join("")
        let loc = cell.source.filter(Boolean).length
        cell_row.classed("markdown", cell.cell_type == "markdown" || Boolean(source.match(LITERATE)))
        cell_row.select(".loc>output").text(loc);
        cell_row.attr("data-loc", loc)
        cell_row.select("td.source>textarea").text(source)
        cell_row.classed("empty", !Boolean(source.trimRight()))
        this.updateOutputs(cell_row)
    }

    enterOutputs(selection) {
        let self = this;
        selection.select("td.outputs").append(
            d => self.templates.new_cell_output()
        )
        self.updateOutputs(selection)
    }

    updateOutputs(selection) {
        let cell = selection.datum();
        let self = this;
        selection.select("td.outputs table.outputs").selectChildren("tbody").data(
            selection.datum().outputs || []
        ).join("tbody").selectChildren("tr").data(
            d => d.data ? Object.entries(d.data).map(x => x.concat([d.metadata])) : [d],
            (d, i) => self.outputKey(cell, d, i)
        ).join(
            (enter) => enter.append(
                (output, i) => {
                    let selection = select(self.templates.new_cell_output_row())
                        .datum(output);
                    this.enterOutput(selection);
                    return selection.node()
                }),
            (update) => self.enterOutput(update), // updateOutput method that will be used with markdown cells to avoid extra work
            exit => exit.remove()
        )
    }

    enterOutput(output_row) {
        if (!output_row.node()) { return }
        let output = output_row.datum();
        if (!output.output_type) {
            this.enterOutputBundle(output_row)
        }
    }

    updateOutput(output_row) { }

    enterOutputBundle(output_row) {
        let [type, bundle, metadata] = output_row.datum();
        output_row.select("td.output_type>label").text(type);
        if (type == "text/html") {
            output_row.select("td.data").html(bundle.join(""))
        } else if (type == "text/markdown") {
            let data = output_row.select("td.data");
            let body = bundle.join("");
            data.html("")
            data.html(this.markdownify(body)).call(replaceAttachments)
            // this lets us retrieve the source
            // data.append("script").attr("type", type).text(body)
        } else if (type == "text/plain") {
            output_row.select("td.data").append("samp").text(bundle.join(""))
        } else if (type.startsWith("image")) {
            output_row.select("td.data").append("img")
                .attr("src", `data:${type};base64,${bundle.join("")}`);
        }
    }

    enterCellIds(cell_row) {
        let cell_id = cell_row.datum().id;
        [cell_row.node()].concat(Array.from(cell_row.selectAll(IDREF_SELECTOR))).map(select).forEach(
            (node) => {
                for (const ref of IDREF_SELECTORS) {
                    let refs = node.attr(ref);
                    if (!refs) { continue }
                    refs = refs.split(" ");
                    let idref = ""
                    for (let [j, label] of refs.entries()) {
                        idref += idref ? " " : "";
                        if (label.startsWith(":")) { idref += cell_id }
                        else if (label == "#") { label = cell_id }
                        else if (label == ":") { label = cell_id }
                        idref += label;
                    }
                    node.attr(ref, idref)
                }
            })
    }

    enterToc() {
        let self = this;
        self.document.select("section table.toc").selectAll("tbody").data(
            Array.from(self.document.node().querySelectorAll("tbody.cells"))
        ).join(
            enter => {
                enter.append(
                    (document, i) => {
                        let body = select(self.templates.new_toc_body());
                        body.selectAll("tr").data(
                            document.querySelectorAll(HEADINGS).map(
                                (element) => {
                                    return {
                                        heading: element.innerText,
                                        cell: Number(element.closest("tr.cell[aria-posinset]").getAttribute("aria-posinset")),
                                        level: Number(element.tagName[1]),
                                        id: element.getAttribute("id")
                                    }
                                }
                            )
                        ).join(
                            (enter) => {
                                enter.append(
                                    (entry, i) => {
                                        let toc_row = select(self.templates.toc_row.cloneNode(true));
                                        toc_row.select(".level").text(entry.level)
                                        toc_row.select(".heading > a").text(entry.heading).attr("href", `#${entry.id}:`)
                                        toc_row.select(".cell > a").text(entry.cell).attr("href", `#c${entry.cell}`);
                                        return toc_row.node()
                                    }
                                );
                            }
                        )
                        return body.node()
                    }
                )
            }
        )
    }
    enterSummary() {
        let document = this.document;
        let nb = document.select("section").datum();
        // let loc = Array.from(document.selectAll(".cell[data-loc]")).map(
        //     (element) => {
        //         return Number(element.dataset.loc)
        //     }
        // ).reduce((a, b) => a + b)
        // document.select("#cells\\:loc").text(loc)

        // document.select("#cells\\:total").text(nb.cells.length)
        // document.select("#cells\\:md").text(nb.cells.filter(x => x.cell_type == "markdown").length)
        // document.select("#cells\\:code").text(nb.cells.filter(x => x.cell_type == "code").length)
        // document.select("#cells\\:outputs").text(
        //     nb.cells.filter(x => x.outputs).reduce((p, n) => p + n.outputs.length, 0)
        // )
        // let current_execution_count = 0, monotonic = true, partial = false;
        // for (const cell of nb.cells) {
        //     let trivial = cell.source.map(x => x.trim()).filter(Boolean).length;
        //     if (trivial) {
        //         if (cell.execution_count) { monotonic = false; break }
        //         continue
        //     }
        //     if (cell.cell_type == "code" && cell.execution_count) {
        //         if (cell.execution_count == current_execution_count + 1) {
        //             current_execution_count += 1
        //         } else {
        //             monotonic = false
        //             break
        //         }
        //     } else {
        //         partial = true
        //     }
        // }
        // let state = monotonic ? "executed in order" : "executed out of order";
        // state = partial ? "partially " + state : state;
        // document.select("#cells\\:state").text(state)
    }
    cellsKey(d, i) { return d == undefined ? i : d.id }
    outputKey(cell, d, i) { return [cell.execution_count, i] }

    markdownify(body) {
        let md = markdownit({
            html: true,
            linkify: true,
            highlight: function (str, lang) {
                if (lang && hljs.getLanguage(lang)) {
                    try {
                        return hljs.highlight(str, { language: lang }).value;
                    } catch (__) { }
                }
                return ''; // use external default escaping
            }
        }).use(footnote_plugin).use(deflist_plugin).render(body, self.md_env)
        return md
    }

}

function replaceAttachments(selection) {
    let cell = selection.node()?.closest("tr.cell")?.getAttribute("__data__");
    if (!cell) { return }
    select.selectAll("img").each(
        (_, j, nodes) => {
            let img = select(nodes[j]);
            if (cell.attachments && img.attr("src").startsWith("attachment:")) {
                let [_, attachment] = img.attr("src").split(":", 2);
                let bundle = cell.attachments[attachment];
                if (bundle) {
                    for ([type, bundle] of Object.entries(bundle)) {
                        img.attr("src", `data:${type};base64,${bundle}`)
                        break
                    }
                }
            }
        }
    )
}


function headingToId(heading, document) {
    let id = heading.attr("id") || slugify(heading.text());
    heading.attr("id", id)
    let a = document.createElement("a")
    select(a).attr("href", `#${id}:`).attr("id", `${id}:`).attr("aria-labelledby", id).text("🔗").classed("perma", true)
    let h = heading.node();
    // it makes sense for the link to be before the header.
    // mvoing forward int eh document confirms the location.
    h.insertBefore(a, h)
}
globalThis.d3 = { select: select, drag: drag };
globalThis.Notebook = Notebook



function onSubmit(event) {
    event.preventDefault();
    let cell_row = event.target.closest("tr.cell");
    let cell = Notebook.fromDomCell(select(cell_row))
    if (cell.cell_type == "markdown") {
        cell.outputs = [{ output_type: "display_data", data: { "text/markdown": cell.source } }]
        console.log("md", cell)
        nb.updateCell(select(cell_row).datum(cell))
    } else if (document.getElementById("BODY").dataset.runtime == "html") {
        cell.outputs = [{
            output_type: "display_data", data: {
                "text/html": cell.source
            }, metadata: {}
        }]

        nb.updateCell(select(event.target).datum(cell))
    }
}

function getNextCell(cell) {
    if (cell.nextElementSibling) {
        return cell.nextElementSibling
    } else if (cell.parentElement.nextElementSibling) {
        let next = cell.parentElement.nextElementSibling.querySelector("tr");
        if (next) { return next }
    }
    return cell
}
function textKeyboard(event) {
    console.log("ev")
    if ((event.keyCode == 10 || event.keyCode == 13) && event.ctrlKey) {
        event.target.form.requestSubmit();
    } else if ((event.keyCode == 10 || event.keyCode == 13) && event.shiftKey) {
        getNextCell(event.target.form.closest("tr.cell")).querySelector("textarea").focus()
        event.target.form.requestSubmit();
    } else if (event.keyCode == 27) {
        event.target.blur()
    }
}
try {
    globalThis.nb = new Notebook(document);
    document.getElementById("BODY").querySelectorAll("tr.cell").forEach(
        (element) => {
            element.querySelector("form").onsubmit = onSubmit
            element.querySelector("textarea").onkeyup = textKeyboard
        }
    )

} catch (error) { }
try {
    document;

    const ydoc = new Y.Doc()
    const provider = new WebrtcProvider('codemirror-demo-room', ydoc)

    document.querySelectorAll("tr.cell>td.source").forEach(
        (parent) => {
            let textarea = parent.querySelector("textarea");
            let updateListenerExtension = EditorView.updateListener.of((update) => {
                let textarea = parent.querySelector("textarea");
                if (update.docChanged) {
                    textarea.innerHTML = update.state.doc.toString();
                }
            });
            textarea.style.display = "none";

            let yText = ydoc.getText(textarea.value);
            let yUndoManager = new Y.UndoManager(yText)

            let editor = new EditorView({
                doc: textarea.value,
                extensions: [
                    // basicSetup,
                    updateListenerExtension,
                    keymap.of([indentWithTab]),
                    yCollab(yText, provider.awareness, { yUndoManager })
                ],
                parent: parent
            })

        }
    )
} catch (error) { console.error(error) }