Skip to content

Instantly share code, notes, and snippets.

@retorquere
Created June 28, 2021 21:01
Show Gist options
  • Save retorquere/69975f9fdfab6606577be925c2df5004 to your computer and use it in GitHub Desktop.
Save retorquere/69975f9fdfab6606577be925c2df5004 to your computer and use it in GitHub Desktop.
/* content/auto-export
TypeError: Cannot read property 'value' of undefined
at isSkip (file:///Users/emile/github/better-bibtex/node_modules/.pnpm/[email protected]/node_modules/estrace/lib/trace/plugin-trace/index.js:167:20)
at Function (file:///Users/emile/github/better-bibtex/node_modules/.pnpm/[email protected]/node_modules/estrace/lib/trace/plugin-trace/index.js:36:17)
at NodePath._call (/Users/emile/github/better-bibtex/node_modules/.pnpm/@[email protected]/node_modules/@babel/traverse/lib/path/context.js:53:20)
at NodePath.call (/Users/emile/github/better-bibtex/node_modules/.pnpm/@[email protected]/node_modules/@babel/traverse/lib/path/context.js:40:17)
at NodePath.visit (/Users/emile/github/better-bibtex/node_modules/.pnpm/@[email protected]/node_modules/@babel/traverse/lib/path/context.js:90:31)
at TraversalContext.visitQueue (/Users/emile/github/better-bibtex/node_modules/.pnpm/@[email protected]/node_modules/@babel/traverse/lib/context.js:99:16)
at TraversalContext.visitMultiple (/Users/emile/github/better-bibtex/node_modules/.pnpm/@[email protected]/node_modules/@babel/traverse/lib/context.js:68:17)
at TraversalContext.visit (/Users/emile/github/better-bibtex/node_modules/.pnpm/@[email protected]/node_modules/@babel/traverse/lib/context.js:125:19)
at Function.traverse.node (/Users/emile/github/better-bibtex/node_modules/.pnpm/@[email protected]/node_modules/@babel/traverse/lib/index.js:76:17)
at NodePath.visit (/Users/emile/github/better-bibtex/node_modules/.pnpm/@[email protected]/node_modules/@babel/traverse/lib/path/context.js:97:18)
*/
/Components.utils.import("resource://gre/modules/FileUtils.jsm");
import { log } from "./logger";
import { Events } from "./events";
import { DB } from "./db/main";
import { DB as Cache, selector as cacheSelector } from "./db/cache";
import { $and } from "./db/loki";
import { Translators } from "./translators";
import { Preference } from "../gen/preferences";
import { schema } from "../gen/preferences/meta";
import * as ini from "ini";
import fold2ascii from "fold-to-ascii";
import { pathSearch } from "./path-search";
import { Scheduler } from "./scheduler";
import { flash } from "./flash";
class Git {
constructor(parent) {
this.enabled = false;
if (parent) {
this.git = parent.git;
}
}
async init() {
this.git = await pathSearch("git");
return this;
}
async repo(bib) {
const repo = new Git(this);
if (!this.git)
return repo;
switch (Preference.git) {
case "off":
return repo;
case "always":
try {
repo.path = OS.Path.dirname(bib);
} catch (err) {
log.error("git.repo:", err);
return repo;
}
break;
case "config":
let config = null;
for (let root = OS.Path.dirname(bib); await OS.File.exists(root) && (await OS.File.stat(root)).isDir && root !== OS.Path.dirname(root); root = OS.Path.dirname(root)) {
config = OS.Path.join(root, ".git");
if (await OS.File.exists(config) && (await OS.File.stat(config)).isDir)
break;
config = null;
}
if (!config)
return repo;
repo.path = OS.Path.dirname(config);
config = OS.Path.join(config, "config");
if (!await OS.File.exists(config) || (await OS.File.stat(config)).isDir) {
return repo;
}
try {
const enabled = ini.parse(Zotero.File.getContents(config))['zotero "betterbibtex"']?.push;
if (enabled !== "true" && enabled !== true)
return repo;
} catch (err) {
log.error("git.repo: error parsing config", config.path, err);
return repo;
}
break;
default:
log.error("git.repo: unexpected git config", Preference.git);
return repo;
}
const sep = Zotero.isWin ? "\\" : "/";
if (bib[repo.path.length] !== sep)
throw new Error(`git.repo: ${bib} not in directory ${repo.path} (${bib[repo.path.length]} vs ${sep})?!`);
repo.enabled = true;
repo.bib = bib.substring(repo.path.length + 1);
return repo;
}
async pull() {
if (!this.enabled)
return;
try {
const warning = await this.exec(this.git, ["-C", this.path, "pull"]);
if (warning)
flash("autoexport git pull warning", `${this.quote(this.path)}: ${warning}`, 1);
} catch (err) {
flash("autoexport git pull failed", `${this.quote(this.path)}: ${err.message}`, 1);
log.error(`could not pull in ${this.path}:`, err);
}
}
async push(msg) {
if (!this.enabled)
return;
const warnings = [];
try {
warnings.push(await this.exec(this.git, ["-C", this.path, "add", this.bib]));
warnings.push(await this.exec(this.git, ["-C", this.path, "commit", "-m", msg]));
warnings.push(await this.exec(this.git, ["-C", this.path, "push"]));
const warning = warnings.find((w) => w);
if (warning)
flash("autoexport git pull warning", `${this.quote(this.path)}: ${warning}`, 1);
} catch (err) {
flash("autoexport git push failed", `${this.quote(this.path)}: ${err.message}`, 1);
log.error(`could not push ${this.bib} in ${this.path}`, err);
}
}
quote(cmd, args) {
return [cmd].concat(args || []).map((arg) => arg.match(/['"]|\s/) ? JSON.stringify(arg) : arg).join(" ");
}
async exec(cmd, args) {
if (typeof cmd === "string")
cmd = new FileUtils.File(cmd);
if (!cmd.isExecutable())
throw new Error(`${cmd.path} is not an executable`);
const proc = Components.classes["@mozilla.org/process/util;1"].createInstance(Components.interfaces.nsIProcess);
proc.init(cmd);
return new Promise((resolve, reject) => {
proc.runwAsync(args, args.length, { observe: function(subject, topic) {
if (topic !== "process-finished") {
reject(new Error(`${this.quote(cmd.path, args)} failed, exit status: ${topic}`));
} else if (proc.exitValue !== 0) {
resolve(`${this.quote(cmd.path, args)} returned exit status ${proc.exitValue}`);
} else {
resolve("");
}
} });
});
}
}
const git = new Git();
function scrub(ae) {
const translator = schema.translator[Translators.byId[ae.translatorID].label];
for (const k of schema.autoExport.preferences.concat(schema.autoExport.displayOptions)) {
if (!translator.types[k]) {
delete ae[k];
log.debug("ae: stripping", k, "from", ae);
}
}
return ae;
}
if (Preference.autoExportDelay < 1)
Preference.autoExportDelay = 1;
const queue = new class TaskQueue {
constructor() {
this.scheduler = new Scheduler("autoExportDelay", 1e3);
this.started = false;
this.pause();
}
start() {
if (this.started)
return;
this.started = true;
if (Preference.autoExport === "immediate")
this.resume();
const idleService = Components.classes["@mozilla.org/widget/idleservice;1"].getService(Components.interfaces.nsIIdleService);
idleService.addIdleObserver(this, Preference.autoExportIdleWait);
Zotero.Notifier.registerObserver(this, ["sync"], "BetterBibTeX", 1);
}
init(autoexports) {
this.autoexports = autoexports;
}
pause() {
this.scheduler.paused = true;
}
resume() {
this.scheduler.paused = false;
}
add(ae) {
const $loki = typeof ae === "number" ? ae : ae.$loki;
Events.emit("export-progress", 0, Translators.byId[ae.translatorID].label, $loki);
this.scheduler.schedule($loki, () => {
this.run($loki).catch((err) => log.error("autoexport failed:", { $loki }, err));
});
}
cancel(ae) {
const $loki = typeof ae === "number" ? ae : ae.$loki;
this.scheduler.cancel($loki);
}
async run($loki) {
await Zotero.BetterBibTeX.ready;
const ae = this.autoexports.get($loki);
Events.emit("export-progress", 0, Translators.byId[ae.translatorID].label, $loki);
if (!ae)
throw new Error(`AutoExport ${$loki} not found`);
ae.status = "running";
this.autoexports.update(scrub(ae));
const started = Date.now();
log.debug("auto-export", ae.type, ae.id, "started");
try {
let scope;
switch (ae.type) {
case "collection":
scope = { type: "collection", collection: ae.id };
break;
case "library":
scope = { type: "library", id: ae.id };
break;
default:
throw new Error(`Unexpected auto-export scope ${ae.type}`);
}
const repo = await git.repo(ae.path);
await repo.pull();
const displayOptions = {
exportNotes: ae.exportNotes,
useJournalAbbreviation: ae.useJournalAbbreviation
};
for (const pref of schema.translator[Translators.byId[ae.translatorID].label].preferences) {
displayOptions[`preference_${pref}`] = ae[pref];
}
displayOptions.auto_export_id = ae.$loki;
const jobs = [{ scope, path: ae.path }];
if (ae.recursive) {
const collections = scope.type === "library" ? Zotero.Collections.getByLibrary(scope.id, true) : Zotero.Collections.getByParent(scope.collection, true);
const ext = `.${Translators.byId[ae.translatorID].target}`;
const root = scope.type === "collection" ? scope.collection : false;
const dir = OS.Path.dirname(ae.path);
const base = OS.Path.basename(ae.path).replace(new RegExp(`${ext.replace(/[-/\\^$*+?.()|[\]{}]/g, "\\$&")}$`), "");
const autoExportPathReplaceDiacritics = Preference.autoExportPathReplaceDiacritics;
const autoExportPathReplaceDirSep = Preference.autoExportPathReplaceDirSep;
const autoExportPathReplaceSpace = Preference.autoExportPathReplaceSpace;
for (const collection of collections) {
const path = OS.Path.join(dir, [base].concat(this.getCollectionPath(collection, root)).map((p) => p.replace(/[<>:'"/\\|?*\u0000-\u001F]/g, "")).map((p) => p.replace(/ +/g, autoExportPathReplaceSpace || "")).map((p) => autoExportPathReplaceDiacritics ? fold2ascii.foldMaintaining(p) : p).join(autoExportPathReplaceDirSep || "-") + ext);
jobs.push({ scope: { type: "collection", collection: collection.id }, path });
}
}
await Promise.all(jobs.map((job) => Translators.exportItems(ae.translatorID, displayOptions, job.scope, job.path)));
await repo.push(Zotero.BetterBibTeX.getString("Preferences.auto-export.git.message", { type: Translators.byId[ae.translatorID].label.replace("Better ", "") }));
ae.error = "";
log.debug("auto-export", ae.type, ae.id, "took", Date.now() - started, "msecs");
} catch (err) {
log.error("auto-export", ae.type, ae.id, "failed:", ae, err);
ae.error = `${err}`;
}
ae.status = "done";
log.debug("auto-export done");
this.autoexports.update(scrub(ae));
}
getCollectionPath(coll, root) {
let path = [coll.name];
if (coll.parentID && coll.parentID !== root)
path = this.getCollectionPath(Zotero.Collections.get(coll.parentID), root).concat(path);
return path;
}
observe(_subject, topic, _data) {
if (!this.started || Preference.autoExport === "off")
return;
switch (topic) {
case "back":
case "active":
if (Preference.autoExport === "idle")
this.pause();
break;
case "idle":
this.resume();
break;
default:
log.error("Unexpected idle state", topic);
break;
}
}
notify(action, type) {
if (!this.started || Preference.autoExport === "off")
return;
switch (`${type}.${action}`) {
case "sync.start":
this.pause();
break;
case "sync.finish":
this.resume();
break;
default:
log.error("Unexpected Zotero notification state", { action, type });
break;
}
}
}();
export const AutoExport = new class _AutoExport {
constructor() {
this.progress = new Map();
Events.on("libraries-changed", (ids) => this.schedule("library", ids));
Events.on("libraries-removed", (ids) => this.remove("library", ids));
Events.on("collections-changed", (ids) => this.schedule("collection", ids));
Events.on("collections-removed", (ids) => this.remove("collection", ids));
Events.on("export-progress", (percent, _translator, ae) => {
if (typeof ae === "number")
this.progress.set(ae, percent);
});
}
async init() {
await git.init();
this.db = DB.getCollection("autoexport");
queue.init(this.db);
for (const ae of this.db.find({ status: { $ne: "done" } })) {
queue.add(ae);
}
this.db.on(["delete"], (ae) => {
this.progress.delete(ae.$loki);
});
if (Preference.autoExport === "immediate") {
queue.resume();
}
}
start() {
queue.start();
}
add(ae, schedule = false) {
const translator = schema.translator[Translators.byId[ae.translatorID].label];
for (const pref of translator.preferences) {
ae[pref] = Preference[pref];
}
for (const option of translator.displayOptions) {
ae[option] = ae[option] || false;
}
this.db.removeWhere({ path: ae.path });
this.db.insert(scrub(ae));
git.repo(ae.path).then((repo) => {
if (repo.enabled || schedule)
this.schedule(ae.type, [ae.id]);
}).catch((err) => {
log.error("AutoExport.add:", err);
});
}
schedule(type, ids) {
for (const ae of this.db.find({ $and: [{ type: { $eq: type } }, { id: { $in: ids } }] })) {
queue.add(ae);
}
}
remove(type, ids) {
for (const ae of this.db.find({ $and: [{ type: { $eq: type } }, { id: { $in: ids } }] })) {
queue.cancel(ae);
this.db.remove(ae);
}
}
run(id) {
queue.run(id).catch((err) => log.error("AutoExport.run:", err));
}
async cached($loki) {
log.debug("cache-rate: calculating cache rate for", $loki);
const start = Date.now();
const ae = this.db.get($loki);
const itemTypeIDs = ["attachment", "note", "annotation"].map((type) => {
try {
return Zotero.ItemTypes.getID(type);
} catch (err) {
return void 0;
}
});
const itemIDs = new Set();
await this.itemIDs(ae, ae.id, itemTypeIDs, itemIDs);
if (itemIDs.size === 0)
return 100;
const options = {
exportNotes: !!ae.exportNotes,
useJournalAbbreviation: !!ae.useJournalAbbreviation
};
const prefs = schema.translator[Translators.byId[ae.translatorID].label].preferences.reduce((acc, pref) => {
acc[pref] = ae[pref];
return acc;
}, {});
const label = Translators.byId[ae.translatorID].label;
const cached = {
serialized: Cache.getCollection("itemToExportFormat").find({ itemID: { $in: [...itemIDs] } }).length,
export: Cache.getCollection(label).find($and({ ...cacheSelector(label, options, prefs), $in: itemIDs })).length
};
log.debug("cache-rate: cache rate for", $loki, { ...cached, items: itemIDs.size }, "took", Date.now() - start);
return Math.min(Math.round(100 * (cached.serialized + cached.export) / (itemIDs.size * 2)), 100);
}
async itemIDs(ae, id, itemTypeIDs, itemIDs) {
let items;
if (ae.type === "collection") {
const coll = await Zotero.Collections.getAsync(id);
if (ae.recursive) {
for (const collID of coll.getChildren(true)) {
await this.itemIDs(ae, collID, itemTypeIDs, itemIDs);
}
}
items = coll.getChildItems();
} else if (ae.type === "library") {
items = await Zotero.Items.getAll(id);
}
items.filter((item) => !itemTypeIDs.includes(item.itemTypeID)).forEach((item) => itemIDs.add(item.id));
}
}();
Events.on("preference-changed", (pref) => {
if (pref !== "autoExport")
return;
switch (Preference.autoExport) {
case "immediate":
queue.resume();
break;
default:
queue.pause();
}
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment