crawl-bikeblog.js

//run with: for file in ~/Downloads/bikeblog/*.html; do node index.js "$file" ~/dev/nonwork/bikeblog/bikeblog/content/; done
const cheerio = require("cheerio");
const slug = require("slug");
const fs = require("fs");
const {execSync} = require("child_process");
const path = require("path");
const file = process.argv[2];
const targetbasedir = process.argv[3];
const dir = path.dirname(file);
console.log(file, targetbasedir);

const contents = fs.readFileSync(file, {encoding: 'utf8'});
const doc = cheerio.load(contents);
const title = doc("article header h1").text();
const date = doc("article header span.date time").attr("datetime");
const content = doc("article div.entry-content > p").toArray().map((p) => {
	const img = doc("img", p).attr("src");
	if(img) {
		return {image: path.resolve(dir, unescape(img))};
	}
	else {
		return {text: doc(p).text()};
	}
});
const targetdir = slug(title.toLowerCase());

execSync(`mkdir ${targetdir}`, {cwd: targetbasedir});
content.reduce((imageCounter, p) => {
	if(p.image) {
		imageCounter += 1;
		const newfile = `${imageCounter}.jpg`;
		execSync(`cp '${p.image}' ${newfile}`, {cwd: path.resolve(targetbasedir, targetdir)})
		p.image = newfile;
	}
	return imageCounter;
}, 0);

let buffer = "";
function append(str) {
	buffer = buffer + str + "\n";
}

append(`+++`);
append(`title = "${title}"`);
append(`date = ${date}`);
append(`+++`);
append("");
content.forEach(p => {
	if (p.text) {
		append(p.text);
		append("");
	}
	else if(p.image) {
		append(`<img src="${p.image}" />`);
		append("");
	}
});

fs.writeFileSync(path.resolve(targetbasedir, targetdir+"/index.md"), buffer, {encoding: 'utf8'});