Created
June 19, 2023 08:30
-
-
Save adrienjoly/10f4e434c7a8e3561b46b819869c0bc0 to your computer and use it in GitHub Desktop.
Login and download a HTTPS response using Puppeteer and request interceptors.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// Note: this script does not work anymore on hackmd.io, because of recaptcha protection. | |
// But the same logic can be reused on other websites, e.g. for backup automation. | |
const puppeteer = require("puppeteer"); | |
const { DEBUG } = process.env; // for verbose request logs, pass DEBUG='*' | |
/** @returns a `untilRequest()` method that resolves when `targetURL` is requested by `page`. */ | |
const interceptRequests = async (page) => { | |
await page.setRequestInterception(true); | |
return { | |
untilRequest: (targetURL) => | |
new Promise((resolve) => { | |
const handler = (req) => { | |
if (req.url() === targetURL) { | |
req.abort(); | |
page.off("request", handler); | |
page.setRequestInterception(false).then(() => resolve(req)); | |
} else { | |
req.continue(); | |
} | |
}; | |
page.on("request", handler); | |
}), | |
}; | |
}; | |
(async () => { | |
const browser = await puppeteer.launch(); | |
const page = await browser.newPage(); | |
// login | |
await page.goto("https://hackmd.io/login", { waitUntil: "networkidle2" }); | |
await page.type("input[type=email]", HACKMD_EMAIL); | |
await page.type("input[type=password]", HACKMD_PWD); | |
if (DEBUG) await page.screenshot({ path: "login-scren.png" }); | |
const intercept = await interceptRequests(page); | |
await page.click("input[type=submit]"); | |
const interceptedRequest = await intercept.untilRequest( | |
"https://hackmd.io/me" | |
); | |
// request the backup | |
const cookies = await page.cookies(); | |
const response = await fetch("https://hackmd.io/exportAllNotes", { | |
encoding: null, | |
headers: { | |
...interceptedRequest.headers, | |
Cookie: cookies.map((ck) => ck.name + "=" + ck.value).join(";"), | |
}, | |
}); | |
// download the backup | |
const blob = await response.blob(); | |
console.warn(`downloading ${blob.size} bytes, type: ${blob.type}...`); | |
process.stdout.write(Buffer.from(await blob.arrayBuffer())); | |
await browser.close(); | |
})(); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment