Skip to content

Instantly share code, notes, and snippets.

@felipecustodio
Last active October 28, 2024 17:49
Show Gist options
  • Save felipecustodio/3df3c02f977fbcf04a0d7df0772df2ef to your computer and use it in GitHub Desktop.
Save felipecustodio/3df3c02f977fbcf04a0d7df0772df2ef to your computer and use it in GitHub Desktop.
Generate Scrapy command on Zyte
// ==UserScript==
// @name Zyte Scrapy Command Generator
// @namespace http://tampermonkey.net/
// @version 1.0
// @description Adds a button to generate and copy 'scrapy crawl' commands on Zyte
// @author Felipe Custódio
// @match https://app.zyte.com/p/*
// @grant none
// ==/UserScript==
(function() {
'use strict';
function getXPathText(xpath) {
const result = document.evaluate(
xpath,
document,
null,
XPathResult.ORDERED_NODE_SNAPSHOT_TYPE,
null
);
let texts = [];
for (let i = 0; i < result.snapshotLength; i++) {
const node = result.snapshotItem(i);
const text = node.textContent.trim();
if (text) {
texts.push(text);
}
}
return texts;
}
function generateScrapyCommand() {
const spiderName = getXPathText("//th[text()='Spider']/following-sibling::td/a/text()")[0];
const args = getXPathText("//th[text()='Arguments']/following-sibling::td//text()");
const cleanArgs = args.filter(arg => arg.trim());
let command = `scrapy crawl ${spiderName}`;
for (let i = 0; i < cleanArgs.length; i += 2) {
if (i + 1 < cleanArgs.length) {
const key = cleanArgs[i].trim();
const value = cleanArgs[i + 1].trim().replace('=', '');
command += ` -a ${key}=${value}`;
}
}
return command;
}
function copyToClipboard(text) {
navigator.clipboard.writeText(text).then(
() => {
const button = document.querySelector('.copy-command-btn .btn-content-text');
const originalText = button.textContent;
button.textContent = 'Copied!';
setTimeout(() => {
button.textContent = originalText;
}, 2000);
}
);
}
function createCopyButton() {
const header = document.querySelector('zy-page-header .actions');
if (!header) return;
const buttonWrapper = document.createElement('zy-button');
buttonWrapper.className = 'copy-command-btn';
const controlElement = document.createElement('div');
controlElement.className = 'mat-ripple control-element default rounded medium';
const btnContent = document.createElement('div');
btnContent.className = 'btn-content';
const btnContentText = document.createElement('div');
btnContentText.className = 'btn-content-text';
btnContentText.textContent = 'Copy Scrapy Command';
btnContent.appendChild(btnContentText);
controlElement.appendChild(btnContent);
buttonWrapper.appendChild(controlElement);
buttonWrapper.addEventListener('click', () => {
const command = generateScrapyCommand();
copyToClipboard(command);
});
// Insert button before the last button in the header
header.insertBefore(buttonWrapper, header.lastChild);
}
// Check periodically for the header and add button when found
const interval = setInterval(() => {
if (document.querySelector('zy-page-header .actions')) {
createCopyButton();
clearInterval(interval);
}
}, 1000);
})();
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment