Last active
April 7, 2023 10:49
-
-
Save martinratinaud/2cb888204ae1a47fb32eaf68619b56bf to your computer and use it in GitHub Desktop.
Check backlink existence and rel after buying them
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import Scraper from './Scraper'; | |
| const scraper = new Scraper(); | |
| import colors, { Color } from 'colors'; | |
| declare global { | |
| interface String { | |
| checking: Color; | |
| notfound: Color; | |
| } | |
| } | |
| colors.setTheme({ | |
| checking: ['green', 'bold'], | |
| notfound: ['red', 'bold'], | |
| }); | |
| const backlinkChecker = async (jsonPath: string) => { | |
| const notFound: string[] = []; | |
| const toCheck = require(jsonPath); | |
| console.log(''.grey); | |
| console.log('Starting backlinks check'.grey); | |
| for (const check of toCheck) { | |
| const { date, target, backlink } = check; | |
| console.log('Checking', target.checking); | |
| try { | |
| const dom = await scraper.getUrlDom(target); | |
| const backlinks = dom.window.document.querySelectorAll(`a[href*="${backlink}"]`); | |
| if (backlinks.length === 0) { | |
| console.log('❌', backlink.notfound, date); | |
| notFound.push(check); | |
| } | |
| [...backlinks].forEach((link) => { | |
| const rel = link.getAttribute('rel'); | |
| if (rel?.includes('nofollow')) { | |
| notFound.push(check); | |
| console.log('❌', backlink.notfound, 'nofollow'); | |
| } else { | |
| console.log('✅', target); | |
| } | |
| }); | |
| } catch (error) { | |
| console.error('Error:', error); | |
| } | |
| await new Promise((resolve) => setTimeout(resolve, 1000)); | |
| } | |
| console.log(''.grey); | |
| console.log(notFound.length, 'Links not found'); | |
| console.log('\n', notFound); | |
| }; | |
| export default backlinkChecker; |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [ | |
| { | |
| "date": "", | |
| "target": "https://cryptomode.com/should-you-start-staking-shiba-inu/", | |
| "backlink": "https://stakingcrypto.io/stake/SHIB/shiba-inu" | |
| }, | |
| { | |
| "date": "", | |
| "target": "https://lespepitestech.com/startup-collection/crypto", | |
| "backlink": "https://stakingcrypto.io" | |
| } | |
| ] |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import axios, { AxiosRequestConfig } from 'axios'; | |
| import { JSDOM } from 'jsdom'; | |
| export default class Scraper { | |
| public JSDOM = JSDOM; | |
| constructor() {} | |
| async getUrl(url: string, axiosConfig?: AxiosRequestConfig) { | |
| const { headers, ...options } = axiosConfig || {}; | |
| const config: AxiosRequestConfig = { | |
| method: 'GET', | |
| headers: { | |
| // https://oxylabs.io/blog/5-key-http-headers-for-web-scraping | |
| 'Accept-Encoding': '*', | |
| Accept: 'test/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', | |
| Cookie: '', | |
| ...(headers || {}), | |
| }, | |
| timeout: 5000, | |
| ...options, | |
| }; | |
| const { data } = await axios(url, config); | |
| return data; | |
| } | |
| async getUrlDom(url: string, axiosConfig?: AxiosRequestConfig, options?: GetUrlConfig) { | |
| const data = await this.getUrl(url, axiosConfig, options); | |
| return new JSDOM(data); | |
| } | |
| } |
Author
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Here is what it will look like in the terminal
