пошук ресурсів за доменом `*.gov.ru`

results.csv

simple-google-crawler.js

import puppeteer from 'puppeteer';
import { createConnection } from "mariadb"
import _ from "lodash";

const { uniq, random } = _;

const sleep = (timeout) => new Promise((res) => setTimeout(() => res(), timeout * 1000));
const randomOfList = (list) => list[random(0, list.length -1)];

const userAgents = [
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4451.0 Safari/537.36",
    "Mozilla/5.0 (X11; Linux i686 (x86_64)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4451.0 Safari/537.36",
    "Mozilla/5.0 (Linux; arm_64; Android 10; ONEPLUS A6013) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 YaApp_Android/21.21 YaSearchBrowser/21.21 BroPP/1.0 Mobile Safari/537.36",
    "Mozilla/5.0 (Linux; Android 10; Joy 4 Build/QKQ1.200603.001; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/89.0.4389.90 Mobile Safari/537.36",
    "Mozilla/5.0 (Linux; Android 10; BMH-AN10 Build/HUAWEIBMH-N19; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/78.0.3904.108 Mobile Safari/537.36 com.yandex.zen/4.7.3.2704 (HUAWEI BMH-AN10; Android 10) ZenKit/1.40.7.5-internalNewdesign-Zen.8861",
    "Mozilla/5.0 (Linux; Android 9; itel L5002) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Mobile Safari/537.36",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 10.0; WOW64; Trident/7.0; .NET4.0C; .NET4.0E; .NET CLR 2.0.50727; .NET CLR 3.0.30729; .NET CLR 3.5.30729; InfoPath.2; ASU2JS)",
    "Mozilla/5.0 (Linux; U; Android 9; en-US; Lenovo L78051 Build/PKQ1.190110.001) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/78.0.3904.108 UCBrowser/13.3.8.1305 Mobile Safari/537.36",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7b) Gecko/20040421",
    "Mozilla/5.0 (Linux; Android 8.0.0; HTC_U-1u Build/OPR1.170623.032) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.86 Mobile Safari/537.36",
];

(async () => {
    const db = await createConnection({
        host: 'localhost',
        user: 'root',
        password: 'root',
        database: 'web_res'
    });

    const browser = await puppeteer.launch({
        headless: false
    });
    const page = await browser.newPage();

    for(let i=30; i<10**6; i++) {
        console.log("CURRENT_PAGE: ", i)
        const urls = await getUrlsByPage(i, page);

        for(const url of urls) {
            try {
                await db.query("INSERT INTO resources (url) VALUES (?)", [url]);
                console.log(url);
            } catch (e) {
                if(e.code !== 'ER_DUP_ENTRY'){
                    throw e;
                }
            }
        }

        await sleep(random(1,3));
    }
})();

async function getUrlsByPage(currentPage = 1, page) {
    const ua = randomOfList(userAgents);
    await page.setUserAgent(ua)

    await page.goto(`https://www.google.com/search?q=site:*.gov.ru&start=${currentPage*10}`, {
        waitUntil: 'networkidle2',

    });

    if((await page.$$('#captcha-form')).length > 0) {
        await sleep(20);
    } else {
        await sleep(1);
    }

    const list = await page.$$('#search a');

    const urls = [];
    for(let current of list) {
        const val = await current.getProperty('href');
        const parsed = await val.jsonValue();
        const pf = parsed.match(/(?!https?:\/\/(www\.)?)[\w\\.-_]+(?=gov\.ru)+/gi);

        urls.push(...pf
            .filter((c) => !/webcache/gi.test(c))
            .map((u) => u.split('//').pop())
            .map((u) => `${u}gov.ru`)
        )
    }

    return uniq(urls);
}

Good Evening. Please inform me if I need to request special permission from the Russian Federation for Internet Privacy and Supervision to document my findings while I attempt to defend myself as many people including entire governments, attempt to pin their crimes on an innocent unsuspecting laborer. I will be happy to work with you in any way to end all of this immediately. Thank you for your consideration in working peacefully together to resolve an enormous issue.