Phone chooser

chooser.js

'use strict';

// На текущий момент страница со списком устройств на сайте /e/ поменяла дизайн,
// так что проверка поддерживает ли устройство /e/ не работает. Но раньше работало :)

const fs = require('fs').promises;
const path = require('path');
const axios = require('axios');
const cheerio = require('cheerio');
const Iconv = require('iconv').Iconv;

const cacheDir = path.join(__dirname, '.cache');
const cachedIconvs = Object.create(null);

function convertEncoding(buffer, encoding) {
    if (!cachedIconvs[encoding]) {
        cachedIconvs[encoding] = new Iconv(encoding, 'utf8');
    }

    return cachedIconvs[encoding].convert(buffer).toString();
}

async function retrieveRawPageContent(url) {
    const cachedPath = path.join(cacheDir, encodeURIComponent(url));
    const isCached = await fs.access(cachedPath).then(() => true).catch(() => false);
    let responseBody;

    if (isCached) {
        responseBody = await fs.readFile(cachedPath, 'utf8');
    } else {
        console.log(`Retrieving "${url}"...`);

        const response = await axios.get(url, {
            responseType: 'arraybuffer'
        });

        let encoding = 'utf8';

        if (response.headers['content-type']) {
            const mt = response.headers['content-type'].match(/charset\s*=\s*(.+)/);

            if (mt) {
                encoding = mt[1].trim().toLowerCase();
            }
        }

        if (encoding === 'utf8' || encoding === 'utf-8') {
            responseBody = response.data.toString('utf-8');
        } else {
            responseBody = convertEncoding(response.data, encoding);
        }

        await fs.writeFile(cachedPath, Buffer.from(responseBody));
    }

    return responseBody;
}

async function retrieveDomPageContent(url) {
    return cheerio.load(await retrieveRawPageContent(url));
}

async function retrieveJsonPageContent(url) {
    return JSON.parse(await retrieveRawPageContent(url));
}

async function retrieveLineageOsSupportedDevices() {
    const $ = await retrieveDomPageContent('https://download.lineageos.org/');
    const result = [];

    $('a.collapsible-header').each((_, vendorElem) => {
        const $vendor = $(vendorElem);
        const vendorName = $vendor.text();

        $vendor.parent().find('a.device-link').each((_, deviceElem) => {
            const $device = $(deviceElem);
            const deviceName = $device.find(':not(.device-model)').text();

            result.push({
                phoneName: vendorName + ' ' + deviceName,
                phoneModel: $device.find('.device-model').text().toLowerCase().trim(),
            });
        });
    });

    return result;
}

async function retrieveMicrogSupportedDevicesMap() {
    const $ = await retrieveDomPageContent('https://download.lineage.microg.org/');
    const result = Object.create(null);

    $('a').each((_, elem) => {
        const $elem = $(elem);
        const phoneModel = $elem.text().toLowerCase().trim();

        if ($elem.attr('href').toLowerCase().trim() === `/${phoneModel}/`) {
            result[phoneModel] = true;
        }
    });

    return result;
}

async function retrieveESupportedDevicesMap() {
    const $ = await retrieveDomPageContent('https://doc.e.foundation/devices/');
    const result = Object.create(null);

    $('a').each((_, elem) => {
        const $elem = $(elem);
        const phoneModel = $elem.text().toLowerCase().trim();

        if ($elem.attr('href').toLowerCase().trim() === phoneModel) {
            result[phoneModel] = true;
        }
    });

    return result;
}

function makeMatcherList(value) {
    return value.toLowerCase()
        .replace(/[^0-9a-z ]/g, ' ')
        .replace(/[ ]{2,}/g, ' ')
        .trim()
        .split(/ /);
}

/*
function computeMatchRatio(srcMl, compareValue) {
    if (!srcMl.length) {
        return 0;
    }

    const compareMl = makeMatcherList(compareValue);
    let matchCount = 0;

    for (part of srcMl) {
        if (compareMl.includes(part)) {
            ++matchCount;
        }
    }

    return matchCount / srcMl.length;
}
*/

// https://4pda.ru/devdb/search?s=Asus%20ZenFone%206
// https://4pda.ru/forum/index.php?act=search&source=all&forums%5B%5D=570&x=0&y=0&subforums=1&query=Asus+ZenFone+6+ZS630KL
// phoneName.replace(/\s+\([^)]+\)\s*$/, ''); -- убрать то, что в скобочках

async function retrieve4PdaPhoneSpecs(phoneName) {
    let $ = await retrieveDomPageContent(
        'https://4pda.ru/forum/index.php?act=search&source=all&forums%5B%5D=570&x=0&y=0&subforums=1&query=' +
        encodeURIComponent(makeMatcherList(phoneName).join(' ')))

    let deviceUrl = null;

    $('.cat_name > a').each((_, topicLinkElem) => {
        const $topicLink = $(topicLinkElem);
        const $deviceLink = $topicLink.parent().parent().find('.post_body > a[href*=devdb]:contains("Описание")');

        if ($deviceLink.length) {
            deviceUrl = $deviceLink.attr('href');
            return false;
        }
    });

    if (deviceUrl === null) {
        $('.ddb-ft-title a[href*=devdb]').each((_, deviceLinkElem) => {
            deviceUrl = $(deviceLinkElem).attr('href');
            return false;
        });
    }

    // const $ = await retrieveDomPageContent('https://4pda.ru/devdb/search?s=' + encodeURIComponent(phoneName));
    // let deviceUrl = null;
    //
    // $('a.dev-compare-item-link').parent().parent().find('.name > a[href*=devdb]').each((_, deviceLinkElem) => {
    //     deviceUrl = $(deviceLinkElem).attr('href');
    //     return false;
    // });

    if (deviceUrl === null) {
        return {
            osText: '',
            yearText: '',
            osCompare: 0,
            year: 0,
        };
    }

    $ = await retrieveDomPageContent(deviceUrl.startsWith('//') ? `https:${deviceUrl}` : deviceUrl);
    let osText = null;
    let yearText = null;

    $('.specifications-row > dt:contains("ОС")').parent().find('dd').each((_, ddElem) => {
        osText = $(ddElem).text();
        return false;
    });

    $('.specifications-row > dt:contains("Год выпуска")').parent().find('dd').each((_, ddElem) => {
        yearText = $(ddElem).text();
        return false;
    });

    if (osText === null || yearText === null) {
        return {
            osText: '',
            yearText: '',
            osCompare: 0,
            year: 0,
        };
    }

    const mtOs = osText.match(/^Android\s+(\d+)(?:\.(\d+))?/);

    return {
        isOn4pda: true,
        osText,
        yearText,
        osCompare: mtOs ? (parseInt(mtOs[1], 10) * 10 + parseInt(mtOs[2] || '0', 10)) : 0,
        year: parseInt(yearText, 10),
    };
}

// https://www.onliner.by/sdapi/catalog.api/search/products?query=Asus+Zenfone+6+ZS630KL

async function retrieveOnlinerPhoneSpecs(phoneName) {
    const phoneNameNormalized = makeMatcherList(phoneName).join(' ');

    const json = await retrieveJsonPageContent(
        'https://www.onliner.by/sdapi/catalog.api/search/products?query=' +
        encodeURIComponent(phoneNameNormalized))

    let deviceUrl = null;

    for (let product of json.products) {
        const fullNameNormalized = makeMatcherList(product.full_name).join(' ');

        if (fullNameNormalized.startsWith(phoneNameNormalized) && product.html_url.indexOf('phoneaccum') === -1) {
            deviceUrl = product.html_url;
            break;
        }
    }

    if (deviceUrl === null) {
        return {
            osText: '',
            yearText: '',
            osCompare: 0,
            year: 0,
        };
    }

    const $ = await retrieveDomPageContent(deviceUrl);
    let osText = null;
    let yearText = null;

    $('.product-specs__table > tbody > tr > td:contains("Версия операционной системы")').parent().find('td > span.value__text').each((_, elem) => {
        osText = $(elem).text();
        return false;
    });

    $('.product-specs__table > tbody > tr > td:contains("Дата выхода на рынок")').parent().find('td > span.value__text').each((_, elem) => {
        yearText = $(elem).text();
        return false;
    });

    if (osText === null || yearText === null) {
        return {
            osText: '',
            yearText: '',
            osCompare: 0,
            year: 0,
        };
    }

    const mtOs = osText.match(/^Android\s+(\d+)(?:\.(\d+))?/);

    return {
        isOnOnliner: true,
        osText,
        yearText,
        osCompare: mtOs ? (parseInt(mtOs[1], 10) * 10 + parseInt(mtOs[2] || '0', 10)) : 0,
        year: parseInt(yearText, 10),
    };
}

async function retrievePhoneSpecs(phoneName) {
    const specsA = await retrieve4PdaPhoneSpecs(phoneName);
    const specsB = await retrieveOnlinerPhoneSpecs(phoneName);

    if (specsA.year === 0 && specsB.year === 0) {
        return specsA;
    }

    if (specsA.year === 0) {
        return specsB;
    }

    if (specsB.year === 0) {
        return specsA;
    }

    return {
        isOn4pda: true,
        isOnOnliner: true,
        osText: (specsA.osText.startsWith(specsB.osText) || specsB.osText.startsWith(specsA.osText))
            ? specsA.osText
            : `${specsA.osText}, ${specsB.osText}`,
        yearText: (specsA.yearText.startsWith(specsB.yearText) || specsB.yearText.startsWith(specsA.yearText))
            ? specsA.yearText
            : `${specsA.yearText}, ${specsB.yearText}`,
        osCompare: Math.min(specsA.osCompare, specsB.osCompare),
        year: Math.min(specsA.year, specsB.year),
    };
}

async function process() {
    await fs.mkdir(cacheDir, { recursive: true });
    const lineageOsSupportedDevices = await retrieveLineageOsSupportedDevices();
    const microgSupportedDevicesMap = await retrieveMicrogSupportedDevicesMap();
    const eSupportedDevicesMap = await retrieveESupportedDevicesMap();
    const specList = [];

    for (const device of lineageOsSupportedDevices) {
        const specs = await retrievePhoneSpecs(device.phoneName);
        const hasMicrog = !!microgSupportedDevicesMap[device.phoneModel];
        const hasE = !!eSupportedDevicesMap[device.phoneModel];

        specList.push(Object.assign({
            hasMicrog,
            hasE,
            hasMicrogOrECompare: (hasMicrog || hasE) ? 1 : 0,
        }, device, specs));
    }

    specList.sort((a, b) => {
        const microgOrECmp = b.hasMicrogOrECompare - a.hasMicrogOrECompare;

        if (microgOrECmp !== 0) {
            return microgOrECmp;
        }

        const yearCmp = b.year - a.year;

        if (yearCmp !== 0) {
            return yearCmp;
        }

        return b.osCompare - a.osCompare;
    });

    for (const spec of specList) {
        const parts = [];

        if (spec.hasMicrog) {
            parts.push('microG');
        } else {
            parts.push('------');
        }

        if (spec.hasE) {
            parts.push('E');
        } else {
            parts.push('-');
        }

        if (spec.isOnOnliner) {
            parts.push('Onliner');
        } else {
            parts.push('-------');
        }

        if (spec.isOn4pda) {
            parts.push('4pda');
        } else {
            parts.push('----');
        }

        parts.push(spec.yearText === '' ? '-' : spec.yearText);
        parts.push(spec.osText === '' ? '-' : spec.osText);
        parts.push(spec.phoneModel);
        parts.push(spec.phoneName);

        console.log(parts.join(' | '));
    }
}

process();

package.json

{
    "name": "phone-chooser.local",
    "version": "0.0.0",
    "description": "Phone Chooser",
    "main": "chooser.js",
    "scripts": {
        "choose": "node chooser.js"
    },
    "engines": {
        "node": ">=7.6.0"
    },
    "dependencies": {
        "axios": "^0.19.2",
        "cheerio": "^1.0.0-rc.3",
        "iconv": "^2.3.5"
    }
}