userAgents.js

/**
 * @fileoverview Defines a list of known user agent strings.
 * This list includes user agents for API testing tools, bots, crawlers, HTTP libraries, automation tools, and scrapers.
 * It can be used to identify and filter traffic from these sources.
 *
 * @module vendor/userAgents
 * @version 1.0.0
 * @license CC BY-NC-ND 4.0
 *
 * @contact Mohammad Montasim-Al-Mamun Shuvo
 * @created 2025-01-28
 * @contactEmail [email protected]
 * @contactGithub https://github.com/montasim
 */

/**
 * A list of known user agent substrings.
 * This array contains substrings that are commonly found in the user-agent strings
 * of API testing tools, bots, crawlers, HTTP libraries, automation tools, and scrapers.
 * Use this list to identify or block unwanted traffic.
 *
 * Example usage:
 * ```javascript
 * const isBlockedUserAgent = userAgents.some(agent => userAgent.includes(agent));
 * if (isBlockedUserAgent) {
 *     // Handle blocked user agents (e.g., deny access or log)
 * }
 * ```
 *
 * @type {string[]}
 */
const userAgents = [
    /** User agents for popular API testing tools. */

    'PostmanRuntime/',
    'Insomnia',
    'RapidAPI-Client',
    'Paw/',
    'curl/',
    'HTTPie/',
    'Thunder Client',
    'AdvancedRestClient',
    'RestSharp',
    'Swagger-UI',
    'k6',
    'ApacheBench',
    'SoapUI',
    'Apache JMeter',
    'Fiddler',
    'Wget',

    /** User agents for well-known bots and crawlers. */
    'Python-urllib',
    'Java/',
    'Googlebot',
    'Bingbot',
    'Yahoo! Slurp',
    'DuckDuckBot',
    'Baiduspider',
    'YandexBot',
    'Sogou',
    'Exabot',
    'facebot',
    'ia_archiver',
    'MJ12bot',
    'AhrefsBot',
    'SemrushBot',
    'DotBot',
    'SerpstatBot',
    'Mediapartners-Google',

    /** User agents for HTTP libraries commonly used for making programmatic requests. */
    'libwww-perl',
    'python-requests',
    'http-client',
    'Go-http-client',
    'axios',
    'Node-fetch',
    'okhttp',
    'WinHttp.WinHttpRequest',
    'GuzzleHttp/',
    'HttpClient/',

    /** User agents for popular browser automation and testing tools. */
    'Selenium',
    'puppeteer',
    'Playwright',
    'PhantomJS',
    'CasperJS',
    'Cypress',
    'webdriver',

    /** User agents for miscellaneous tools and web scrapers. */
    'HTTrack',
    'GoScraper',
    'Scrapy',
    'Mechanize',
    'zgrab',
    'Harvest',
    'URL Grabber',
    'Nikto',
    'Nmap',
    'Netcraft',
    'masscan',

    /** Generic patterns commonly found in the user-agent strings of bots, crawlers, and scrapers. */
    'bot',
    'crawler',
    'spider',
    'http client',
    'scan',
    'scrape',
    'scanner',
    'fetcher',
    'probe',
];

export default userAgents;