Scrape wordpress plugins

wp-plugins.py

#!/bin/python
# @itseco
# https://github.com/itseco/
# Extract download url's for popular Wordpress plugins
# Usage python script.py pages
#       python script.py 99

from lxml import html
import sys
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

# number of pages
pages = int(sys.argv[1]) + 1

for page in range(1, pages):
    # all plugins (3,819 pages)
    # url = "https://wordpress.org/plugins/page/%s/?s" % page
    # popular plugins (99 pages)
    url = "https://cl.wordpress.org/plugins/browse/popular/page/%s/" % page
    page = requests.get(url, verify=False)
    tree = html.fromstring(page.content)
    plugins = tree.xpath('//h2[@class="entry-title"]/a/@href')
    
    for plugin in plugins:
        page = requests.get(plugin, verify=False)
        tree = html.fromstring(page.content)
        link = tree.xpath('//div[@class="plugin-actions"]/a/@href')
        print ''.join(link)