Analyze the percentage of the topic-specific StackOverflow questions for each programming language

00_stakoverflow_questions.py

#!/usr/bin/env python
"""
An ad-hoc script to analyze the percentage of the topic-specific questions for
each programming language.

For each language, the script downloads two pages and looks for the number of questions
in the title, like "NNN,NNN questions". For example, for "python" and
an extra tag "architecture", the script downloads
https://stackoverflow.com/questions/tagged/python and
https://stackoverflow.com/questions/tagged/python+architecture.

The "Tag Popularity" value is the number of tag-specific questions per 1000 of
questions in general.

The script saves the output in a CSV file with the name of the tag.

The output for "architecture.csv" is available here: https://chart-studio.plotly.com/~RomanImankulov/55
"""
import re
import sys
import requests
from lxml.etree import HTML
import pandas as pd
from urllib.parse import quote


language_tags = [
    "c#",
    "java",
    "c++",
    "python",
    "ruby",
    "javascript",
    "php",
    "go",
    "rust",
    "kotlin",
    "swift",
]


def analyze_languages(languages, secondary_tag):
    raw_data = [analyze_language(lang, secondary_tag) for lang in languages]
    return pd.DataFrame.from_records(raw_data)


def analyze_language(language, secondary_tag):
    questions = get_question_count([language])
    questions_with_tag = get_question_count([language, secondary_tag])
    return {
        "language": language,
        "questions": questions,
        "question_with_tag": questions_with_tag,
        "tag_popularity": questions_with_tag * 1000 / questions,
    }


def get_question_count(tags):
    """Return the number of questions, tagged with provided tags."""
    formatted_tags = quote(" ".join(tags))
    resp = requests.get(f"https://stackoverflow.com/questions/tagged/{formatted_tags}")
    tree = HTML(resp.text).getroottree()

    xpath_selector = '//div[@id="mainbar"]//div[contains(@class, "mr12")]'
    count_text = tree.xpath(xpath_selector)[0].text.strip()
    count = int("".join(re.findall(r"\d+", count_text)))
    return count


if __name__ == "__main__":
    secondary_tag = sys.argv[1]
    df = analyze_languages(language_tags, secondary_tag)
    df.to_csv(f"{secondary_tag}.csv", index=False)

algorithm.csv

architecture.csv

design-patterns.csv

security.csv