crawling_gerrit.py

import requests
import json
from tqdm.notebook import tqdm
import time
import multiprocessing as mp
import datetime
import pandas as pd
import itertools

def convert_gerrit_timestamp_to_datetime(s):
    return datetime.datetime.strptime(s[:-3], '%Y-%m-%d %H:%M:%S.%f')
    
def get_changes(url, after):
    changes = []
    more_changes = True
    last_timestamp = datetime.datetime.now()

    while more_changes and after<last_timestamp:
        params = [('pp', 0), ('o', 'MESSAGES'), ('o', 'DETAILED_ACCOUNTS'), ('q', 'before:"{}"'.format(str(last_timestamp)))]

        r = requests.get(url + '/changes/', params=params)
        if r.status_code == 200:
            new_changes = json.loads(r.text[len("')]}\'"):])
            changes += new_changes

            more_changes = '_more_changes' in new_changes[-1]
            last_timestamp = convert_gerrit_timestamp_to_datetime(new_changes[-1]['updated'])
            print(str(last_timestamp))
        else:
            raise Exception(r.status_code)
    return changes
    
android_changes = get_changes('https://android-review.googlesource.com', datetime.datetime(year=2019, month=1, day=1))