alanhamlett · August 29, 2015 14:14
diff --git a/wakatime_github_timeline.py b/wakatime_github_timeline.py

 """ File: views.py
 """

 @blueprint.route('/project/<path:project_name>/commits')
 @api_utils.nocache
 @auth.login_required
 def project_commits(project_name):
    user = app.current_user

    limit, offset, page = api_utils.get_page(request, limit=35, include_page=True)

    timezone = user.timezone
    try:
        timezone = pytz.timezone(user.timezone)
    except:
        abort(400)

    project = Project.query.filter_by(user_id=user.id, name=project_name).first()
    if project is None:
        abort(404)

    next_url = u('/project/{project_name}/commits').format(
        project_name=project_name,
    )

    auth_token = user.auth_tokens.filter_by(provider='github').order_by(AuthToken.created_at.desc()).first()
    if not auth_token or not auth_token.is_active:
        url = oauth_utils.add_params_to_url('/oauth/github/authorize', params={
            'reason': 'history',
            'next': next_url,
        })
        return redirect(url)
    scopes = auth_token.scopes.split(',') if auth_token.scopes else []
    if 'repo' not in scopes:
        url = oauth_utils.add_params_to_url('/oauth/github/authorize', params={
            'reason': 'history',
            'next': next_url,
        })
        return redirect(url)

    app.logger.debug(auth_token.access_token)

    status = user.profile.flag('commit-history-status:github')
    if status != 'updating' and status != 'pending_update' and (not app.config['DEBUG'] or status != 'ok'):
        if page == 1:
            status = 'pending_update'
            user.profile.flag('commit-history-status:github', status)
            db.session.commit()
            app.logger.debug('Updating github repos...')
            tasks.fetch_github_commit_history.delay(user_id=user.id, auth_token_id=auth_token.id)

    repository = Repository.query.filter_by(user_id=user.id, name=project.name, provider='github').order_by(Repository.external_modified_at.desc()).first()

    commits = []
    total_pages = 1
    if repository:

        app.logger.debug('Getting time per commit...')

        commits = repository.get_commits(limit=limit, offset=offset)
        query = repository.commits.order_by(Commit.author_date.desc())
        (total, total_pages) = api_utils.get_total(query, limit=limit)

        if len(commits) > 0:

            start_timestamp = commits[-1].author_date
            start_timestamp = start_timestamp.replace(tzinfo=pytz.utc)
            start_timestamp = calendar.timegm(start_timestamp.utctimetuple())

            end_timestamp = commits[0].author_date
            end_timestamp = end_timestamp.replace(tzinfo=pytz.utc)
            end_timestamp = calendar.timegm(end_timestamp.utctimetuple())

            branches = []

            (durations, filters) = api_utils.get_durations(
                user_id=user.id,
                timeout=user.timeout,
                writes_only=user.writes_only,
                start=start_timestamp,
                end=end_timestamp,
                project=project_name,
                branches=branches,
            )
            durations = api_utils.combine_durations(durations, filters, project=project_name, branches=branches)

            starting_at = start_timestamp
            for commit in commits[::-1]: # loop through reversed commits
                if commit.author_username and commit.author_username == auth_token.profile_username:
                    commit.calculate_logged_time(durations['durations'], starting_at=starting_at)
                    commit.is_current_author = True
                    starting_at = calendar.timegm(commit.author_date.utctimetuple())
                else:
                    commit.is_current_author = False
                    commit.calculate_logged_time([], starting_at=starting_at)

    context = {
        'user': user,
        'status': status,
        'use_project_slug': False,
        'project': project,
        'branches': api_utils.get_branches_from_request(request),
        'repository': repository,
        'commits': commits,
        'page': page,
        'prev_page': (page - 1) if int(page) > 1 else None,
        'next_page': (page + 1) if int(page) < total_pages else None,
        'total_pages': total_pages,
    }
    if request.args.get('start'):
        context['start'] = request.args.get('start')
    if request.args.get('end'):
        context['end'] = request.args.get('end')
    return render_template('project/commits.html', **context)
    
    
 """ File: tasks.py
 """

 @celery.task(ignore_result=True)
 def fetch_github_commit_history(user_id=None, auth_token_id=None):
    if user_id is None:
        raise Exception('Missing event user_id argument.')
    user = User.query.filter_by(id=user_id).first()
    if user is None:
        raise Exception('User does not exist.')

    provider = 'GitHub'

    auth_token = None
    if auth_token_id is not None:
        auth_token = user.auth_tokens.filter_by(provider=provider.lower(), id=auth_token_id).first()
    if auth_token is None or not auth_token.is_active:
        auth_token = user.auth_tokens.filter_by(provider=provider.lower()).order_by(AuthToken.created_at.desc()).first()
        if auth_token is None or not auth_token.is_active:
            auth_token = None
    scopes = []
    if auth_token and auth_token.scopes:
        scopes.extend(auth_token.scopes.split(','))
    if 'repo' not in scopes:
        auth_token = None

    user.profile.flag('commit-history-status:{0}'.format(provider.lower()), 'updating')
    db.session.commit()

    if auth_token is None:
        user.profile.flag('commit-history-status:{0}'.format(provider.lower()), 'error')
        db.session.commit()
        raise Exception('Missing valid AuthToken.')

    try:
        repos = get_or_create_github_repos(user, auth_token)
        if len(repos) > 0:
            ok = update_github_commits(user, auth_token, repos)
            if ok:
                user.profile.flag('commit-history-status:{0}'.format(provider.lower()), 'ok')
            else:
                user.profile.flag('commit-history-status:{0}'.format(provider.lower()), 'error')
        else:
            user.profile.flag('commit-history-status:{0}'.format(provider.lower()), 'empty')

        db.session.commit()

    except:

        logger.info(u'Exception occured: {0}({1})'.format(sys.exc_info()[0].__name__, sys.exc_info()[1]))

        try:
            db.session.rollback()
            db.session.rollback()
        except:
            pass

        try:
            user = User.query.filter_by(id=user_id).first() # get user object again because profile will have been removed from session
            user.profile.flag('commit-history-status:{0}'.format(provider.lower()), 'error')
            db.session.commit()
        except:
            pass

        retry = sys.exc_info()[0].__name__ in app.config['CELERY_RETRY_EXCEPTIONS'] or 'SoftTimeLimitExceeded()' in str(sys.exc_info()[1])

        if not retry:
            raise sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]
            

 def get_or_create_github_repos(user, auth_token):
    """Get all repos for this user using GitHub api, supporting pagination.
    """
    repos = []

    provider = 'GitHub'

    headers = {
        'Accept': 'Accept: application/vnd.github.moondragon+json',
        'Authorization': u('token {0}').format(auth_token.access_token),
    }

    url = '{url}/user/repos?per_page=100'.format(
        url='https://api.github.com',
    )

    etag = user.profile.flag('commit-history-repos-etag:{0}'.format(provider.lower()))
    app.logger.debug(etag)
    if etag:
        headers['If-None-Match'] = '"{0}"'.format(etag)
    etag = None


    while True:

        response = requests.get(url, headers=headers)

        # if repos have not changed since last time, return repos from our db
        if response.status_code == 304:
            return user.repositories.filter_by(provider=provider.lower()).all()

        if response.status_code == 401:
            auth_token.has_expired = True
            db.session.commit()
            return []

        if response.status_code == 403:
            return []

        if response.status_code != 200:
            oauth_utils.log_error(provider, code=response.status_code, traceback=response.text)
            return []

        if etag is None:
            etag = response.headers['ETag'].replace('W/', '', 1).strip('"')

        # delete etag header for next requests
        if 'If-None-Match' in headers:
            del headers['If-None-Match']

        for repo in response.json():
            defaults = {
                'name': repo['name'],
                'url': repo['url'],
                'html_url': repo['html_url'],
                'is_fork': repo['fork'],
                'is_private': repo['private'],
                'external_created_at': datetime.strptime(repo['created_at'], '%Y-%m-%dT%H:%M:%SZ'),
                'external_modified_at': datetime.strptime(repo['updated_at'], '%Y-%m-%dT%H:%M:%SZ'),
            }
            if repo.get('description'):
                defaults['description'] = repo['description']
            if repo.get('homepage'):
                defaults['homepage'] = repo['homepage']
            if repo.get('stargazers_count'):
                defaults['star_count'] = repo['stargazers_count']
            if repo.get('forks_count'):
                defaults['fork_count'] = repo['forks_count']
            if repo.get('watchers_count'):
                defaults['watch_count'] = repo['watchers_count']
            if repo.get('default_branch'):
                defaults['default_branch'] = repo['default_branch']
            if 'language' in repo:
                defaults['language'] = repo['language']
            r = Repository.get_or_create(
                defaults=defaults,
                user_id=user.id,
                provider=provider.lower(),
                full_name=repo['full_name'],
            )
            r.set_columns(**defaults)
            db.session.commit()
            repos.append(r)

        url = None
        if response.headers.get('Link'):
            link = response.headers.get('Link').split(';')[1]
            if 'rel="next"' in link or 'rel="last"' in link:
                url = response.headers.get('Link').split(';')[0].lstrip('<').rstrip('>')
        if url is None:
            break

    if etag:
        user.profile.flag('commit-history-repos-etag:{0}'.format(provider.lower()), etag)
        db.session.commit()

    return repos


 def update_github_commits(user, auth_token, repos):
    """Create commits for this user's repos using GitHub api, supporting
    pagination. Return True if all commits were created, otherwise return
    False.
    """

    provider = 'GitHub'

    headers = {
        'Accept': 'Accept: application/vnd.github.v3+json',
        'Authorization': u('token {0}').format(auth_token.access_token),
    }

    for repo in repos:

        url = '{url}/commits?per_page=100'.format(
            url=repo.url,
        )

        etag = repo.commits_etag
        if etag:
            headers['If-None-Match'] = '"{0}"'.format(etag)
        etag = None

        page = 1

        while True:

            response = requests.get(url, headers=headers)

            # if commits have not changed since last time, skip these commits
            if response.status_code == 304:
                break

            # repo has not been setup
            if response.status_code == 409:
                break

            if response.status_code == 403:
                break

            if response.status_code == 401:
                auth_token.has_expired = True
                db.session.commit()
                return False

            if response.status_code == '404':
                repo.commits.delete()
                db.session.delete(repo)
                db.session.commit()
                break

            if response.status_code != 200:
                oauth_utils.log_error(provider, code=response.status_code, traceback=response.text)
                return False

            if etag is None:
                etag = response.headers['ETag'].replace('W/', '', 1).strip('"')

            # delete etag header for next requests
            if 'If-None-Match' in headers:
                del headers['If-None-Match']

            for commit in response.json():
                defaults = {
                    'author_date': datetime.strptime(commit['commit']['author']['date'], '%Y-%m-%dT%H:%M:%SZ'),
                    'committer_date': datetime.strptime(commit['commit']['committer']['date'], '%Y-%m-%dT%H:%M:%SZ'),
                    'url': commit['url'],
                    'html_url': commit['html_url'],
                    'message': commit['commit']['message'],
                }
                if commit.get('author'):
                    if commit.get('author', {}).get('avatar_url'):
                        defaults['author_avatar_url'] = commit['author']['avatar_url']
                    if commit.get('author', {}).get('url'):
                        defaults['author_url'] = commit['author']['url']
                    if commit.get('author', {}).get('html_url'):
                        defaults['author_html_url'] = commit['author']['html_url']
                    if commit.get('author', {}).get('login'):
                        defaults['author_username'] = commit['author']['login']
                if commit['commit'].get('author', {}).get('name'):
                    defaults['author_name'] = commit['commit']['author']['name']
                if commit['commit'].get('author', {}).get('email'):
                    defaults['author_email'] = commit['commit']['author']['email']
                if commit.get('committer'):
                    if commit.get('committer', {}).get('avatar_url'):
                        defaults['committer_avatar_url'] = commit['committer']['avatar_url']
                    if commit.get('committer', {}).get('url'):
                        defaults['committer_url'] = commit['committer']['url']
                    if commit.get('committer', {}).get('html_url'):
                        defaults['committer_html_url'] = commit['committer']['html_url']
                    if commit.get('committer', {}).get('login'):
                        defaults['committer_username'] = commit['committer']['login']
                if commit['commit'].get('committer', {}).get('name'):
                    defaults['committer_name'] = commit['commit']['committer']['name']
                if commit['commit'].get('committer', {}).get('email'):
                    defaults['committer_email'] = commit['commit']['committer']['email']
                c = Commit.get_or_create(
                    defaults=defaults,
                    user_id=user.id,
                    hash=commit['sha'],
                    repository_id=repo.id,
                )
                c.set_columns(**defaults)
                db.session.commit()

            url = None
            if response.headers.get('Link'):
                link = response.headers.get('Link').split(';')[1]
                if 'rel="next"' in link or 'rel="last"' in link:
                    url = response.headers.get('Link').split(';')[0].lstrip('<').rstrip('>')
            if url is None:
                break

            page += 1

            if page > 100:
                break

        if etag:
            repo.commits_etag = etag
            db.session.commit()

    return True

	""" File: views.py
	"""

	@blueprint.route('/project/<path:project_name>/commits')
	@api_utils.nocache
	@auth.login_required
	def project_commits(project_name):
	user = app.current_user

	limit, offset, page = api_utils.get_page(request, limit=35, include_page=True)

	timezone = user.timezone
	try:
	timezone = pytz.timezone(user.timezone)
	except:
	abort(400)

	project = Project.query.filter_by(user_id=user.id, name=project_name).first()
	if project is None:
	abort(404)

	next_url = u('/project/{project_name}/commits').format(
	project_name=project_name,
	)

	auth_token = user.auth_tokens.filter_by(provider='github').order_by(AuthToken.created_at.desc()).first()
	if not auth_token or not auth_token.is_active:
	url = oauth_utils.add_params_to_url('/oauth/github/authorize', params={
	'reason': 'history',
	'next': next_url,
	})
	return redirect(url)
	scopes = auth_token.scopes.split(',') if auth_token.scopes else []
	if 'repo' not in scopes:
	url = oauth_utils.add_params_to_url('/oauth/github/authorize', params={
	'reason': 'history',
	'next': next_url,
	})
	return redirect(url)

	app.logger.debug(auth_token.access_token)

	status = user.profile.flag('commit-history-status:github')
	if status != 'updating' and status != 'pending_update' and (not app.config['DEBUG'] or status != 'ok'):
	if page == 1:
	status = 'pending_update'
	user.profile.flag('commit-history-status:github', status)
	db.session.commit()
	app.logger.debug('Updating github repos...')
	tasks.fetch_github_commit_history.delay(user_id=user.id, auth_token_id=auth_token.id)

	repository = Repository.query.filter_by(user_id=user.id, name=project.name, provider='github').order_by(Repository.external_modified_at.desc()).first()

	commits = []
	total_pages = 1
	if repository:

	app.logger.debug('Getting time per commit...')

	commits = repository.get_commits(limit=limit, offset=offset)
	query = repository.commits.order_by(Commit.author_date.desc())
	(total, total_pages) = api_utils.get_total(query, limit=limit)

	if len(commits) > 0:

	start_timestamp = commits[-1].author_date
	start_timestamp = start_timestamp.replace(tzinfo=pytz.utc)
	start_timestamp = calendar.timegm(start_timestamp.utctimetuple())

	end_timestamp = commits[0].author_date
	end_timestamp = end_timestamp.replace(tzinfo=pytz.utc)
	end_timestamp = calendar.timegm(end_timestamp.utctimetuple())

	branches = []

	(durations, filters) = api_utils.get_durations(
	user_id=user.id,
	timeout=user.timeout,
	writes_only=user.writes_only,
	start=start_timestamp,
	end=end_timestamp,
	project=project_name,
	branches=branches,
	)
	durations = api_utils.combine_durations(durations, filters, project=project_name, branches=branches)

	starting_at = start_timestamp
	for commit in commits[::-1]: # loop through reversed commits
	if commit.author_username and commit.author_username == auth_token.profile_username:
	commit.calculate_logged_time(durations['durations'], starting_at=starting_at)
	commit.is_current_author = True
	starting_at = calendar.timegm(commit.author_date.utctimetuple())
	else:
	commit.is_current_author = False
	commit.calculate_logged_time([], starting_at=starting_at)

	context = {
	'user': user,
	'status': status,
	'use_project_slug': False,
	'project': project,
	'branches': api_utils.get_branches_from_request(request),
	'repository': repository,
	'commits': commits,
	'page': page,
	'prev_page': (page - 1) if int(page) > 1 else None,
	'next_page': (page + 1) if int(page) < total_pages else None,
	'total_pages': total_pages,
	}
	if request.args.get('start'):
	context['start'] = request.args.get('start')
	if request.args.get('end'):
	context['end'] = request.args.get('end')
	return render_template('project/commits.html', **context)


	""" File: tasks.py
	"""

	@celery.task(ignore_result=True)
	def fetch_github_commit_history(user_id=None, auth_token_id=None):
	if user_id is None:
	raise Exception('Missing event user_id argument.')
	user = User.query.filter_by(id=user_id).first()
	if user is None:
	raise Exception('User does not exist.')

	provider = 'GitHub'

	auth_token = None
	if auth_token_id is not None:
	auth_token = user.auth_tokens.filter_by(provider=provider.lower(), id=auth_token_id).first()
	if auth_token is None or not auth_token.is_active:
	auth_token = user.auth_tokens.filter_by(provider=provider.lower()).order_by(AuthToken.created_at.desc()).first()
	if auth_token is None or not auth_token.is_active:
	auth_token = None
	scopes = []
	if auth_token and auth_token.scopes:
	scopes.extend(auth_token.scopes.split(','))
	if 'repo' not in scopes:
	auth_token = None

	user.profile.flag('commit-history-status:{0}'.format(provider.lower()), 'updating')
	db.session.commit()

	if auth_token is None:
	user.profile.flag('commit-history-status:{0}'.format(provider.lower()), 'error')
	db.session.commit()
	raise Exception('Missing valid AuthToken.')

	try:
	repos = get_or_create_github_repos(user, auth_token)
	if len(repos) > 0:
	ok = update_github_commits(user, auth_token, repos)
	if ok:
	user.profile.flag('commit-history-status:{0}'.format(provider.lower()), 'ok')
	else:
	user.profile.flag('commit-history-status:{0}'.format(provider.lower()), 'error')
	else:
	user.profile.flag('commit-history-status:{0}'.format(provider.lower()), 'empty')

	db.session.commit()

	except:

	logger.info(u'Exception occured: {0}({1})'.format(sys.exc_info()[0].__name__, sys.exc_info()[1]))

	try:
	db.session.rollback()
	db.session.rollback()
	except:
	pass

	try:
	user = User.query.filter_by(id=user_id).first() # get user object again because profile will have been removed from session
	user.profile.flag('commit-history-status:{0}'.format(provider.lower()), 'error')
	db.session.commit()
	except:
	pass

	retry = sys.exc_info()[0].__name__ in app.config['CELERY_RETRY_EXCEPTIONS'] or 'SoftTimeLimitExceeded()' in str(sys.exc_info()[1])

	if not retry:
	raise sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]


	def get_or_create_github_repos(user, auth_token):
	"""Get all repos for this user using GitHub api, supporting pagination.
	"""
	repos = []

	provider = 'GitHub'

	headers = {
	'Accept': 'Accept: application/vnd.github.moondragon+json',
	'Authorization': u('token {0}').format(auth_token.access_token),
	}

	url = '{url}/user/repos?per_page=100'.format(
	url='https://api.github.com',
	)

	etag = user.profile.flag('commit-history-repos-etag:{0}'.format(provider.lower()))
	app.logger.debug(etag)
	if etag:
	headers['If-None-Match'] = '"{0}"'.format(etag)
	etag = None


	while True:

	response = requests.get(url, headers=headers)

	# if repos have not changed since last time, return repos from our db
	if response.status_code == 304:
	return user.repositories.filter_by(provider=provider.lower()).all()

	if response.status_code == 401:
	auth_token.has_expired = True
	db.session.commit()
	return []

	if response.status_code == 403:
	return []

	if response.status_code != 200:
	oauth_utils.log_error(provider, code=response.status_code, traceback=response.text)
	return []

	if etag is None:
	etag = response.headers['ETag'].replace('W/', '', 1).strip('"')

	# delete etag header for next requests
	if 'If-None-Match' in headers:
	del headers['If-None-Match']

	for repo in response.json():
	defaults = {
	'name': repo['name'],
	'url': repo['url'],
	'html_url': repo['html_url'],
	'is_fork': repo['fork'],
	'is_private': repo['private'],
	'external_created_at': datetime.strptime(repo['created_at'], '%Y-%m-%dT%H:%M:%SZ'),
	'external_modified_at': datetime.strptime(repo['updated_at'], '%Y-%m-%dT%H:%M:%SZ'),
	}
	if repo.get('description'):
	defaults['description'] = repo['description']
	if repo.get('homepage'):
	defaults['homepage'] = repo['homepage']
	if repo.get('stargazers_count'):
	defaults['star_count'] = repo['stargazers_count']
	if repo.get('forks_count'):
	defaults['fork_count'] = repo['forks_count']
	if repo.get('watchers_count'):
	defaults['watch_count'] = repo['watchers_count']
	if repo.get('default_branch'):
	defaults['default_branch'] = repo['default_branch']
	if 'language' in repo:
	defaults['language'] = repo['language']
	r = Repository.get_or_create(
	defaults=defaults,
	user_id=user.id,
	provider=provider.lower(),
	full_name=repo['full_name'],
	)
	r.set_columns(**defaults)
	db.session.commit()
	repos.append(r)

	url = None
	if response.headers.get('Link'):
	link = response.headers.get('Link').split(';')[1]
	if 'rel="next"' in link or 'rel="last"' in link:
	url = response.headers.get('Link').split(';')[0].lstrip('<').rstrip('>')
	if url is None:
	break

	if etag:
	user.profile.flag('commit-history-repos-etag:{0}'.format(provider.lower()), etag)
	db.session.commit()

	return repos


	def update_github_commits(user, auth_token, repos):
	"""Create commits for this user's repos using GitHub api, supporting
	pagination. Return True if all commits were created, otherwise return
	False.
	"""

	provider = 'GitHub'

	headers = {
	'Accept': 'Accept: application/vnd.github.v3+json',
	'Authorization': u('token {0}').format(auth_token.access_token),
	}

	for repo in repos:

	url = '{url}/commits?per_page=100'.format(
	url=repo.url,
	)

	etag = repo.commits_etag
	if etag:
	headers['If-None-Match'] = '"{0}"'.format(etag)
	etag = None

	page = 1

	while True:

	response = requests.get(url, headers=headers)

	# if commits have not changed since last time, skip these commits
	if response.status_code == 304:
	break

	# repo has not been setup
	if response.status_code == 409:
	break

	if response.status_code == 403:
	break

	if response.status_code == 401:
	auth_token.has_expired = True
	db.session.commit()
	return False

	if response.status_code == '404':
	repo.commits.delete()
	db.session.delete(repo)
	db.session.commit()
	break

	if response.status_code != 200:
	oauth_utils.log_error(provider, code=response.status_code, traceback=response.text)
	return False

	if etag is None:
	etag = response.headers['ETag'].replace('W/', '', 1).strip('"')

	# delete etag header for next requests
	if 'If-None-Match' in headers:
	del headers['If-None-Match']

	for commit in response.json():
	defaults = {
	'author_date': datetime.strptime(commit['commit']['author']['date'], '%Y-%m-%dT%H:%M:%SZ'),
	'committer_date': datetime.strptime(commit['commit']['committer']['date'], '%Y-%m-%dT%H:%M:%SZ'),
	'url': commit['url'],
	'html_url': commit['html_url'],
	'message': commit['commit']['message'],
	}
	if commit.get('author'):
	if commit.get('author', {}).get('avatar_url'):
	defaults['author_avatar_url'] = commit['author']['avatar_url']
	if commit.get('author', {}).get('url'):
	defaults['author_url'] = commit['author']['url']
	if commit.get('author', {}).get('html_url'):
	defaults['author_html_url'] = commit['author']['html_url']
	if commit.get('author', {}).get('login'):
	defaults['author_username'] = commit['author']['login']
	if commit['commit'].get('author', {}).get('name'):
	defaults['author_name'] = commit['commit']['author']['name']
	if commit['commit'].get('author', {}).get('email'):
	defaults['author_email'] = commit['commit']['author']['email']
	if commit.get('committer'):
	if commit.get('committer', {}).get('avatar_url'):
	defaults['committer_avatar_url'] = commit['committer']['avatar_url']
	if commit.get('committer', {}).get('url'):
	defaults['committer_url'] = commit['committer']['url']
	if commit.get('committer', {}).get('html_url'):
	defaults['committer_html_url'] = commit['committer']['html_url']
	if commit.get('committer', {}).get('login'):
	defaults['committer_username'] = commit['committer']['login']
	if commit['commit'].get('committer', {}).get('name'):
	defaults['committer_name'] = commit['commit']['committer']['name']
	if commit['commit'].get('committer', {}).get('email'):
	defaults['committer_email'] = commit['commit']['committer']['email']
	c = Commit.get_or_create(
	defaults=defaults,
	user_id=user.id,
	hash=commit['sha'],
	repository_id=repo.id,
	)
	c.set_columns(**defaults)
	db.session.commit()

	url = None
	if response.headers.get('Link'):
	link = response.headers.get('Link').split(';')[1]
	if 'rel="next"' in link or 'rel="last"' in link:
	url = response.headers.get('Link').split(';')[0].lstrip('<').rstrip('>')
	if url is None:
	break

	page += 1

	if page > 100:
	break

	if etag:
	repo.commits_etag = etag
	db.session.commit()

	return True