Forked from Kebiled/export_multi_repo_issues_to_csv.py
Last active
May 25, 2021 08:01
-
-
Save Jammizzle/ad2a94008b56a6f9d17cfdddf5a6dd4d to your computer and use it in GitHub Desktop.
Export Issues from Github repo to CSV (API v3 and ZenHub API)
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/python | |
| import csv | |
| import json | |
| import requests | |
| import configparser | |
| """ | |
| Usage: | |
| Add the following to config.ini with appropriate values: | |
| [DEFAULT] | |
| AUTH_TOKEN = | |
| ZEN_ACCESS = | |
| Exports Issues from a list of repositories to individual CSV files | |
| Uses basic authentication (Github API Token and Zenhub API Token) | |
| to retrieve Issues from a repository that token has access to. | |
| Supports Github API v3 and ZenHubs current working API. | |
| Derived from https://gist.github.com/Kebiled/7b035d7518fdfd50d07e2a285aff3977 | |
| @PinnaclePSM Author Jamie Belcher | |
| """ | |
| def write_issues(r, csvout, repo_name, repo_ID): | |
| if not r.status_code == 200: | |
| raise Exception("Request returned status of:"+str(r.status_code)) | |
| r_json = r.json() | |
| for issue in r_json: | |
| print(repo_name + ' issue Number: ' + str(issue['number'])) | |
| zenhub_issue_url = 'https://api.zenhub.io/p1/repositories/' + str(repo_ID) + '/issues/' + str(issue['number']) + '?access_token=' + ACCESS_TOKEN | |
| zen_r = requests.get(zenhub_issue_url).json() | |
| DateCreated = issue['created_at'][:-10] | |
| if 'pull_request' not in issue: | |
| global ISSUES | |
| ISSUES += 1 | |
| assignees, tag, category, priority = '', '', '', '' | |
| for i in issue['assignees'] if issue['assignees'] else []: | |
| assignees += i['login'] + ',' | |
| for x in issue['labels'] if issue['labels'] else []: | |
| if "Category" in x['name']: | |
| category = x['name'][11:11 + len(x['name'])] | |
| if "Tag" in x['name']: | |
| tag = x['name'][6:6 + len(x['name'])] | |
| if "Priority" in x['name']: | |
| priority = x['name'][11:11 + len(x['name'])] | |
| estimate = zen_r.get('estimate', dict()).get('value', "") | |
| if category != 'BUG': | |
| Pipeline = zen_r.get('pipeline', dict()).get('name', "") | |
| csvout.writerow([repo_name, issue['number'], issue['title'].encode('utf-8'), category, | |
| tag, assignees[:-1], | |
| priority, Pipeline, DateCreated, | |
| estimate]) | |
| def get_issues(repo_data): | |
| repo_name = repo_data[0] | |
| repo_ID = repo_data[1] | |
| issues_for_repo_url = 'https://api.github.com/repos/%s/issues?since=2017-05-01&state=closed' % repo_name | |
| r = requests.get(issues_for_repo_url, auth=AUTH) | |
| write_issues(r, FILEOUTPUT, repo_name, repo_ID) | |
| # more pages? examine the 'link' header returned | |
| if 'link' in r.headers: | |
| pages = dict( | |
| [(rel[6:-1], url[url.index('<') + 1:-1]) for url, rel in | |
| [link.split(';') for link in | |
| r.headers['link'].split(',')]]) | |
| while 'last' in pages and 'next' in pages: | |
| pages = dict( | |
| [(rel[6:-1], url[url.index('<') + 1:-1]) for url, rel in | |
| [link.split(';') for link in | |
| r.headers['link'].split(',')]]) | |
| r = requests.get(pages['next'], auth=AUTH) | |
| write_issues(r, FILEOUTPUT, repo_name, repo_ID) | |
| if pages['next'] == pages['last']: | |
| break | |
| REPO_LIST = [("*GITHUB REPO*", "*ZENHUB REPOID*")] | |
| config = configparser.ConfigParser() | |
| config.read('config.ini') | |
| AUTH = ('token', config['DEFAULT']['AUTH_TOKEN']) | |
| ACCESS_TOKEN = config['DEFAULT']['ZEN_ACCESS'] | |
| ISSUES = 0 | |
| FILENAME = 'output.csv' | |
| OPENFILE = open(FILENAME, 'wb') | |
| FILEOUTPUT = csv.writer(OPENFILE) | |
| FILEOUTPUT.writerow(('Repository', 'Issue Number', 'Issue Title', 'Category', | |
| 'Tag', 'Assigned To', | |
| 'Priority', 'Pipeline', | |
| 'Issue Author', | |
| 'Created At', 'Estimate Value' | |
| )) | |
| for repo_data in REPO_LIST: | |
| get_issues(repo_data) | |
| OPENFILE.close() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [DEFAULT] | |
| AUTH_TOKEN = | |
| ZEN_ACCESS = |
Hi! this only downloads open issues, how can I download all issues?
Has anyone tried pulling the threaded comments too?
@Jammizzle Any chance you've got time to Add in the ability to export the Epics and Labels, that would make my life complete...
Getting this issue any one else faced the same issue
Traceback (most recent call last):
File "C:/Users/Manu/Desktop/export repository1.py", line 95, in
get_issues(repo_data)
File "C:/Users/Manu/Desktop/export repository1.py", line 60, in get_issues
write_issues(r, FILEOUTPUT, repo_name, repo_ID)
File "C:/Users/Manu/Desktop/export repository1.py", line 18, in write_issues
raise Exception(r.status_code)
Exception: 404
@manu4387 The request perhaps couldn't find your repo, which could be either due to permissions or a potential typo
i have 2 factor authentication enabled , i tried to use the Personal authentication token but same issue.
i tried another code for connectivity to get the data which is working fine to test the connectivity with authentication token
Ahh I haven't tested this with 2FA but I don't know why that'd cause an issue, unless it's Zenhubs permissions? I also haven't used this in quite some time so I'll have a look into setting it up again for myself then I'll get back to you
Ok thanks a lot i will wait for your reply. I only need to get the github
repo,issues and comments.
…On Thu, 24 Oct 2019 at 22:27, Jamie Belcher ***@***.***> wrote:
Ahh I haven't tested this with 2FA but I don't know why that'd cause an
issue, unless it's Zenhubs permissions? I also haven't used this in quite
some time so I'll have a look into setting it up again for myself then I'll
get back to you
—
You are receiving this because you were mentioned.
Reply to this email directly, view it on GitHub
<https://gist.github.com/ad2a94008b56a6f9d17cfdddf5a6dd4d?email_source=notifications&email_token=ANOUZL4XQVQWHV6YIMDHNVLQQHHYZA5CNFSM4JEDFXZKYY3PNVWWK3TUL52HS4DFVNDWS43UINXW23LFNZ2KUY3PNVWWK3TUL5UWJTQAF3CDQ#gistcomment-3064888>,
or unsubscribe
<https://github.com/notifications/unsubscribe-auth/ANOUZL3B5M2RPNSLRJMJB3DQQHHYZANCNFSM4JEDFXZA>
.
--
Manu Agarwal
Senior Implementation Consultant, Siren
A Unit 3, GTC, Mervue Business Park, Galway, H91 CR20, Ireland
P +353 (0)91 704 885 <+353+(0)91+704+885>
M +91-7620037800 <+91-7620037800>
E [email protected] <[email protected]>
W www.siren.io
<http://www.siren.io?utm_source=WiseStamp&utm_medium=email&utm_term=&utm_content=&utm_campaign=signature>
<https://twitter.com/sirensearch?utm_source=WiseStamp&utm_medium=email&utm_term=&utm_content=&utm_campaign=signature>
<https://www.youtube.com/channel/UCKGsC-vD28r7hW6T9QspKPA?utm_source=WiseStamp&utm_medium=email&utm_term=&utm_content=&utm_campaign=signature>
<https://vimeo.com/sirenio?utm_source=WiseStamp&utm_medium=email&utm_term=&utm_content=&utm_campaign=signature>
<https://www.facebook.com/sirensearch?utm_source=WiseStamp&utm_medium=email&utm_term=&utm_content=&utm_campaign=signature>
<https://www.linkedin.com/company/11117365?utm_source=WiseStamp&utm_medium=email&utm_term=&utm_content=&utm_campaign=signature>
@manu4387 Should be able to just delete all zenhub requests and everything's fine, remember to remove it from the CSV builder as well
Thanks for the reply i did the same thing new error comes:
Traceback (most recent call last):
File "C:/Users/Manu/Desktop/ss.py", line 199, in <module>
get_issues(repo_data)
File "C:/Users/Manu/Desktop/ss.py", line 121, in get_issues
write_issues(r, FILEOUTPUT, repo_name, repo_ID)
File "C:/Users/Manu/Desktop/ss.py", line 51, in write_issues
raise Exception("Request returned status of:"+str(r.status_code))
Exception: Request returned status of:404
It seems it not able to create the API URL of my repository.
Although I have given the username/repositoryname but it fails.
issues_for_repo_url = 'https://api.github.com/repos/%s/issues?since=2017-05-01&state=closed' % repo_name
it doesn’t able to bring the URL. Do I need to change anything in URL creation?
From: Jamie Belcher <[email protected]>
Sent: 25 October 2019 20:40
To: Jammizzle <[email protected]>
Cc: manu4387 <[email protected]>; Mention <[email protected]>
Subject: Re: Jammizzle/config.ini
@manu4387 <https://github.com/manu4387> Should be able to just delete all zenhub requests and everything's fine, remember to remove it from the CSV builder as well
—
You are receiving this because you were mentioned.
Reply to this email directly, view it on GitHub <https://gist.github.com/ad2a94008b56a6f9d17cfdddf5a6dd4d?email_source=notifications&email_token=ANOUZL45TDPPCH2XVPUIFU3QQMD4LA5CNFSM4JEDFXZKYY3PNVWWK3TUL52HS4DFVNDWS43UINXW23LFNZ2KUY3PNVWWK3TUL5UWJTQAF3DZC#gistcomment-3065745> , or unsubscribe <https://github.com/notifications/unsubscribe-auth/ANOUZL2TTIJEWKI27VMP5CDQQMD4LANCNFSM4JEDFXZA> . <https://github.com/notifications/beacon/ANOUZL3D4WT5Q6ZSDDLRUWDQQMD4LA5CNFSM4JEDFXZKYY3PNVWWK3TUL52HS4DFVNDWS43UINXW23LFNZ2KUY3PNVWWK3TUL5UWJTQAF3DZC.gif>
Can you send me an email at [email protected] of your REPO_LIST variable?
Thanks for it, I am able to resolve it now.
Wanted to check in the csv file it is putting 2 entry for every issues?
Also I want take the comments too out from every issues?
Is there any specific issue you have given date in the parameter of the repository scan
https://api.github.com/repos/%s/issues?since=2017-05-01 <https://api.github.com/repos/%25s/issues?since=2017-05-01&state=closed> &state=closed'
From: Jamie Belcher <[email protected]>
Sent: 25 October 2019 22:00
To: Jammizzle <[email protected]>
Cc: manu4387 <[email protected]>; Mention <[email protected]>
Subject: Re: Jammizzle/export_multi_repo_issues_to_csv.py
Can you send me an email at [email protected] <mailto:[email protected]> of your REPO_LIST variable?
—
You are receiving this because you were mentioned.
Reply to this email directly, view it on GitHub <https://gist.github.com/ad2a94008b56a6f9d17cfdddf5a6dd4d?email_source=notifications&email_token=ANOUZLYUWTPMBF2RMPP67NTQQMNIVA5CNFSM4JEDFXZKYY3PNVWWK3TUL52HS4DFVNDWS43UINXW23LFNZ2KUY3PNVWWK3TUL5UWJTQAF3D5U#gistcomment-3065818> , or unsubscribe <https://github.com/notifications/unsubscribe-auth/ANOUZLZVO3TSPW3V3GSZCYDQQMNIVANCNFSM4JEDFXZA> .
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
@sparda2 - You'll need to change:
GITHUB_ACCESS TOKENto your personal tokenZENHUB_ACCESS_TOKENto the personal token provided by ZenhubFilename- Desired filename with extension of .csvGITHUB_REPOto the repo name (User/RepoName)ZENHUB_REPO_ID- If you wish to use Zenhub you'll need to set this to the unique ID of the repoThen from the terminal just run
python script.py