crawldoc.py

import argparse
import csv
import json
import requests
import sys

from dataclasses import dataclass
from pprint import pprint

# Use python >= 3.9 (datatypes and list)

parser = argparse.ArgumentParser(description='Github mardown docs crawler.')
parser.add_argument('-r','--repo', type=str, help='Repository name (example: org/reponame)',
                    nargs='?', default='')
parser.add_argument('-o','--org', type=str, help='Organization name',
                    nargs='?', default='')
parser.add_argument('-v','--verbose', dest='verbose', action='store_const', const=True, default=False,
                    help='Print verbose output (including all API requests).')
args = parser.parse_args()

############################################################

BASE_HOSTNAME = "https://api.github.com"
TOKEN = ""
EXT = ".md" # consider adoc etc. or regex query expression

############################################################

@dataclass
class DocEntry:
    repo: str   # maybe dup
    path: str
    url: str
    title: str
    keywords: list[str]
    content: str = ''

############################################################

def debugPrint(str):
    if args.verbose:
        pprint(str)

def req(url, data = None):
        print("Querying %s" % url)
        h = {"Authorization": "Bearer %s" % TOKEN, "Accept": "application/vnd.github+json"}
        if data:
            r = requests.post(url, data=data, headers=h)
        else:     
            r = requests.get(url, headers=h)
        if r.ok:
            respData = json.loads(r.text)
            debugPrint(r)
            debugPrint(respData)
            return respData
        else:
            print("ERROR", r, data)
            exit(1)

#def find_default_brach(repo):
#     req("%s/git/trees/main?recursive=1" % repo)

def list_repo_docs(repo):
    files = req("%s/repos/%s/git/trees/main?recursive=1" % (BASE_HOSTNAME, repo))
    docfiles = [f for f in files['tree'] if f['path'].endswith(EXT)]
    docs = []
    for docfile in docfiles:
        docs.append(process_doc(repo, docfile))
    return docs


def process_doc(repo, doc):
    doc_url = "%s/repos/%s/git/trees/main/%s" % (BASE_HOSTNAME, repo, doc['path'])  # špatně
    title = doc['path'].split(EXT)[0].replace("/", "_").lower()
    keywords = []
    keywords.append(repo.split("/")[-1])  # skip org name
    keywords.append(title)  # omit extension
    # fetch and analyze the file
    # raw, nojson# content = req(doc_url)
    return DocEntry(repo=repo, path=doc['path'], url=doc_url, keywords=keywords, title=title)

def print_doc_list(docs):
    print("Captured docs:")
    csvw = csv.writer(sys.stdout, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
    csvw.writerow(docs[0].__annotations__.keys())
    for doc in docs:
        csvw.writerow(doc.__dict__.values())

#############################################################

if args.repo:
    docs = list_repo_docs(args.repo)
    pprint(docs)
    print_doc_list(docs)

out.csv