API calls to Gitea to migrate repos from GitHub, Gitlab, etc.

# Gitea API for Migration

..

A Readme.md

Operation

Update .env file with

GITEA_BASE=https://yourserver
GITEA_API_KEY=fffffyourapikeygoesherefffff

And npm i.

Migrate from Gitlab

Add to .env file

GITLAB_BASE=https://gitlabinstance
GITLAB_GROUP=some/body
GITLAB_ORG=oncetoldme

to target the repos in a particular organization/project (within a particular group).

Migrate from GitHub

Add to .env file

GITHUB_BASE=https://api.github.com
GITHUB_AUTH_TOKEN=ghp_0000eeee00000eee0000eeee00000000000
GITHUB_ORG=this-example

to target the repos in a particular organization/user.

Common Steps

Add to .env file

GITEA_ORG=chooseyourown

to perform the migration with a destination at the chosen organization/owner.

Run npm i followed by either node clone_org.js for Gitlab, or node clone_gh.js for GitHub.

Fun Facts

• clone_gh.js will check the GitHub API rate limit on the first request, and halt if the remaining quota is less than 50.
• The gitea organization does not have to be named the same as the source organization. Repos from multiple source owners can be collected into one Gitea owner/organization.
• Theoretically, the scripts are automation-friendly as-is, because the parameters can be specified as environment variables instead of .env entries.
• Migrating private repos is an exercise left to the reader.

clone_gh.js

require('dotenv').config();
require('node:http');
const moment = require('moment');

const githubBase = process.env.GITHUB_BASE;
const githubOrg = process.env.GITHUB_ORG;
const repos = `${githubBase}/orgs/${githubOrg}/repos`;
let projects = [];

const ghInit = { headers: { 'Authorization': `Bearer ${process.env.GITHUB_AUTH_TOKEN}` } };

let P = [];


P.push(Promise.all([...P]).then(async () => {
    let resp = await fetch(repos, ghInit);
    let nextPage, nextHref;
    let [pageLink, rateUse, rateRemain, rateLimit, rateReset] = [
        resp.headers.get('link'),
        resp.headers.get('x-ratelimit-used'),
        resp.headers.get('x-ratelimit-remaining'),
        resp.headers.get('x-ratelimit-limit'),
        resp.headers.get('x-ratelimit-reset')
    ];
    console.debug(`GitHub API Rate Limit Usage: ${rateUse}/${rateLimit} Used (${rateRemain} Remain) - Resets ${moment.unix(rateReset).fromNow()}`);
    if (rateRemain < 50) {
        console.warn('Critical Rate Limit Reached -- Stopping Operation');
        return;
    }
    let links = pageLink ? pageLink.split(',').map(link => {
        let [href, rel] = link.split(";").map(_ => _.trim());
        return { href, rel }
    }) : [];
    [nextPage] = links.filter(_ => _.rel === 'rel="next"').map(_ => _.href.match(/page=(\d+)/)[1]);
    [nextHref] = links.filter(_ => _.rel === 'rel="next"').map(_ => _.href.match(/<(.*)>/)[1]);
    let json, new_proj;
    do {
        json = await resp.json();
        new_proj = json.filter(_filter1).map(_map1);
        projects.push(...new_proj);
        // Fetch next page
        if (!nextHref)
            break;
        resp = await fetch(nextHref, ghInit);
        [pageLink] = [
            resp.headers.get('link')
        ];
        links = pageLink ? pageLink.split(',').map(link => {
            let [href, rel] = link.split(";").map(_ => _.trim())
            return { href, rel }
        }) : [];
        [nextPage] = links.filter(_ => _.rel === 'rel="next"').map(_ => _.href.match(/page=(\d+)/)[1]);
        [nextHref] = links.filter(_ => _.rel === 'rel="next"').map(_ => _.href.match(/<(.*)>/)[1]);
    } while (nextHref);
}));

P.push(Promise.all([...P]).then(async () => {
    const orgName = process.env.GITEA_ORG;
    const isOrg = await fetch(_gitea(`/orgs/${orgName}`));
    let newOrg;
    if (!isOrg.ok) {
        newOrg = await fetch(_post('/orgs', { username: orgName, visibility: 'public', repo_admin_change_team_access: true }));
    }
}));

P.push(Promise.all([...P]).then(async () => {
    const orgName = process.env.GITEA_ORG;
    for (const p of projects) {
        const repoName = p.name;
        const cloneUrl = p.clone_url;
        const isPrivate = !!p.private;
        const isRepo = await fetch(_gitea(`/repos/${orgName}/${repoName}`));
        let newRepo;
        if (isRepo.ok) {
            console.log(`Repo [${repoName}] found!`);
        } else {
            console.log(`Repo [${repoName}] not found...`);
            const postReq = _post('/repos/migrate', {
                clone_addr: cloneUrl,
                service: 'github',
                auth_token: process.env.GITHUB_AUTH_TOKEN,
                lfs: true,
                mirror: true,
                private: isPrivate,
                repo_owner: orgName,
                repo_name: repoName
            });
            newRepo = await Promise.resolve(fetch(postReq).catch(()=>{}));
            console.log(`${repoName}: ${newRepo.status} ${newRepo.statusText}`);
        }
    }
}));

function _filter1(_) {
    const isPublic = _.visibility === 'public';
    const nameMatch = _.name.match(process.env.NAME_FILTER ?? '') != null;
    return isPublic && nameMatch;
}
function _map1(_) { return ({ name: _.name, clone_url: _.clone_url, private: _.private }) }
function _gitea(path) { return `${process.env.GITEA_BASE}/api/v1${path}?token=${process.env.GITEA_API_KEY}` }
function _headers(h) {
    let headers = new Headers();
    headers.append('accept', 'application/vnd.github+json');
    headers.append('user-agent',
        `node-fetch/1.0 (Node.js ${process.version}; ${process.release.name}; ${process.platform}; ${process.arch})` +
        ` node/${process.versions.node} v8/${process.versions.v8}` +
        ` openssl/${process.versions.openssl}`)
    Object.entries(h).forEach(hh => headers.append(...hh));
    return headers;
}
function _post(path, body) {
    return new Request(_gitea(path),
        {
            method: 'POST',
            headers: _headers({
                'Content-Type': 'application/json'
            }),
            body: JSON.stringify(body)
        }
    );
}

Promise.all([...P]).then(() => {
    console.log("All done!");
});

clone_org.js

const path = require('node:path');
require('dotenv').config();
require('node:http');


const gitlabBase = process.env.GITLAB_BASE;
const gitlabGroup = process.env.GITLAB_GROUP;
const repos = `${gitlabBase}/groups/${gitlabGroup}/-/children.json`;
let projects = [];

let P = [];

P.push(fetch(repos).then(async (resp) => {
    let [totalPages, nextPage] = [
        resp.headers.get('x-total-pages'),
        resp.headers.get('x-next-page')
    ];
    let json = await resp.json();
    projects.push(...(json).filter(_filter1).map(_map1));
    while (nextPage) {
        r = await fetch(repos + `?page=${nextPage}`);
        nextPage = r.headers.get('x-next-page');
        projects.push(...(await r.json()).filter(_filter1).map(_map1));
    }
}));

P.push(Promise.all([...P]).then(async () => {
    const orgName = process.env.GITEA_ORG;
    const isOrg = await fetch(_gitea(`/orgs/${orgName}`));
    let newOrg;
    if (!isOrg.ok) {
        newOrg = await fetch(_post('/orgs', { username: orgName, visibility: 'public', repo_admin_change_team_access: true }));
    }
}));

P.push(Promise.all([...P]).then(async () => {
    const orgName = process.env.GITEA_ORG;
    for (const p of projects) {
        const repoPath = p.relative_path;
        const repoName = path.posix.basename(repoPath);
        const isRepo = await fetch(_gitea(`/repos/${orgName}/${repoName}`));
        let newRepo;
        if (isRepo.ok) {
            console.log(`Repo [${repoName}] found!`);
        } else {
            console.log(`Repo [${repoName}] not found...`);
            const postReq = _post('/repos/migrate', {
                clone_addr: _repo(repoPath),
                service: 'gitlab',
                lfs: true,
                mirror: true,
                private: false,
                repo_owner: orgName,
                repo_name: repoName
            });
            newRepo = await Promise.resolve(fetch(postReq).catch(()=>{}));
            console.log(`${repoName}: ${newRepo.status} ${newRepo.statusText}`);
        }
    }
}));

function _filter1(_) { return _.type === "project" }
function _map1(_) { return ({ name: _.name, relative_path: _.relative_path }) }
function _repo(_) { return `${process.env.GITLAB_BASE}${_}.git` }
function _gitea(path) { return `${process.env.GITEA_BASE}/api/v1${path}?token=${process.env.GITEA_API_KEY}` }
function _headers(h) {
    let headers = new Headers();
    headers.append('accept', 'application/json');
    Object.entries(h).forEach(hh => headers.append(...hh));
    return headers;
}
function _post(path, body) {
    return new Request(_gitea(path),
        {
            method: 'POST',
            headers: _headers({
                'Content-Type': 'application/json'
            }),
            body: JSON.stringify(body)
        }
    );
}

Promise.all([...P]).then(() => {
    console.log("All done!");
});

package.json.txt

Partial contents of package.json

// ...
  "license": "WTFPL",
  "dependencies": {
    "dotenv": "^16.0.2",
    "moment": "^2.29.4",
    "node-fetch": "^3.2.10",
    "param-case": "^3.0.4"
  }
// ...