Clone Stash/BitBucket - all repositories from all projects which user has access to

Cloner.java

package org.prokyon.management.bitbucket_cloner;

import com.google.common.base.CaseFormat;
import com.mashape.unirest.http.HttpResponse;
import com.mashape.unirest.http.JsonNode;
import com.mashape.unirest.http.Unirest;
import com.mashape.unirest.http.exceptions.UnirestException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
import org.json.JSONArray;
import org.json.JSONObject;

import java.io.File;


/**
 * Created by zangetsu on 1/31/17.
 */
public class Cloner {

    private static final Logger log = LogManager.getLogger(Cloner.class);

    String BIT_BUCKET_SERVER = "http://stash.domain.com";
    String REST_API_SUFFIX ="/rest/api/1.0";
    String API_PROJECTS = "/projects";
    String API_REPOSITORIES ="/repos";
    String DEFAULT_DIR = "/home/user/projects..";

    String USER = "user";
    String PASSWORD = "password";

    public static JSONArray projects = null;
    public static JSONArray repositories = null;

    private void obtainProjects() {
        log.debug("Going to obtain projects from: " + BIT_BUCKET_SERVER +REST_API_SUFFIX + API_PROJECTS);

        try {
            projects = Unirest.get(BIT_BUCKET_SERVER +REST_API_SUFFIX + API_PROJECTS)
                    .basicAuth(USER, PASSWORD)
                    .header("accept", "application/json")
                    .asJson()
                    .getBody()
                    .getObject().getJSONArray("values");
        } catch (UnirestException e) {
            log.error("Cannot obtain list of projects from " + BIT_BUCKET_SERVER +REST_API_SUFFIX + API_PROJECTS);
        }
    }

    private void obtainRepositories(String projectKey) {
        String projectReposURL = BIT_BUCKET_SERVER +REST_API_SUFFIX + API_PROJECTS + "/" + projectKey + API_REPOSITORIES;
        log.info("Obtaining repos from " + projectReposURL);
        try {
            repositories = Unirest.get(projectReposURL)
                    .basicAuth(USER, PASSWORD)
                    .header("accept", "application/json")
                    .asJson()
                    .getBody()
                    .getObject().getJSONArray("values");
        } catch (UnirestException e) {
            log.error("Cannot obtain project repositories from " + projectReposURL);
        }
        //log.debug("Repositories obtained: " + repositories);

    }
    private void cloneRepository(String gitURL,String repoDir){
        log.info("Going to clone repo " + gitURL);
        log.info("Repository will be stored at " + repoDir);
        try {
            Git git = Git.cloneRepository()
               .setURI(gitURL)
               .setDirectory(new File(repoDir))
               .setCloneAllBranches(true)
               .setCredentialsProvider(new UsernamePasswordCredentialsProvider(USER, PASSWORD))
               .call();

        } catch (GitAPIException e) {
            log.error("Error on cloning repository from " + gitURL + " into local directory " + repoDir + ". Check the path.");
        }
    }
    public void cloner(){
        // Obtain projects
        obtainProjects();
        if (projects.length()<1){
            log.info("There are no projects available to process.");
        } else{
            for (int i = 0; i < projects.length();i++) {
                JSONObject project = (JSONObject) projects.get(i);
                //log.debug("Project data: " + project);
                String projectName = project.getString("name").toLowerCase().replace(" ","_");
                log.debug("Project name: " + projectName);
                String projectKey = project.get("key").toString();
                log.debug("Project key: " + projectKey);


                // Obtain repos
                obtainRepositories(projectKey);
                for (int k = 0; k < repositories.length();k++){
                    JSONObject repository = (JSONObject) repositories.get(k);
                    //log.debug("DEBUG Repository " + repository);
                    String repoName = repository.getString("name");
                    log.debug("Repository name: " + repoName);
                    String repoDir = DEFAULT_DIR + "/" + projectName + "/" + repoName;
                    log.debug("Repository local directory where clone to: " + repoDir);
                    final JSONArray cloneURLs = (JSONArray) ((JSONObject) repository.get("links")).get("clone");
                    for (int r = 0; r < cloneURLs.length();r++){
                        if (((JSONObject)cloneURLs.get(r)).get("name").toString().equals("http")){
                         log.debug("HTTP repository link for clone found.");
                         String repoURL = ((JSONObject) cloneURLs.get(r)).getString("href");
                         cloneRepository(repoURL,repoDir);
                        } else{
                            log.debug(((JSONObject)cloneURLs.get(r)).get("name").toString());
                        }

                    }

                }
            }
        }

    }
    public static void main(String[] args) throws UnirestException {
        Cloner c = new Cloner();
        c.cloner();
    }
}

log4j2.xml

<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="INFO">
    <Appenders>
        <Console name="console-log" target="SYSTEM_OUT">
            <PatternLayout pattern="[%-5level] %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %c{1} - %msg%n"/>
        </Console>
    </Appenders>
    <Loggers>
        <Logger name="org.prokyon.management.bitbucket_cloner" level="debug" additivity="false">
            <appender-ref ref="console-log" level="debug"/>
        </Logger>
        <Root level="info" additivity="false">
            <AppenderRef ref="console-log"/>
        </Root>
    </Loggers>
</Configuration>

pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>org.prokyon.management</groupId>
    <artifactId>bitbucket-cloner</artifactId>
    <version>1.0-SNAPSHOT</version>

<dependencies>
    <dependency>
        <groupId>com.mashape.unirest</groupId>
        <artifactId>unirest-java</artifactId>
        <version>1.4.9</version>
    </dependency>
    <dependency>
        <groupId>org.apache.logging.log4j</groupId>
        <artifactId>log4j-api</artifactId>
        <version>2.4.1</version>
    </dependency>
    <dependency>
        <groupId>org.apache.logging.log4j</groupId>
        <artifactId>log4j-core</artifactId>
        <version>2.4.1</version>
    </dependency>
    <dependency>
    <groupId>org.eclipse.jgit</groupId>
    <artifactId>org.eclipse.jgit</artifactId>
        <version>RELEASE</version>
    </dependency>
</dependencies>
</project>

This is a great solution. Can you please add a readme on how to use this?

Hi, not a problem, actually I might little bit redesign it to be more externally configurable... via some properties file or similar, so you can just provide url, credentials, etc. and you go..

maybe we can create a generic service for cloning repositories and keep them uptodate accross all git/svn/hg systems (they have some remote API), if someone is interested ping me, I have quite nice template about how to implement such system.

The functionality could be for example:

• clone
• continuously compile after clone (so will support build commands/command sequence for every possible build system - autogen, cmake, maven, gradle, nodejs, etc.... whatever)

Ping me if someone would be interested in this... I will join the mission as this is handy.

Hii Archenroot,
I am new in git, can you explain little bit about below lines
String BIT_BUCKET_SERVER = "http://stash.domain.com";
String REST_API_SUFFIX ="/rest/api/1.0";
String API_PROJECTS = "/projects";
String API_REPOSITORIES ="/repos";
String DEFAULT_DIR = "/home/user/projects..";

how we should construct url to hit git