Clone Stash/BitBucket - all repositories from all projects which user has access to

Cloner.java

package org.prokyon.management.bitbucket_cloner;

import com.google.common.base.CaseFormat;
import com.mashape.unirest.http.HttpResponse;
import com.mashape.unirest.http.JsonNode;
import com.mashape.unirest.http.Unirest;
import com.mashape.unirest.http.exceptions.UnirestException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
import org.json.JSONArray;
import org.json.JSONObject;

import java.io.File;


/**
 * Created by zangetsu on 1/31/17.
 */
public class Cloner {

    private static final Logger log = LogManager.getLogger(Cloner.class);

    String BIT_BUCKET_SERVER = "http://stash.domain.com";
    String REST_API_SUFFIX ="/rest/api/1.0";
    String API_PROJECTS = "/projects";
    String API_REPOSITORIES ="/repos";
    String DEFAULT_DIR = "/home/user/projects..";

    String USER = "user";
    String PASSWORD = "password";

    public static JSONArray projects = null;
    public static JSONArray repositories = null;

    private void obtainProjects() {
        log.debug("Going to obtain projects from: " + BIT_BUCKET_SERVER +REST_API_SUFFIX + API_PROJECTS);

        try {
            projects = Unirest.get(BIT_BUCKET_SERVER +REST_API_SUFFIX + API_PROJECTS)
                    .basicAuth(USER, PASSWORD)
                    .header("accept", "application/json")
                    .asJson()
                    .getBody()
                    .getObject().getJSONArray("values");
        } catch (UnirestException e) {
            log.error("Cannot obtain list of projects from " + BIT_BUCKET_SERVER +REST_API_SUFFIX + API_PROJECTS);
        }
    }

    private void obtainRepositories(String projectKey) {
        String projectReposURL = BIT_BUCKET_SERVER +REST_API_SUFFIX + API_PROJECTS + "/" + projectKey + API_REPOSITORIES;
        log.info("Obtaining repos from " + projectReposURL);
        try {
            repositories = Unirest.get(projectReposURL)
                    .basicAuth(USER, PASSWORD)
                    .header("accept", "application/json")
                    .asJson()
                    .getBody()
                    .getObject().getJSONArray("values");
        } catch (UnirestException e) {
            log.error("Cannot obtain project repositories from " + projectReposURL);
        }
        //log.debug("Repositories obtained: " + repositories);

    }
    private void cloneRepository(String gitURL,String repoDir){
        log.info("Going to clone repo " + gitURL);
        log.info("Repository will be stored at " + repoDir);
        try {
            Git git = Git.cloneRepository()
               .setURI(gitURL)
               .setDirectory(new File(repoDir))
               .setCloneAllBranches(true)
               .setCredentialsProvider(new UsernamePasswordCredentialsProvider(USER, PASSWORD))
               .call();

        } catch (GitAPIException e) {
            log.error("Error on cloning repository from " + gitURL + " into local directory " + repoDir + ". Check the path.");
        }
    }
    public void cloner(){
        // Obtain projects
        obtainProjects();
        if (projects.length()<1){
            log.info("There are no projects available to process.");
        } else{
            for (int i = 0; i < projects.length();i++) {
                JSONObject project = (JSONObject) projects.get(i);
                //log.debug("Project data: " + project);
                String projectName = project.getString("name").toLowerCase().replace(" ","_");
                log.debug("Project name: " + projectName);
                String projectKey = project.get("key").toString();
                log.debug("Project key: " + projectKey);


                // Obtain repos
                obtainRepositories(projectKey);
                for (int k = 0; k < repositories.length();k++){
                    JSONObject repository = (JSONObject) repositories.get(k);
                    //log.debug("DEBUG Repository " + repository);
                    String repoName = repository.getString("name");
                    log.debug("Repository name: " + repoName);
                    String repoDir = DEFAULT_DIR + "/" + projectName + "/" + repoName;
                    log.debug("Repository local directory where clone to: " + repoDir);
                    final JSONArray cloneURLs = (JSONArray) ((JSONObject) repository.get("links")).get("clone");
                    for (int r = 0; r < cloneURLs.length();r++){
                        if (((JSONObject)cloneURLs.get(r)).get("name").toString().equals("http")){
                         log.debug("HTTP repository link for clone found.");
                         String repoURL = ((JSONObject) cloneURLs.get(r)).getString("href");
                         cloneRepository(repoURL,repoDir);
                        } else{
                            log.debug(((JSONObject)cloneURLs.get(r)).get("name").toString());
                        }

                    }

                }
            }
        }

    }
    public static void main(String[] args) throws UnirestException {
        Cloner c = new Cloner();
        c.cloner();
    }
}

log4j2.xml

<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="INFO">
    <Appenders>
        <Console name="console-log" target="SYSTEM_OUT">
            <PatternLayout pattern="[%-5level] %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %c{1} - %msg%n"/>
        </Console>
    </Appenders>
    <Loggers>
        <Logger name="org.prokyon.management.bitbucket_cloner" level="debug" additivity="false">
            <appender-ref ref="console-log" level="debug"/>
        </Logger>
        <Root level="info" additivity="false">
            <AppenderRef ref="console-log"/>
        </Root>
    </Loggers>
</Configuration>

pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>org.prokyon.management</groupId>
    <artifactId>bitbucket-cloner</artifactId>
    <version>1.0-SNAPSHOT</version>

<dependencies>
    <dependency>
        <groupId>com.mashape.unirest</groupId>
        <artifactId>unirest-java</artifactId>
        <version>1.4.9</version>
    </dependency>
    <dependency>
        <groupId>org.apache.logging.log4j</groupId>
        <artifactId>log4j-api</artifactId>
        <version>2.4.1</version>
    </dependency>
    <dependency>
        <groupId>org.apache.logging.log4j</groupId>
        <artifactId>log4j-core</artifactId>
        <version>2.4.1</version>
    </dependency>
    <dependency>
    <groupId>org.eclipse.jgit</groupId>
    <artifactId>org.eclipse.jgit</artifactId>
        <version>RELEASE</version>
    </dependency>
</dependencies>
</project>

maybe we can create a generic service for cloning repositories and keep them uptodate accross all git/svn/hg systems (they have some remote API), if someone is interested ping me, I have quite nice template about how to implement such system.

The functionality could be for example:

• clone
• continuously compile after clone (so will support build commands/command sequence for every possible build system - autogen, cmake, maven, gradle, nodejs, etc.... whatever)

Ping me if someone would be interested in this... I will join the mission as this is handy.

Hii Archenroot,
I am new in git, can you explain little bit about below lines
String BIT_BUCKET_SERVER = "http://stash.domain.com";
String REST_API_SUFFIX ="/rest/api/1.0";
String API_PROJECTS = "/projects";
String API_REPOSITORIES ="/repos";
String DEFAULT_DIR = "/home/user/projects..";

how we should construct url to hit git