Experiment getting gitlab runner to run jobs without gitlab

main.go

package main

import (
    "fmt"
    "net/http"
    "log"
    "os"
    "io/ioutil"
    "encoding/json"
    b64 "encoding/base64"
    "reflect"
    mux "github.com/gorilla/mux"
    "strconv"
    //"github.com/ghodss/yaml"
)

// Gitea API structs

type GiteaProject struct {
    Id int `json:"id"`
    Name string `json:"name"`
    FullName string `json:"full_name"`
}


// Gitlab API structs

type GitlabFeatureFlags struct {
    Variables bool `json: "variables"`
    Image  bool `json: "image"`
    Services  bool `json: "services"`
    Artifacts bool `json: "artifacts"`
    Cache bool `json: "cache"`
    Shared bool `json: "shared"`
    UploadMultipleArtifacts bool `json: "upload_multiple_artifacts"`
    UploadRawArtifacts bool `json: "upload_raw_artifacts"`
    Session bool `json: "session"`
    Terminal bool `json: "terminal"`
    Refspecs bool `json: "refspecs"`
    Masking bool `json: "masking"`
    Proxy  bool `json: "proxy"`
    RawVariables bool `json: "raw_variables"`
    ArtifactsExclude bool `json: "artifacts_exclude"`
    MultiBuildSteps bool `json: "multi_build_steps"`
    TraceReset bool `json: "trace_reset"`
    TraceChecksum bool `json: "trace_checksum"`
    TraceSize bool `json: "trace_size"`
    VaultSecrets bool `json: "vault_secrets"`
    Cancelable bool `json: "cancelable"`
    ReturnExitCode bool `json: "return_exit_code"`
}

type GitlabFeatureInfo struct {
    TraceSections bool `json: "trac_sections"`
    FailureReasons string `json: "failure_reasons"`
}

type GitlabConfig struct {
    Gpus int `json: "gpus"`
}

type GitlabVersionInfo struct {
    Name string `json:"name,omitempty"`
    Version string `json:"version,omitempty"`
    Revision string `json:"revision,omitempty"`
    Platform string `json:"platform,omitempty"`
    Architecture string `json:"architecture,omitempty"`
    Executor string `json:"executor,omitempty"`
    Shell string `json:"shell,omitempty"`
    Features GitlabFeatureFlags `json:"features,omitempty"`
    Config GitlabConfig `json:"config,omitempty"`
}

type GitlabOutputInfo struct {
    Checksum string `json: "checksum"`
    Bytesize int `json: "bytesize"`
}

type GitlabRequestQuery struct {
    Info GitlabVersionInfo `json: "info"`
    Token string `json: "token"`
    LastUpdate string `json:"last_update" `

    // If something happened
    State string `json:"state,omitempty" `
    Checksum string `json:"checksum,omitempty" `
    Output GitlabOutputInfo `json:"output,omitempty" `
}

// NewSomething create new instance of Something
func NewGitlabFeatureFlags() GitlabFeatureFlags {
    feat := GitlabFeatureFlags{}
    feat.Variables = true
    feat.Image  = false
    feat.Services  = false
    feat.Artifacts = true
    feat.Cache = true
    feat.Shared = true
    feat.UploadMultipleArtifacts = true
    feat.UploadRawArtifacts = true
    feat.Session = true
    feat.Terminal = true
    feat.Refspecs = true
    feat.Masking = true
    feat.Proxy  = false
    feat.RawVariables = true
    feat.ArtifactsExclude = true
    feat.MultiBuildSteps = true
    feat.TraceReset = true
    feat.TraceChecksum = true
    feat.TraceSize = true
    feat.VaultSecrets = true
    feat.Cancelable = true
    feat.ReturnExitCode = true
    return feat
}

type GitlabJobInfo struct {
    Id int `json: "id"`
    Name string `json: "name"`
    Stage string `json: "stag"`
    ProjectId int `json: "project_id"`
    ProjectName string `json: "project_name"`
}

type GitlabGitInfo struct {
    RepoUrl int `json: "repo_url"`
    Ref int `json: "ref"`
    Sha int `json: "sha"`
    BeforeSha int `json: "before_sha"`
    RefType int `json: "ref_type"`
    RefSpecs []string `json: "refspecs"`
    Depth int `json: "depth"`
}

type GitlabRunnerInfo struct {
    Timeout int `json: "timeout"`
    RunnerSessionUrl string `json: "runner_session_url"`
}

type GitlabVariable struct {
    Key string `json: "key"`
    Value string `json: "value"`
    Public bool `json: "public"`
    Masked bool `json: "masked"`
}

type GitlabStep struct {
    Name string `json: "name"`
    Script []string `json: "script"`
    Timeout int `json: "timeout"`
    When string `json: "when"`
    AllowFailure bool `json: "allow_failure"`
}

type GitlabCredential struct {
    Type string `json: "type"`
    Url string `json: "url"`
    Username string `json: "username"`
    Password string `json: "password"`
}

type GitlabDependencyArtifactsFile struct {
    Filename string `json:"filename"`
    Size     int64  `json:"size"`
}

type GitlabDependency struct {
    Id int `json: "id"`
    Name string `json: "name"`
    Token string `json: "token"`
    ArtifactsFile GitlabDependencyArtifactsFile `json:"artifacts_file"`
}

type GitlabCacheKey struct {
    Files []string `json: "files"`
    Key string
}

func (p GitlabCacheKey) MarshalJSON() ([]byte, error) {
    if (p.Key != "") {
        return json.Marshal(p.Key)
    } else {
        return json.Marshal(struct {
            Files []string `json:"files"`
        }{
            Files: p.Files,
        })
    }
}

func (p GitlabCacheKey) UnmarshalJSON(b []byte) error {
    var stuff string
    err := json.Unmarshal(b, &stuff)
    if err != nil {
        print("GitlabCacheKey (prob struct) err=")
        print(err)

        err := json.Unmarshal(b, &p)
        return err
    }

    p.Key = stuff
    return nil
}


type GitlabCacheEntry struct {
    Key GitlabCacheKey `json: "key"`
    Untracked string `json: "untracked"`
    Paths string `json: "paths"`
    When string `json: "when"`
    Policy string `json: "policy"`
}

type GitlabImagePort struct {
    Number   int    `json:"number,omitempty"`
    Protocol string `json:"protocol,omitempty"`
    Name     string `json:"name,omitempty"`
}

type GitlabImage struct {
    Name       string   `json:"name"`
    Alias      string   `json:"alias,omitempty"`
    Command    []string `json:"command,omitempty"`
    Entrypoint []string `json:"entrypoint,omitempty"`
    Ports      []GitlabImagePort   `json:"ports,omitempty"`
}

type GitlabArtifact struct {
    Name      string          `json:"name"`
    Untracked bool            `json:"untracked"`
    Paths     []string   `json:"paths"`
    Exclude   []string `json:"exclude"`
    When      []string    `json:"when"`
    Type      string          `json:"artifact_type"`
    Format    string  `json:"artifact_format"`
    ExpireIn  string          `json:"expire_in"`
}


type GitlabSecrets map[string]GitlabSecret

type GitlabSecret struct {
    Vault *GitlabVaultSecret `json:"vault,omitempty"`

    File *bool `json:"file,omitempty"`
}


type GitlabVaultSecret struct {
    Server GitlabVaultServer `json:"server"`
    Engine GitlabVaultEngine `json:"engine"`
    Path   string      `json:"path"`
    Field  string      `json:"field"`
}

type GitlabVaultServer struct {
    URL  string    `json:"url"`
    Auth GitlabVaultAuth `json:"auth"`
}

type GitlabVaultAuthData map[string]interface{}

type GitlabVaultAuth struct {
    Name string        `json:"name"`
    Path string        `json:"path"`
    Data GitlabVaultAuthData `json:"data"`
}

type GitlabVaultEngine struct {
    Name string `json:"name"`
    Path string `json:"path"`
}


type GitlabJobRequest struct {
    Info       GitlabVersionInfo  `json:"info,omitempty"`
    Token      string       `json:"token,omitempty"`
    LastUpdate string       `json:"last_update,omitempty"`
    Session    *GitlabSessionInfo `json:"session,omitempty"`
}

type GitlabSessionInfo struct {
    URL           string `json:"url,omitempty"`
    Certificate   string `json:"certificate,omitempty"`
    Authorization string `json:"authorization,omitempty"`
}

type GitlabJobResponse struct {
    Id int `json: "id"`
    Token string `json: "token"`
    AllowGitFetch bool `json: "allow_git_fetch"`
    JobInfo GitlabJobInfo `json: "job_info"`
    GitInfo GitlabGitInfo `json: "git_info"`
    RunnerInfo GitlabRunnerInfo `json: "runner_info"`
    Variables []GitlabVariable `json: "variables"`
    Steps []GitlabStep `json: "steps"`
    Image GitlabImage `json: "image"`
    Services []string `json: "services"`
    Artifacts []GitlabArtifact `json: "artifacts"`
    Cache []GitlabCacheEntry `json: "cache"`
    Credentials []GitlabCredential `json: "credentials"`
    Dependencies []GitlabDependency `json: "dependencies"`
    Features GitlabFeatureInfo `json: "features"`
    Secrets       GitlabSecrets        `json:"secrets,omitempty"`
}


type GitlabRegisterRunnerParameters struct {
    Description    string `json:"description,omitempty"`
    Tags           string `json:"tag_list,omitempty"`
    RunUntagged    bool   `json:"run_untagged"`
    Locked         bool   `json:"locked"`
    AccessLevel    string `json:"access_level,omitempty"`
    MaximumTimeout int    `json:"maximum_timeout,omitempty"`
    Active         bool   `json:"active"`
}

type GitlabRegisterRunnerRequest struct {
    GitlabRegisterRunnerParameters
    Info  GitlabVersionInfo `json:"info,omitempty"`
    Token string      `json:"token,omitempty"`
}

type GitlabTokenQuery struct {
    Token string `json:"token,omitempty"`
}

type GitlabRegistrationResponse struct {
    Message map[string]string `json: "message"`
}



func main() {

    var JobSent bool
    TestJob := GitlabJobResponse{}
    TestJobWorkerToken := "TEST_WORKER_TOKEN"
    TestWorkerRegistered := false

    // Build folder should be of form: project-ProjectID-ProjectRunnerID
    CurrentJobId := 100

    TestJob.Id = 100
    TestJob.Token = "TEST_JOB_TOKEN"
    TestJob.AllowGitFetch = false
    TestJob.Variables = []GitlabVariable{
        GitlabVariable{ Key: "CI_PIPELINE_ID", Value: "", Public: true, Masked: false },
        GitlabVariable{ Key: "CI_PIPELINE_URL", Value: "http://lappo.local:8020/mango/project1/-/pipelines/1", Public: true, Masked: false },
        GitlabVariable{ Key: "CI_JOB_URL", Value: "http://lappo.local:8020/mango/project1/-/jobs/" + string(CurrentJobId), Public: true, Masked: false },
        GitlabVariable{ Key: "CI_JOB_TOKEN", Value: "AAA", Public: true, Masked: false },
        GitlabVariable{ Key: "CI_REPOSITORY_URL", Value: "AAA", Public: true, Masked: false },
        GitlabVariable{ Key: "CI_JOB_NAME", Value: "build-job", Public: true, Masked: false },
        GitlabVariable{ Key: "CI_JOB_STAGE", Value: "build", Public: true, Masked: false },
        GitlabVariable{ Key: "CI_BUILD_NAME", Value: "build-job", Public: true, Masked: false },
        GitlabVariable{ Key: "CI", Value: "true", Public: true, Masked: false },
        GitlabVariable{ Key: "GITLAB_CI", Value: "true", Public: true, Masked: false } }

    TestJob.JobInfo.Id = CurrentJobId
    TestJob.JobInfo.Name = "test_job"
    TestJob.JobInfo.ProjectId = 1
    TestJob.JobInfo.ProjectName = "project1"
    TestJob.JobInfo.Stage = "test_job"
    
    TestJob.GitInfo = GitlabGitInfo{}

    TestJob.RunnerInfo = GitlabRunnerInfo{ Timeout: 10000 }
    TestJob.Steps = []GitlabStep{
        GitlabStep{ Name: "script", Script: []string{"echo \"Helllo world!\""}, Timeout: 3600, When: "on_success", AllowFailure: false },
    }

    router := mux.NewRouter()

    // Jobs
    router.HandleFunc("/api/v4/runners", func(w http.ResponseWriter, r *http.Request) {
        // Should return a list of runners OR register one

        w.Header().Set("Content-Type", "application/json")

        switch r.Method {
        //case http.MethodGet:
        case http.MethodPost:

            var reg_params GitlabRegisterRunnerRequest 
            json.NewDecoder(r.Body).Decode(&reg_params)

            fmt.Printf("REG REQUEST: %+v\n", reg_params)

            response := GitlabTokenQuery{}
            response.Token = TestJobWorkerToken

            w.WriteHeader(http.StatusCreated)
            json.NewEncoder(w).Encode(response)

            TestWorkerRegistered = true

        case http.MethodDelete:

            var token_params GitlabTokenQuery 
            json.NewDecoder(r.Body).Decode(&token_params)

            fmt.Printf("UNREG REQUEST: %+v\n", token_params)

            if TestWorkerRegistered {
                w.WriteHeader(http.StatusNoContent)
            } else {
                w.WriteHeader(http.StatusNoContent)
            }
            TestWorkerRegistered = false

        default:
            http.Error(w, "{}", http.StatusBadRequest)
        }

    })

    router.HandleFunc("/api/v4/jobs/request", func(w http.ResponseWriter, r *http.Request) {

        // Posts basic feature info with no state usually
        // Should return 200 OR
        // 201 Created & GitlabJobResponse
        // Gitlab-Ci-Builds-Polling: yes
        fmt.Println("REQ JOB!")

        w.Header().Set("Content-Type", "application/json")

        switch r.Method {
        case http.MethodPost:
            var status GitlabJobRequest
            json.NewDecoder(r.Body).Decode(&status)
            
            if status.Token != TestJobWorkerToken {
                fmt.Println("Worker submitted bad token!")
                w.WriteHeader(http.StatusBadRequest)
                return;
            }

            if !JobSent {
                fmt.Println("Sending example job")
                w.WriteHeader(http.StatusCreated)
                json.NewEncoder(w).Encode(TestJob)
            } else {
                fmt.Println("Job submitted already, no more jobs")
                w.WriteHeader(http.StatusNoContent)
            }

            JobSent = true
        default:
            http.Error(w, "Invalid", http.StatusMethodNotAllowed)
        }

    })

    router.HandleFunc("/api/v4/jobs/runner/verify", func(w http.ResponseWriter, r *http.Request) {

        w.Header().Set("Content-Type", "application/json")

        switch r.Method {
        case http.MethodPost:
            if !JobSent {
                json.NewEncoder(w).Encode(TestJob)
            } else {
                w.WriteHeader(200)
            }
        default:
            http.Error(w, "Invalid", http.StatusMethodNotAllowed)
        }
    })

    router.HandleFunc("/api/v4/jobs/{id}", func(w http.ResponseWriter, r *http.Request) {
        // Should normally return 200
        // Basically updates state of runner 

        vars := mux.Vars(r)
        jobId, _ := strconv.Atoi(vars["id"])

        if jobId != CurrentJobId {
            w.WriteHeader(http.StatusNotFound)
            return
        }

        switch r.Method {
        case http.MethodPut:
            bytes, err := ioutil.ReadAll(r.Body)
            if err != nil {
                http.Error(w, "Invalid body", http.StatusMethodNotAllowed)
                return
            }
            os.Stdout.Write(bytes[:])
            w.WriteHeader(http.StatusOK)

        default:
            http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
        }
    })

    router.HandleFunc("/api/v4/jobs/{id}/trace", func(w http.ResponseWriter, r *http.Request) {
        // Requires: job-token
        // TODO: check this
        //jobToken := r.Header.Get("Job-Token")

        vars := mux.Vars(r)
        jobId, _ := strconv.Atoi(vars["id"])

        if jobId != CurrentJobId {
            w.WriteHeader(http.StatusNotFound)
            return
        }

        switch r.Method {
        case http.MethodPatch:
            bytes, err := ioutil.ReadAll(r.Body)
            if err != nil {

                fmt.Printf("error=%+v\n", err);
                http.Error(w, "Invalid body", http.StatusMethodNotAllowed)
                return
            }
            os.Stdout.Write(bytes[:])

            w.WriteHeader(http.StatusOK)

        default:
            http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
        }
        // 
    })

    // Build artifacts
    router.HandleFunc("/api/v4/jobs/{id}/artifacts", func(w http.ResponseWriter, r *http.Request) {
        // TODO
    })

    router.HandleFunc("/api/v4/jobs/{id}/artifacts/{artifact_id}", func(w http.ResponseWriter, r *http.Request) {
        // TODO
    })

    router.HandleFunc("/test", func(w http.ResponseWriter, r *http.Request) {

        var body []byte = nil
        finished := make(chan bool)

        poll_gitea := func () {
            client := &http.Client{}
            req, err := http.NewRequest("GET", "http://192.168.2.112:8081/api/v1/repos/mango/hello-woodpecker/contents/.woodpecker.yml", nil)
            req.Header.Add("Authorization", "token " + "f67a78c84db0eff0c7aced2f676a039b06f712ca")

            if err != nil {
                log.Print(err)
                os.Exit(1)
            }

            // token=f67a78c84db0eff0c7aced2f676a039b06f712ca

            resp, err := client.Do(req)
            if err != nil {
                panic(err)
            }
            defer resp.Body.Close()

            ibody, _ := ioutil.ReadAll(resp.Body)
            body = ibody
            finished <- true
        }

        go poll_gitea()
        <- finished


        var objmap map[string]json.RawMessage
        err := json.Unmarshal(body, &objmap)
        if err != nil {
            panic(err)
        }

        var filedata string = ""
        json.Unmarshal(objmap["content"], &filedata)

        fdatas, _ := b64.StdEncoding.DecodeString(filedata)




        fmt.Fprintf(w, "Response %s\n", fdatas)
    })

    router.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {

        var body []byte = nil
        finished := make(chan bool)

        poll_gitea := func () {
            client := &http.Client{}
            req, err := http.NewRequest("GET", "http://192.168.2.112:8081/api/v1/repos/search", nil)
            req.Header.Add("Authorization", "token " + "f67a78c84db0eff0c7aced2f676a039b06f712ca")

            if err != nil {
                log.Print(err)
                os.Exit(1)
            }

            // token=f67a78c84db0eff0c7aced2f676a039b06f712ca

            resp, err := client.Do(req)
            if err != nil {
                panic(err)
            }
            defer resp.Body.Close()

            ibody, _ := ioutil.ReadAll(resp.Body)
            body = ibody
            finished <- true
        }

        go poll_gitea()
        <- finished

        var objmap map[string]json.RawMessage
        err := json.Unmarshal(body, &objmap)
        if err != nil {
            panic(err)
        }

        keys := reflect.ValueOf(objmap).MapKeys()
        fmt.Printf("%+v\n", keys)


        var plist []GiteaProject = make([]GiteaProject,0);

        json.Unmarshal(objmap["data"], &plist)

        fmt.Fprintf(w, "<table><thead><tr><th>id</th><th>tag</th><th>name</th></tr></thead><tbody>")
        for _, proj := range plist {
            fmt.Printf("%+v\n", proj)

            fmt.Fprintf(w, "<tr><td>%d</td><td>%s</td><td>%s</td></tr>\n", proj.Id, proj.Name, proj.FullName)
        }
        fmt.Fprintf(w, "</tbody></table>")


        fmt.Fprintf(w, "Response %s\n", body)
    })

    http.Handle("/", router)

    http.ListenAndServe("0.0.0.0:8020", nil)
}