ps_test.go

package tests

import (
	"fmt"
	"math/rand"
	"os"
	"regexp"
	"sort"
	"strconv"
	"sync"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
)

var procsRegexp = `(%s-v\d+-[\w-]+) up \(v\d+\)`

// TODO: https://github.com/deis/workflow-e2e/issues/108
//       for example, these could live in common/certs.go
// certs-specific common actions and expectations
func listProcs(testApp App) *Session {
	sess, err := start("deis ps:list --app=%s", testApp.Name)
	Expect(err).NotTo(HaveOccurred())
	Eventually(sess).Should(Say("=== %s Processes", testApp.Name))
	Eventually(sess).Should(Exit(0))
	return sess
}

// scrapeProcs returns the sorted process names for an app from the given output.
// It matches the current "deis ps" output for a healthy container:
//   earthy-vocalist-v2-cmd-1d73e up (v2)
//   myapp-v16-web-bujlq up (v16)
func scrapeProcs(app string, output []byte) []string {
	re := regexp.MustCompile(fmt.Sprintf(procsRegexp, app))
	found := re.FindAllSubmatch(output, -1)
	procs := make([]string, len(found))
	for i := range found {
		procs[i] = string(found[i][1])
	}
	sort.Strings(procs)
	return procs
}

var _ = Describe("Processes", func() {

	Context("with a deployed app", func() {

		var testApp App
		once := &sync.Once{}

		BeforeEach(func() {
			// Set up the Processes test app only once and assume the suite will clean up.
			once.Do(func() {
				os.Chdir("example-go")
				appName := getRandAppName()
				createApp(appName)
				testApp = deployApp(appName)
			})
		})

		DescribeTable("can scale up and down",

			func(scaleTo int, respCode []int) {
				// TODO: need some way to choose between "web" and "cmd" here!
				// scale the app's processes to the desired number
				sess, err := start("deis ps:scale web=%d --app=%s", scaleTo, testApp.Name)
				Expect(err).NotTo(HaveOccurred())
				Eventually(sess).Should(Say("Scaling processes... but first,"))
				Eventually(sess, defaultMaxTimeout).Should(Say(`done in \d+s`))
				Eventually(sess).Should(Say("=== %s Processes", testApp.Name))
				Eventually(sess).Should(Exit(0))

				// test that there are the right number of processes listed
				procsListing := listProcs(testApp).Out.Contents()
				procs := scrapeProcs(testApp.Name, procsListing)
				Expect(len(procs)).To(Equal(scaleTo))

				// curl the app's root URL and print just the HTTP response code
				// sess, err = start(`curl -sL -w "%%{http_code}\\n" "%s" -o /dev/null`, testApp.URL)
				// Eventually(sess).Should(Say(strconv.Itoa(respCode)))
				// Eventually(sess).Should(Exit(0))
				code, err := curl(testApp, scaleTo, respCode)
				Expect(err).NotTo(HaveOccurred())
				//Expect(code).To(Equal(respCode))
				Expect(respCode).To(ContainElement(code))
			},

			Entry("scales to 1", 1, []int{200, 404}),
			Entry("scales to 3", 3, []int{200}),
			Entry("scales to 0", 0, []int{502}),
			Entry("scales to 5", 5, []int{200}),
			Entry("scales to 0", 0, []int{502}),
			Entry("scales to 1", 1, []int{200}),
		)

		DescribeTable("can restart processes",

			func(restart string, scaleTo int, respCode int) {
				// TODO: need some way to choose between "web" and "cmd" here!
				// scale the app's processes to the desired number
				sess, err := start("deis ps:scale web=%d --app=%s", scaleTo, testApp.Name)
				Expect(err).NotTo(HaveOccurred())
				Eventually(sess).Should(Say("Scaling processes... but first,"))
				Eventually(sess, defaultMaxTimeout).Should(Say(`done in \d+s`))
				Eventually(sess).Should(Say("=== %s Processes", testApp.Name))
				Eventually(sess).Should(Exit(0))

				// capture the process names
				beforeProcs := scrapeProcs(testApp.Name, sess.Out.Contents())

				// restart the app's process(es)
				var arg string
				switch restart {
				case "all":
					arg = ""
				case "by type":
					// TODO: need some way to choose between "web" and "cmd" here!
					arg = "web"
				case "by wrong type":
					// TODO: need some way to choose between "web" and "cmd" here!
					arg = "cmd"
				case "one":
					procsLen := len(beforeProcs)
					Expect(procsLen).To(BeNumerically(">", 0))
					arg = beforeProcs[rand.Intn(procsLen)]
				}
				sess, err = start("deis ps:restart %s --app=%s", arg, testApp.Name)
				Expect(err).NotTo(HaveOccurred())
				Eventually(sess).Should(Say("Restarting processes... but first,"))
				if scaleTo == 0 || restart == "by wrong type" {
					Eventually(sess).Should(Say("Could not find any processes to restart"))
				} else {
					Eventually(sess, defaultMaxTimeout).Should(Say(`done in \d+s`))
					Eventually(sess).Should(Say("=== %s Processes", testApp.Name))
				}
				Eventually(sess).Should(Exit(0))

				// capture the process names
				procsListing := listProcs(testApp).Out.Contents()
				afterProcs := scrapeProcs(testApp.Name, procsListing)

				// compare the before and after sets of process names
				Expect(len(afterProcs)).To(Equal(scaleTo))
				if scaleTo > 0 && restart != "by wrong type" {
					Expect(beforeProcs).NotTo(Equal(afterProcs))
				}

				// curl the app's root URL and print just the HTTP response code
				sess, err = start(`curl -sL -w "%%{http_code}\\n" "%s" -o /dev/null`, testApp.URL)
				Eventually(sess).Should(Say(strconv.Itoa(respCode)))
				Eventually(sess).Should(Exit(0))

			},

			Entry("restarts one of 1", "one", 1, 200),
			Entry("restarts all of 1", "all", 1, 200),
			Entry("restarts all of 1 by type", "by type", 1, 200),
			Entry("restarts all of 1 by wrong type", "by wrong type", 1, 200),
			Entry("restarts one of 6", "one", 6, 200),
			Entry("restarts all of 6", "all", 6, 200),
			Entry("restarts all of 6 by type", "by type", 6, 200),
			Entry("restarts all of 6 by wrong type", "by wrong type", 6, 200),
			Entry("restarts all of 0", "all", 0, 502),
			Entry("restarts all of 0 by type", "by type", 0, 502),
			Entry("restarts all of 0 by wrong type", "by wrong type", 0, 502),
		)
	})
})

tests_suite_test.go

package tests

import (
	"crypto/tls"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	neturl "net/url"
	"os"
	"os/exec"
	"path"
	"regexp"
	"runtime"
	"strings"
	"testing"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"

	"github.com/onsi/ginkgo/reporters"
	client "k8s.io/kubernetes/pkg/client/unversioned"
)

type Cmd struct {
	Env               []string
	CommandLineString string
}

type App struct {
	Name string
	URL  string
}

const (
	deisRouterServiceHost = "DEIS_ROUTER_SERVICE_HOST"
	deisRouterServicePort = "DEIS_ROUTER_SERVICE_PORT"
)

var (
	errMissingRouterHostEnvVar = fmt.Errorf("missing %s", deisRouterServiceHost)
	defaultMaxTimeout          = 5 * time.Minute // gomega's default is 2 minutes
)

func getDir() string {
	var _, inDockerContainer = os.LookupEnv("DOCKERIMAGE")
	if inDockerContainer {
		return "/"
	}
	_, filename, _, _ := runtime.Caller(1)
	return path.Dir(filename)
}

func init() {
	rand.Seed(time.Now().UnixNano())
}

func getRandAppName() string {
	return fmt.Sprintf("test-%d", rand.Intn(999999999))
}

func TestTests(t *testing.T) {
	RegisterFailHandler(Fail)

	enableJunit := os.Getenv("JUNIT")
	if enableJunit == "true" {
		junitReporter := reporters.NewJUnitReporter("junit.xml")
		RunSpecsWithDefaultAndCustomReporters(t, "Deis Workflow", []Reporter{junitReporter})
	} else {
		RunSpecs(t, "Deis Workflow")
	}
}

var (
	randSuffix        = rand.Intn(1000)
	testUser          = fmt.Sprintf("test-%d", randSuffix)
	testPassword      = "asdf1234"
	testEmail         = fmt.Sprintf("test-%[email protected]", randSuffix)
	testAdminUser     = "admin"
	testAdminPassword = "admin"
	testAdminEmail    = "[email protected]"
	keyName           = fmt.Sprintf("deiskey-%v", randSuffix)
	url               = getController()
	debug             = os.Getenv("DEBUG") != ""
	homeHome          = os.Getenv("HOME")
)

var testRoot, testHome, keyPath, gitSSH string

var _ = BeforeSuite(func() {
	SetDefaultEventuallyTimeout(10 * time.Second)

	// use the "deis" executable in the search $PATH
	output, err := exec.LookPath("deis")
	Expect(err).NotTo(HaveOccurred(), output)

	testHome, err = ioutil.TempDir("", "deis-workflow-home")
	Expect(err).NotTo(HaveOccurred())
	os.Setenv("HOME", testHome)

	// register the test-admin user
	registerOrLogin(url, testAdminUser, testAdminPassword, testAdminEmail)

	// verify this user is an admin by running a privileged command
	sess, err := start("deis users:list")
	Expect(err).To(BeNil())
	Eventually(sess).Should(Exit(0))

	sshDir := path.Join(testHome, ".ssh")

	// register the test user and add a key
	registerOrLogin(url, testUser, testPassword, testEmail)

	keyPath = createKey(keyName)

	// Write out a git+ssh wrapper file to avoid known_hosts warnings
	gitSSH = path.Join(sshDir, "git-ssh")
	sshFlags := "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
	if debug {
		sshFlags = sshFlags + " -v"
	}
	ioutil.WriteFile(gitSSH, []byte(fmt.Sprintf(
		"#!/bin/sh\nSSH_ORIGINAL_COMMAND=\"ssh $@\"\nexec /usr/bin/ssh %s -i %s \"$@\"\n",
		sshFlags, keyPath)), 0777)

	sess, err = start("deis keys:add %s.pub", keyPath)
	Expect(err).To(BeNil())
	Eventually(sess).Should(Exit(0))
	Eventually(sess).Should(Say("Uploading %s.pub to deis... done", keyName))

	time.Sleep(5 * time.Second) // wait for ssh key to propagate
})

var _ = BeforeEach(func() {
	var err error
	var output string

	testRoot, err = ioutil.TempDir("", "deis-workflow-test")
	Expect(err).NotTo(HaveOccurred())

	os.Chdir(testRoot)
	output, err = execute(`git clone https://github.com/deis/example-go.git`)
	Expect(err).NotTo(HaveOccurred(), output)
	output, err = execute(`git clone https://github.com/deis/example-perl.git`)
	Expect(err).NotTo(HaveOccurred(), output)

	login(url, testUser, testPassword)
})

var _ = AfterEach(func() {
	err := os.RemoveAll(testRoot)
	Expect(err).NotTo(HaveOccurred())
})

var _ = AfterSuite(func() {
	os.Chdir(testHome)

	cancel(url, testUser, testPassword)
	cancel(url, testAdminUser, testAdminPassword)

	os.RemoveAll(fmt.Sprintf("~/.ssh/%s*", keyName))

	err := os.RemoveAll(testHome)
	Expect(err).NotTo(HaveOccurred())

	os.Setenv("HOME", homeHome)
})

func register(url, username, password, email string) {
	sess, err := start("deis register %s --username=%s --password=%s --email=%s", url, username, password, email)
	Expect(err).To(BeNil())
	Eventually(sess).Should(Say("Registered %s", username))
	Eventually(sess).Should(Say("Logged in as %s", username))
}

func registerOrLogin(url, username, password, email string) {
	sess, err := start("deis register %s --username=%s --password=%s --email=%s", url, username, password, email)

	Expect(err).To(BeNil())

	sess.Wait()

	if strings.Contains(string(sess.Err.Contents()), "must be unique") {
		// Already registered
		login(url, username, password)
	} else {
		Eventually(sess).Should(Exit(0))
		Eventually(sess).Should(SatisfyAll(
			Say("Registered %s", username),
			Say("Logged in as %s", username)))
	}
}

func cancel(url, username, password string) {
	// log in to the account
	login(url, username, password)

	// remove any existing test-* apps
	sess, err := start("deis apps")
	Expect(err).To(BeNil())
	Eventually(sess).Should(Exit(0))
	re := regexp.MustCompile("test-.*")
	for _, app := range re.FindAll(sess.Out.Contents(), -1) {
		sess, err = start("deis destroy --app=%s --confirm=%s", app, app)
		Expect(err).To(BeNil())
		Eventually(sess).Should(Say("Destroying %s...", app))
		Eventually(sess).Should(Exit(0))
	}

	// cancel the account
	sess, err = start("deis auth:cancel --username=%s --password=%s --yes", username, password)
	Expect(err).To(BeNil())
	Eventually(sess).Should(Exit(0))
	Eventually(sess).Should(Say("Account cancelled"))
}

func login(url, user, password string) {
	sess, err := start("deis login %s --username=%s --password=%s", url, user, password)
	Expect(err).To(BeNil())
	Eventually(sess).Should(Exit(0))
	Eventually(sess).Should(Say("Logged in as %s", user))
}

func logout() {
	sess, err := start("deis auth:logout")
	Expect(err).To(BeNil())
	Eventually(sess).Should(Exit(0))
	Eventually(sess).Should(Say("Logged out\n"))
}

// execute executes the command generated by fmt.Sprintf(cmdLine, args...) and returns its output as a cmdOut structure.
// this structure can then be matched upon using the SucceedWithOutput matcher below
func execute(cmdLine string, args ...interface{}) (string, error) {
	var cmd *exec.Cmd
	shCommand := fmt.Sprintf(cmdLine, args...)

	if debug {
		fmt.Println(shCommand)
	}

	cmd = exec.Command("/bin/sh", "-c", shCommand)
	outputBytes, err := cmd.CombinedOutput()

	output := string(outputBytes)

	if debug {
		fmt.Println(output)
	}

	return output, err
}

func start(cmdLine string, args ...interface{}) (*Session, error) {
	ourCommand := Cmd{Env: os.Environ(), CommandLineString: fmt.Sprintf(cmdLine, args...)}
	return startCmd(ourCommand)
}

func startCmd(command Cmd) (*Session, error) {
	cmd := exec.Command("/bin/sh", "-c", command.CommandLineString)
	cmd.Env = command.Env
	io.WriteString(GinkgoWriter, fmt.Sprintf("$ %s\n", command.CommandLineString))
	return Start(cmd, GinkgoWriter, GinkgoWriter)
}

func createKey(name string) string {
	keyPath := path.Join(testHome, ".ssh", name)
	os.MkdirAll(path.Join(testHome, ".ssh"), 0777)
	// create the key under ~/.ssh/<name> if it doesn't already exist
	if _, err := os.Stat(keyPath); os.IsNotExist(err) {
		sess, err := start("ssh-keygen -q -t rsa -b 4096 -C %s -f %s -N ''", name, keyPath)
		Expect(err).To(BeNil())
		Eventually(sess).Should(Exit(0))
	}

	os.Chmod(keyPath, 0600)

	return keyPath
}

func getController() string {
	host := os.Getenv(deisRouterServiceHost)
	if host == "" {
		panicStr := fmt.Sprintf(`Set the router host and port for tests, such as:

$ %s=192.0.2.10 %s=31182 make test-integration`, deisRouterServiceHost, deisRouterServicePort)
		panic(panicStr)
	}
	// Make a xip.io URL if DEIS_ROUTER_SERVICE_HOST is an IP V4 address
	ipv4Regex := `^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$`
	matched, err := regexp.MatchString(ipv4Regex, host)
	if err != nil {
		panic(err)
	}
	if matched {
		host = fmt.Sprintf("deis.%s.xip.io", host)
	}
	port := os.Getenv(deisRouterServicePort)
	switch port {
	case "443":
		return "https://" + host
	case "80", "":
		return "http://" + host
	default:
		return fmt.Sprintf("http://%s:%s", host, port)
	}
}

// getRawRouter returns the URL to the deis router according to env vars.
//
// Returns an error if the minimal env vars are missing, or there was an error creating a URL from them.
func getRawRouter() (*neturl.URL, error) {
	host := os.Getenv(deisRouterServiceHost)
	if host == "" {
		return nil, errMissingRouterHostEnvVar
	}
	portStr := os.Getenv(deisRouterServicePort)
	switch portStr {
	case "443":
		return neturl.Parse(fmt.Sprintf("https://%s", host))
	case "80", "":
		return neturl.Parse(fmt.Sprintf("http://%s", host))
	default:
		return neturl.Parse(fmt.Sprintf("http://%s:%s", host, portStr))
	}
}

func createApp(name string, options ...string) *Session {
	var noRemote = false
	cmd, err := start("deis apps:create %s %s", name, strings.Join(options, " "))
	Expect(err).NotTo(HaveOccurred())
	Eventually(cmd).Should(Say("created %s", name))
	Eventually(cmd).Should(Exit(0))

	for _, option := range options {
		if option == "--no-remote" {
			noRemote = true
		}
	}

	if !noRemote {
		Eventually(cmd).Should(Say("Git remote deis added"))
	}
	Eventually(cmd).Should(Say("remote available at "))

	return cmd
}

func curl(app App, scalefactor int, statuscodes []int) (int, error) {
	kubeClient, err := client.NewInCluster()
	if err != nil {
		return 0, fmt.Errorf("unable to create client")
	}
	epclient := kubeClient.Endpoints(app.Name)
	maxretries := 3
	retries := 0
	statuscode := 0
	for {
		eps, err := epclient.Get(app.Name)
		if err != nil {
			return 0, fmt.Errorf("not able to get app service endpoints")
		}
		if len(eps.Subsets) != 0 && scalefactor != 0 {
			time.Sleep(400 * time.Millisecond)
			if scalefactor == len(eps.Subsets[0].Addresses) && retries <= maxretries {
				statuscode, err = doCurl(app.URL, false)
				for _, code := range statuscodes {
					if statuscode == code {
						return statuscode, nil
					}
				}
				retries++
			}
			if retries > maxretries {
				return statuscode, fmt.Errorf("request timedout number of endpoints %s", eps.Subsets[0].Addresses)
			}
		}
		if scalefactor != 0 {
			return doCurl(app.URL, true)
		}
	}

}

func doCurl(url string, expecttimeout bool) (int, error) {
	time.Sleep(400 * time.Millisecond)
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}
	timeout := time.Duration(3 * time.Second)
	client := &http.Client{Transport: tr, Timeout: timeout}
	response, err := client.Get(url)
	if err != nil {
		if expecttimeout == strings.Contains(err.Error(), "Client.Timeout exceeded") {
			return 502, nil
		}
		return 0, err
	}
	return response.StatusCode, nil
}

func destroyApp(app App) *Session {
	cmd, err := start("deis apps:destroy --app=%s --confirm=%s", app.Name, app.Name)
	Expect(err).NotTo(HaveOccurred())
	Eventually(cmd, defaultMaxTimeout).Should(Exit(0))
	Eventually(cmd).Should(SatisfyAll(
		Say("Destroying %s...", app.Name),
		Say(`done in `)))

	return cmd
}

func deployApp(name string) App {
	app := App{Name: name, URL: strings.Replace(url, "deis", name, 1)}
	cmd, err := start("GIT_SSH=%s git push deis master", gitSSH)
	Expect(err).NotTo(HaveOccurred())
	Eventually(cmd.Err, "5m").Should(Say(`Done, %s:v\d deployed to Deis`, app.Name))
	Eventually(cmd).Should(Exit(0))

	return app
}

// cmdWithRetry runs the provided <cmd> repeatedly, once a second up to the
// supplied <timeout> until the <cmd> result contains the <expectedResult>
// An example use of this utility would be curl-ing a url and waiting
// until the response code matches the expected response
func cmdWithRetry(cmd Cmd, expectedResult string, timeout int) bool {
	var result string
	fmt.Printf("Waiting up to %d seconds for `%s` to return %s...\n", timeout, cmd.CommandLineString, expectedResult)
	for i := 0; i < timeout; i++ {
		sess, err := startCmd(cmd)
		Expect(err).NotTo(HaveOccurred())
		result = string(sess.Wait().Out.Contents())
		if strings.Contains(result, expectedResult) {
			return true
		}
		time.Sleep(1 * time.Second)
	}
	fmt.Printf("FAIL: '%s' does not match expected result of '%s'\n", result, expectedResult)
	return false
}

// gitInit simply invokes 'git init' and verifies the command is successful
func gitInit() {
	cmd, err := start("git init")
	Expect(err).NotTo(HaveOccurred())
	Eventually(cmd).Should(Say("Initialized empty Git repository"))
}

// gitClean destroys the .git directory and verifies the command is successful
func gitClean() {
	cmd, err := start("rm -rf .git")
	Expect(err).NotTo(HaveOccurred())
	Eventually(cmd).Should(Exit(0))
}