actions/actions-runner-controller
1
0
Fork 0
mirror of https://github.com/actions/actions-runner-controller.git synced 2025-12-11 03:57:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Cover ARC upgrade in E2E test (#1592)

Browse Source 
* Cover ARC upgrade in E2E test

so that we can make it extra sure that you can upgrade the existing installation of ARC to the next and also (hopefully) it is backward-compatible, or at least it does not break immediately after upgrading.

* Consolidate E2E tests for RS and RD

* Fix E2E for RD to pass

* Add some comment in E2E for how to release disk consumed after dozens of test runs
This commit is contained in:
Yusuke Kuoka
2022-07-01 21:32:05 +09:00
committed by GitHub
parent d62c8a4697
commit 4446ba57e1
5 changed files with 296 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

312
test/e2e/e2e_test.go
View File

@@ -13,6 +13,13 @@ import (
"sigs.k8s.io/yaml"
)
type DeployKind int
const (
RunnerSets DeployKind = iota
RunnerDeployments
)
var (
controllerImageRepo = "actionsrunnercontrollere2e/actions-runner-controller"
controllerImageTag = "e2e"
@@ -72,8 +79,6 @@ var (
commonScriptEnv = []string{
"SYNC_PERIOD=" + "30s",
"NAME=" + controllerImageRepo,
"VERSION=" + controllerImageTag,
"RUNNER_TAG=" + runnerImageTag,
}
@@ -101,13 +106,27 @@ var (
// whenever the whole test failed, so that you can immediately start fixing issues and rerun inidividual tests.
// See the below link for how terratest handles this:
// https://terratest.gruntwork.io/docs/testing-best-practices/iterating-locally-using-test-stages/
//
// This functions leaves PVs undeleted. To delete PVs, run:
// kubectl get pv -ojson | jq -rMc '.items[] | select(.status.phase == "Available") | {name:.metadata.name, status:.status.phase} | .name' | xargs kubectl delete pv
//
// If you disk full after dozens of test runs, try:
// docker system prune
// and
// kind delete cluster --name teste2e
//
// The former tend to release 200MB-3GB and the latter can result in releasing like 100GB due to kind node contains loaded container images and
// (in case you use it) local provisioners disk image(which is implemented as a directory within the kind node).
func TestE2E(t *testing.T) {
if testing.Short() {
t.Skip("Skipped as -short is set")
}
skipRunnerCleanUp := os.Getenv("ARC_E2E_SKIP_RUNNER_CLEANUP") != ""
retainCluster := os.Getenv("ARC_E2E_RETAIN_CLUSTER") != ""
skipTestIDCleanUp := os.Getenv("ARC_E2E_SKIP_TEST_ID_CLEANUP") != ""
env := initTestEnv(t)
env.useRunnerSet = true
t.Run("build and load images", func(t *testing.T) {
env.buildAndLoadImages(t)
@@ -121,72 +140,107 @@ func TestE2E(t *testing.T) {
return
}
t.Run("install actions-runner-controller and runners", func(t *testing.T) {
env.installActionsRunnerController(t)
t.Run("RunnerSets", func(t *testing.T) {
var (
testID string
)
t.Run("get or generate test ID", func(t *testing.T) {
testID = env.GetOrGenerateTestID(t)
})
if !skipTestIDCleanUp {
t.Cleanup(func() {
env.DeleteTestID(t)
})
}
t.Run("install actions-runner-controller v0.24.1", func(t *testing.T) {
env.installActionsRunnerController(t, "summerwind/actions-runner-controller", "v0.24.1", testID)
})
t.Run("deploy runners", func(t *testing.T) {
env.deploy(t, RunnerSets, testID)
})
if !skipRunnerCleanUp {
t.Cleanup(func() {
env.undeploy(t, RunnerSets, testID)
})
}
t.Run("install edge actions-runner-controller", func(t *testing.T) {
env.installActionsRunnerController(t, controllerImageRepo, controllerImageTag, testID)
})
if t.Failed() {
return
}
t.Run("Install workflow", func(t *testing.T) {
env.installActionsWorkflow(t, RunnerSets, testID)
})
if t.Failed() {
return
}
t.Run("Verify workflow run result", func(t *testing.T) {
env.verifyActionsWorkflowRun(t, testID)
})
})
if t.Failed() {
return
}
t.Run("RunnerDeployments", func(t *testing.T) {
var (
testID string
)
t.Run("Install workflow", func(t *testing.T) {
env.installActionsWorkflow(t)
t.Run("get or generate test ID", func(t *testing.T) {
testID = env.GetOrGenerateTestID(t)
})
if !skipTestIDCleanUp {
t.Cleanup(func() {
env.DeleteTestID(t)
})
}
t.Run("install actions-runner-controller v0.24.1", func(t *testing.T) {
env.installActionsRunnerController(t, "summerwind/actions-runner-controller", "v0.24.1", testID)
})
t.Run("deploy runners", func(t *testing.T) {
env.deploy(t, RunnerDeployments, testID)
})
if !skipRunnerCleanUp {
t.Cleanup(func() {
env.undeploy(t, RunnerDeployments, testID)
})
}
t.Run("install edge actions-runner-controller", func(t *testing.T) {
env.installActionsRunnerController(t, controllerImageRepo, controllerImageTag, testID)
})
if t.Failed() {
return
}
t.Run("Install workflow", func(t *testing.T) {
env.installActionsWorkflow(t, RunnerDeployments, testID)
})
if t.Failed() {
return
}
t.Run("Verify workflow run result", func(t *testing.T) {
env.verifyActionsWorkflowRun(t, testID)
})
})
if t.Failed() {
return
}
t.Run("Verify workflow run result", func(t *testing.T) {
env.verifyActionsWorkflowRun(t)
})
if os.Getenv("ARC_E2E_NO_CLEANUP") != "" {
t.FailNow()
}
}
func TestE2ERunnerDeploy(t *testing.T) {
if testing.Short() {
t.Skip("Skipped as -short is set")
}
env := initTestEnv(t)
env.useApp = true
t.Run("build and load images", func(t *testing.T) {
env.buildAndLoadImages(t)
})
t.Run("install cert-manager", func(t *testing.T) {
env.installCertManager(t)
})
if t.Failed() {
return
}
t.Run("install actions-runner-controller and runners", func(t *testing.T) {
env.installActionsRunnerController(t)
})
if t.Failed() {
return
}
t.Run("Install workflow", func(t *testing.T) {
env.installActionsWorkflow(t)
})
if t.Failed() {
return
}
t.Run("Verify workflow run result", func(t *testing.T) {
env.verifyActionsWorkflowRun(t)
})
if os.Getenv("ARC_E2E_NO_CLEANUP") != "" {
if retainCluster {
t.FailNow()
}
}
@@ -194,23 +248,20 @@ func TestE2ERunnerDeploy(t *testing.T) {
type env struct {
*testing.Env
useRunnerSet bool
// Uses GITHUB_APP_ID, GITHUB_APP_INSTALLATION_ID, and GITHUB_APP_PRIVATE_KEY
// to let ARC authenticate as a GitHub App
useApp bool
testID string
testName string
repoToCommit string
appID, appInstallationID, appPrivateKeyFile string
runnerLabel, githubToken, testRepo, testOrg, testOrgRepo string
githubTokenWebhook string
testEnterprise string
testEphemeral string
scaleDownDelaySecondsAfterScaleOut int64
minReplicas int64
dockerdWithinRunnerContainer bool
testJobs []job
testName string
repoToCommit string
appID, appInstallationID, appPrivateKeyFile string
githubToken, testRepo, testOrg, testOrgRepo string
githubTokenWebhook string
testEnterprise string
testEphemeral string
scaleDownDelaySecondsAfterScaleOut int64
minReplicas int64
dockerdWithinRunnerContainer bool
}
func initTestEnv(t *testing.T) *env {
@@ -220,15 +271,11 @@ func initTestEnv(t *testing.T) *env {
e := &env{Env: testingEnv}
id := e.ID()
testName := t.Name() + " " + id
testName := t.Name()
t.Logf("Initializing test with name %s", testName)
e.testID = id
e.testName = testName
e.runnerLabel = "test-" + id
e.githubToken = testing.Getenv(t, "GITHUB_TOKEN")
e.appID = testing.Getenv(t, "GITHUB_APP_ID")
e.appInstallationID = testing.Getenv(t, "GITHUB_APP_INSTALLATION_ID")
@@ -240,7 +287,6 @@ func initTestEnv(t *testing.T) *env {
e.testOrgRepo = testing.Getenv(t, "TEST_ORG_REPO", "")
e.testEnterprise = testing.Getenv(t, "TEST_ENTERPRISE", "")
e.testEphemeral = testing.Getenv(t, "TEST_EPHEMERAL", "")
e.testJobs = createTestJobs(id, testResultCMNamePrefix, 6)
e.scaleDownDelaySecondsAfterScaleOut, _ = strconv.ParseInt(testing.Getenv(t, "TEST_RUNNER_SCALE_DOWN_DELAY_SECONDS_AFTER_SCALE_OUT", "10"), 10, 32)
e.minReplicas, _ = strconv.ParseInt(testing.Getenv(t, "TEST_RUNNER_MIN_REPLICAS", "1"), 10, 32)
@@ -281,7 +327,7 @@ func (e *env) installCertManager(t *testing.T) {
e.KubectlWaitUntilDeployAvailable(t, "cert-manager", waitCfg.WithTimeout(60*time.Second))
}
func (e *env) installActionsRunnerController(t *testing.T) {
func (e *env) installActionsRunnerController(t *testing.T, repo, tag, testID string) {
t.Helper()
e.createControllerNamespaceAndServiceAccount(t)
@@ -291,25 +337,11 @@ func (e *env) installActionsRunnerController(t *testing.T) {
"ACCEPTANCE_TEST_DEPLOYMENT_TOOL=" + "helm",
}
if e.useRunnerSet {
scriptEnv = append(scriptEnv, "USE_RUNNERSET=1")
} else {
scriptEnv = append(scriptEnv, "USE_RUNNERSET=false")
}
varEnv := []string{
"TEST_ENTERPRISE=" + e.testEnterprise,
"TEST_REPO=" + e.testRepo,
"TEST_ORG=" + e.testOrg,
"TEST_ORG_REPO=" + e.testOrgRepo,
"WEBHOOK_GITHUB_TOKEN=" + e.githubTokenWebhook,
"RUNNER_LABEL=" + e.runnerLabel,
"TEST_ID=" + e.testID,
"TEST_EPHEMERAL=" + e.testEphemeral,
fmt.Sprintf("RUNNER_SCALE_DOWN_DELAY_SECONDS_AFTER_SCALE_OUT=%d", e.scaleDownDelaySecondsAfterScaleOut),
fmt.Sprintf("REPO_RUNNER_MIN_REPLICAS=%d", e.minReplicas),
fmt.Sprintf("ORG_RUNNER_MIN_REPLICAS=%d", e.minReplicas),
fmt.Sprintf("ENTERPRISE_RUNNER_MIN_REPLICAS=%d", e.minReplicas),
"TEST_ID=" + testID,
"NAME=" + repo,
"VERSION=" + tag,
}
if e.useApp {
@@ -326,6 +358,54 @@ func (e *env) installActionsRunnerController(t *testing.T) {
)
}
scriptEnv = append(scriptEnv, varEnv...)
scriptEnv = append(scriptEnv, commonScriptEnv...)
e.RunScript(t, "../../acceptance/deploy.sh", testing.ScriptConfig{Dir: "../..", Env: scriptEnv})
}
func (e *env) deploy(t *testing.T, kind DeployKind, testID string) {
t.Helper()
e.do(t, "apply", kind, testID)
}
func (e *env) undeploy(t *testing.T, kind DeployKind, testID string) {
t.Helper()
e.do(t, "delete", kind, testID)
}
func (e *env) do(t *testing.T, op string, kind DeployKind, testID string) {
t.Helper()
e.createControllerNamespaceAndServiceAccount(t)
scriptEnv := []string{
"KUBECONFIG=" + e.Kubeconfig(),
"OP=" + op,
}
switch kind {
case RunnerSets:
scriptEnv = append(scriptEnv, "USE_RUNNERSET=1")
case RunnerDeployments:
scriptEnv = append(scriptEnv, "USE_RUNNERSET=false")
default:
t.Fatalf("Invalid deploy kind %v", kind)
}
varEnv := []string{
"TEST_ENTERPRISE=" + e.testEnterprise,
"TEST_REPO=" + e.testRepo,
"TEST_ORG=" + e.testOrg,
"TEST_ORG_REPO=" + e.testOrgRepo,
"RUNNER_LABEL=" + e.runnerLabel(testID),
"TEST_EPHEMERAL=" + e.testEphemeral,
fmt.Sprintf("RUNNER_SCALE_DOWN_DELAY_SECONDS_AFTER_SCALE_OUT=%d", e.scaleDownDelaySecondsAfterScaleOut),
fmt.Sprintf("REPO_RUNNER_MIN_REPLICAS=%d", e.minReplicas),
fmt.Sprintf("ORG_RUNNER_MIN_REPLICAS=%d", e.minReplicas),
fmt.Sprintf("ENTERPRISE_RUNNER_MIN_REPLICAS=%d", e.minReplicas),
}
if e.dockerdWithinRunnerContainer {
varEnv = append(varEnv,
"RUNNER_DOCKERD_WITHIN_RUNNER_CONTAINER=true",
@@ -341,7 +421,11 @@ func (e *env) installActionsRunnerController(t *testing.T) {
scriptEnv = append(scriptEnv, varEnv...)
scriptEnv = append(scriptEnv, commonScriptEnv...)
e.RunScript(t, "../../acceptance/deploy.sh", testing.ScriptConfig{Dir: "../..", Env: scriptEnv})
e.RunScript(t, "../../acceptance/deploy_runners.sh", testing.ScriptConfig{Dir: "../..", Env: scriptEnv})
}
func (e *env) runnerLabel(testID string) string {
return "test-" + testID
}
func (e *env) createControllerNamespaceAndServiceAccount(t *testing.T) {
@@ -351,16 +435,20 @@ func (e *env) createControllerNamespaceAndServiceAccount(t *testing.T) {
e.KubectlEnsureClusterRoleBindingServiceAccount(t, "default-admin", "cluster-admin", "default:default", testing.KubectlConfig{})
}
func (e *env) installActionsWorkflow(t *testing.T) {
func (e *env) installActionsWorkflow(t *testing.T, kind DeployKind, testID string) {
t.Helper()
installActionsWorkflow(t, e.testName, e.runnerLabel, testResultCMNamePrefix, e.repoToCommit, e.testJobs)
installActionsWorkflow(t, e.testName+" "+testID, e.runnerLabel(testID), testResultCMNamePrefix, e.repoToCommit, kind, e.testJobs(testID))
}
func (e *env) verifyActionsWorkflowRun(t *testing.T) {
func (e *env) testJobs(testID string) []job {
return createTestJobs(testID, testResultCMNamePrefix, 6)
}
func (e *env) verifyActionsWorkflowRun(t *testing.T, testID string) {
t.Helper()
verifyActionsWorkflowRun(t, e.Env, e.testJobs)
verifyActionsWorkflowRun(t, e.Env, e.testJobs(testID))
}
type job struct {
@@ -383,7 +471,7 @@ func createTestJobs(id, testResultCMNamePrefix string, numJobs int) []job {
const Branch = "main"
func installActionsWorkflow(t *testing.T, testName, runnerLabel, testResultCMNamePrefix, testRepo string, testJobs []job) {
func installActionsWorkflow(t *testing.T, testName, runnerLabel, testResultCMNamePrefix, testRepo string, kind DeployKind, testJobs []job) {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
@@ -415,6 +503,14 @@ func installActionsWorkflow(t *testing.T, testName, runnerLabel, testResultCMNam
}
if !kubernetesContainerMode {
if kind == RunnerDeployments {
steps = append(steps,
testing.Step{
Run: "sudo mkdir -p \"${RUNNER_TOOL_CACHE}\" \"${HOME}/.cache\" \"/var/lib/docker\"",
},
)
}
steps = append(steps,
testing.Step{
// This might be the easiest way to handle permissions without use of securityContext
@@ -579,5 +675,5 @@ func verifyActionsWorkflowRun(t *testing.T, env *testing.Env, testJobs []job) {
}
return results, err
}, 3*60*time.Second, 10*time.Second).Should(gomega.Equal(expected))
}, 8*60*time.Second, 30*time.Second).Should(gomega.Equal(expected))
}