Expose workflow job metrics via new actions-metrics-server (#2057)

* Add workflow job metrics to Github webhook server

* Fix handling of workflow_job.Conclusion

* Make the prometheus metrics exporter for the workflow jobs a dedicated application

* chart: Add support for deploying actions-metrics-server

* A few improvements to make it easy to cover in E2E

* chart: Add missing actionsmetrics.service.yaml

* chart: Do not modify actionsMetricsServer.replicaCount

* chart: Add documentation for actionsMetrics and actionsMetricsServer

Co-authored-by: Colin Heathman <cheathman@benchsci.com>

acceptance/argotunnel.sh

@@ -88,6 +88,9 @@ data:
     no-autoupdate: true
     ingress:
     # The first rule proxies traffic to the httpbin sample Service defined in app.yaml
+    - hostname: ${TUNNEL_HOSTNAME}
+      service: http://actions-runner-controller-actions-metrics-server.actions-runner-system:80
+      path: /metrics$
     - hostname: ${TUNNEL_HOSTNAME}
       service: http://actions-runner-controller-github-webhook-server.actions-runner-system:80
     # This rule matches any traffic which didn't match a previous rule, and responds with HTTP 404.

acceptance/deploy.sh

@@ -35,6 +35,16 @@ else
   echo 'Skipped deploying secret "github-webhook-server". Set WEBHOOK_GITHUB_TOKEN to deploy.' 1>&2
 fi
 
+if [ -n "${WEBHOOK_GITHUB_TOKEN}" ]; then
+  kubectl -n actions-runner-system delete secret \
+      actions-metrics-server || :
+  kubectl -n actions-runner-system create secret generic \
+      actions-metrics-server \
+      --from-literal=github_token=${WEBHOOK_GITHUB_TOKEN:?WEBHOOK_GITHUB_TOKEN must not be empty}
+else
+  echo 'Skipped deploying secret "actions-metrics-server". Set WEBHOOK_GITHUB_TOKEN to deploy.' 1>&2
+fi
+
 tool=${ACCEPTANCE_TEST_DEPLOYMENT_TOOL}
 
 TEST_ID=${TEST_ID:-default}
@@ -49,6 +59,7 @@ if [ "${tool}" == "helm" ]; then
     flags+=( --set imagePullSecrets[0].name=${IMAGE_PULL_SECRET})
     flags+=( --set image.actionsRunnerImagePullSecrets[0].name=${IMAGE_PULL_SECRET})
     flags+=( --set githubWebhookServer.imagePullSecrets[0].name=${IMAGE_PULL_SECRET})
+    flags+=( --set actionsMetricsServer.imagePullSecrets[0].name=${IMAGE_PULL_SECRET})
   fi
   if [ "${CHART_VERSION}" != "" ]; then
     flags+=( --version ${CHART_VERSION})
@@ -56,6 +67,7 @@ if [ "${tool}" == "helm" ]; then
   if [ "${LOG_FORMAT}" != "" ]; then
     flags+=( --set logFormat=${LOG_FORMAT})
     flags+=( --set githubWebhookServer.logFormat=${LOG_FORMAT})
+    flags+=( --set actionsMetricsServer.logFormat=${LOG_FORMAT})
   fi
 
   set -vx
@@ -70,6 +82,7 @@ if [ "${tool}" == "helm" ]; then
     --set image.tag=${VERSION} \
     --set podAnnotations.test-id=${TEST_ID} \
     --set githubWebhookServer.podAnnotations.test-id=${TEST_ID} \
+    --set actionsMetricsServer.podAnnotations.test-id=${TEST_ID} \
     ${flags[@]} --set image.imagePullPolicy=${IMAGE_PULL_POLICY} \
     -f ${VALUES_FILE}
   set +v

acceptance/values.yaml

@@ -33,3 +33,23 @@ githubWebhookServer:
         protocol: TCP
         name: http
         nodePort: 31000
+actionsMetricsServer:
+  imagePullSecrets: []
+  logLevel: "-4"
+  enabled: true
+  labels: {}
+  replicaCount: 1
+  secret:
+    enabled: true
+    # create: true
+    name: "actions-metrics-server"
+    ### GitHub Webhook Configuration
+    #github_webhook_secret_token: ""
+  service:
+    type: NodePort
+    ports:
+      - port: 80
+        targetPort: http
+        protocol: TCP
+        name: http
+        nodePort: 31001