Make webhook-based scale race-free (#1477)

* Make webhook-based scale operation asynchronous

This prevents race condition in the webhook-based autoscaler when it received another webhook event while processing another webhook event and both ended up
scaling up the same horizontal runner autoscaler.

Ref #1321

* Fix typos

* Update rather than Patch HRA to avoid race among webhook-based autoscaler servers

* Batch capacity reservation updates for efficient use of apiserver

* Fix potential never-ending HRA update conflicts in batch update

* Extract batchScaler out of webhook-based autoscaler for testability

* Fix log levels and batch scaler hang on start

* Correlate webhook event with scale trigger amount in logs

* Fix log message

controllers/horizontal_runner_autoscaler_webhook_worker_test.go

@@ -0,0 +1,36 @@
+package controllers
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestWorker_Add(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	w := newWorker(ctx, 2, func(st *ScaleTarget) {})
+	require.True(t, w.Add(&ScaleTarget{}))
+	require.True(t, w.Add(&ScaleTarget{}))
+	require.False(t, w.Add(&ScaleTarget{}))
+}
+
+func TestWorker_Work(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	var count int
+
+	w := newWorker(ctx, 1, func(st *ScaleTarget) {
+		count++
+		cancel()
+	})
+	require.True(t, w.Add(&ScaleTarget{}))
+	require.False(t, w.Add(&ScaleTarget{}))
+
+	<-w.Done()
+
+	require.Equal(t, count, 1)
+}