Go runtime auto-tuning GOMEMLIMIT/GOMAXPROCS for application-controller

argocd-operator/api/v1beta1/argocd_types.go

@@ -136,12 +136,68 @@ type ArgoCDApplicationControllerSpec struct {
 
 	// Metrics defines the metrics configuration for the Application Controller ServiceMonitor.
 	Metrics *ArgoCDMetricsSpec `json:"metrics,omitempty"`
+
+	// GoRuntime configures automatic Go runtime tuning for the Application Controller.
+	// Tuning is enabled cluster-wide by default; set ENABLE_APPLICATION_CONTROLLER_GO_RUNTIME_TUNING
+	// (component-specific, takes precedence) or ENABLE_GO_RUNTIME_TUNING (global fallback) to "false"
+	// on the operator to disable it. When both are set, the component-specific setting wins, so a
+	// global "false" does not disable tuning if the component-specific setting is "true".
+	GoRuntime *ArgoCDGoRuntimeSpec `json:"goRuntime,omitempty"`
 }
 
 func (a *ArgoCDApplicationControllerSpec) IsEnabled() bool {
 	return a.Enabled == nil || (a.Enabled != nil && *a.Enabled)
 }
 
+// ArgoCDGoRuntimeSpec defines per-instance options for automatic Go runtime tuning (GOMEMLIMIT, GOMAXPROCS).
+// Tuning is enabled cluster-wide by default; set ENABLE_APPLICATION_CONTROLLER_GO_RUNTIME_TUNING
+// (component-specific, takes precedence) or ENABLE_GO_RUNTIME_TUNING (global fallback) to "false"
+// on the operator to disable. This spec allows per-instance opt-out
+// and customization of memory and CPU percentages. Currently used only in ArgoCDApplicationControllerSpec.
+type ArgoCDGoRuntimeSpec struct {
+	// Enabled toggles automatic Go runtime tuning for this instance. Defaults to true.
+	// Cannot override operator-level disable (ENABLE_APPLICATION_CONTROLLER_GO_RUNTIME_TUNING=false,
+	// or ENABLE_GO_RUNTIME_TUNING=false when the component-specific setting is unset).
+	//+operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Enable Go Runtime Auto-Tuning",xDescriptors={"urn:alm:descriptor:com.tectonic.ui:fieldGroup:Controller","urn:alm:descriptor:com.tectonic.ui:booleanSwitch"}
+	Enabled *bool `json:"enabled,omitempty"`
+
+	// MemoryLimitPercent is the percentage (1-100) of the memory limit used for GOMEMLIMIT. Defaults to 90.
+	// Represents headroom reserved for non-heap memory and GC pauses.
+	//+kubebuilder:validation:Minimum=1
+	//+kubebuilder:validation:Maximum=100
+	//+operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Memory Limit Percent",xDescriptors={"urn:alm:descriptor:com.tectonic.ui:fieldGroup:Controller","urn:alm:descriptor:com.tectonic.ui:number"}
+	MemoryLimitPercent *int32 `json:"memoryLimitPercent,omitempty"`
+
+	// CPULimitPercent is the percentage (1-100) of the CPU limit used for GOMAXPROCS. Defaults to 100.
+	// Reserves CPU headroom for the kernel. Result is floored to nearest core, minimum 1.
+	//+kubebuilder:validation:Minimum=1
+	//+kubebuilder:validation:Maximum=100
+	//+operator-sdk:csv:customresourcedefinitions:type=spec,displayName="CPU Limit Percent",xDescriptors={"urn:alm:descriptor:com.tectonic.ui:fieldGroup:Controller","urn:alm:descriptor:com.tectonic.ui:number"}
+	CPULimitPercent *int32 `json:"cpuLimitPercent,omitempty"`
+}
+
+// IsEnabled returns whether Go runtime tuning is enabled for this ArgoCDGoRuntimeSpec.
+// A nil receiver (i.e. .Spec.Controller.GoRuntime is unset) or a nil Enabled field defaults to true.
+func (g *ArgoCDGoRuntimeSpec) IsEnabled() bool {
+	return g == nil || g.Enabled == nil || *g.Enabled
+}
+
+// GetMemoryLimitPercent returns the configured MemoryLimitPercent, or defaultPercent if unset.
+func (g *ArgoCDGoRuntimeSpec) GetMemoryLimitPercent(defaultPercent int32) int32 {
+	if g == nil || g.MemoryLimitPercent == nil {
+		return defaultPercent
+	}
+	return *g.MemoryLimitPercent
+}
+
+// GetCPULimitPercent returns the configured CPULimitPercent, or defaultPercent if unset.
+func (g *ArgoCDGoRuntimeSpec) GetCPULimitPercent(defaultPercent int32) int32 {
+	if g == nil || g.CPULimitPercent == nil {
+		return defaultPercent
+	}
+	return *g.CPULimitPercent
+}
+
 // ArgoCDApplicationControllerShardSpec defines the options available for enabling sharding for the Application Controller component.
 type ArgoCDApplicationControllerShardSpec struct {
 	// Enabled defines whether sharding should be enabled on the Application Controller component.

argocd-operator/common/defaults.go

@@ -27,6 +27,12 @@ const (
 	// ArgoCDApplicationControllerDefaultShardReplicas is the default number of replicas that the ArgoCD Application Controller Should Use
 	ArgocdApplicationControllerDefaultReplicas = 1
 
+	// ArgoCDDefaultControllerMemLimitPercent is the default percentage of the controller's memory limit used to derive GOMEMLIMIT when Go runtime tuning is enabled.
+	ArgoCDDefaultControllerMemLimitPercent = 90
+
+	// ArgoCDDefaultControllerCPULimitPercent is the default percentage of the controller's CPU limit used to derive GOMAXPROCS when Go runtime tuning is enabled.
+	ArgoCDDefaultControllerCPULimitPercent = 100
+
 	// ArgoCDDefaultLogLevel is the default log level to be used by all ArgoCD components.
 	ArgoCDDefaultLogLevel = "info"
 

argocd-operator/common/keys.go

@@ -294,6 +294,16 @@ const (
 	// the Namespace Management feature is enabled.
 	EnableManagedNamespace = "ALLOW_NAMESPACE_MANAGEMENT_IN_NAMESPACE_SCOPED_INSTANCES"
 
+	// EnableApplicationControllerGoRuntimeTuning enables automatic Go runtime tuning (GOMEMLIMIT, GOMAXPROCS)
+	// for the application controller. Enabled by default; set to "false" to disable cluster-wide.
+	EnableApplicationControllerGoRuntimeTuning = "ENABLE_APPLICATION_CONTROLLER_GO_RUNTIME_TUNING"
+
+	// EnableGoRuntimeTuning is a global, component-agnostic alias/fallback for EnableApplicationControllerGoRuntimeTuning.
+	// It is intended to gate Go runtime tuning across all GitOps components (not just the application controller) as the
+	// feature expands to other workloads (e.g. repo-server, server). If EnableApplicationControllerGoRuntimeTuning is unset
+	// on the operator, this variable is consulted as a cluster-wide default; if both are unset, tuning defaults to enabled.
+	EnableGoRuntimeTuning = "ENABLE_GO_RUNTIME_TUNING"
+
 	// ArgoCDImageUpdaterImageEnvName is the environment variable used to get the image
 	// to used for the Image Updater container.
 	ArgoCDImageUpdaterImageEnvName = "ARGOCD_IMAGE_UPDATER_IMAGE"

argocd-operator/controllers/argocd/statefulset.go

@@ -683,6 +683,9 @@ func (r *ReconcileArgoCD) reconcileApplicationControllerStatefulSet(cr *argoproj
 	// Let user specify their own environment first
 	controllerEnv = argoutil.EnvMerge(controllerEnv, proxyEnvVars(), false)
 	controllerEnv = argoutil.EnvMerge(controllerEnv, argoutil.GetRedisAuthEnv(), false)
+	// Go runtime tuning (GOMEMLIMIT, GOMAXPROCS) is derived from resource limits,
+	// but .Spec.Controller.Env always takes precedence.
+	controllerEnv = argoutil.EnvMerge(controllerEnv, getArgoApplicationControllerGoRuntimeEnv(cr), false)
 
 	if cr.Spec.Controller.InitContainers != nil {
 		ss.Spec.Template.Spec.InitContainers = append(ss.Spec.Template.Spec.InitContainers, cr.Spec.Controller.InitContainers...)

argocd-operator/controllers/argocd/statefulset_test.go

@@ -3,6 +3,7 @@ package argocd
 import (
 	"context"
 	"fmt"
+	"os"
 	"reflect"
 	"testing"
 	"time"
@@ -941,6 +942,138 @@ func TestReconcileAppController_Initcontainer(t *testing.T) {
 	assert.Equal(t, 0, len(ss.Spec.Template.Spec.InitContainers))
 }
 
+func TestReconcileAppController_GoRuntimeEnv(t *testing.T) {
+	a := makeTestArgoCD(func(a *argoproj.ArgoCD) {
+		a.Spec.Controller.Resources = &corev1.ResourceRequirements{
+			Limits: corev1.ResourceList{
+				corev1.ResourceMemory: resourcev1.MustParse("2000Mi"),
+				corev1.ResourceCPU:    resourcev1.MustParse("2000m"),
+			},
+		}
+	})
+
+	resObjs := []client.Object{a}
+	subresObjs := []client.Object{a}
+	runtimeObjs := []runtime.Object{}
+	sch := makeTestReconcilerScheme(argoproj.AddToScheme, promoter.AddToScheme)
+	cl := makeTestReconcilerClient(sch, resObjs, subresObjs, runtimeObjs)
+	r := makeTestReconciler(cl, sch, testclient.NewSimpleClientset())
+
+	getControllerEnv := func() []corev1.EnvVar {
+		ss := &appsv1.StatefulSet{}
+		assert.NoError(t, r.Get(
+			context.TODO(),
+			types.NamespacedName{
+				Name:      applicationControllerResourceName(a),
+				Namespace: a.Namespace,
+			},
+			ss))
+		return ss.Spec.Template.Spec.Containers[0].Env
+	}
+
+	// Go runtime tuning is enabled by default: GOMEMLIMIT and GOMAXPROCS should both be derived from the
+	// configured memory/CPU limits.
+	assert.NoError(t, r.reconcileApplicationControllerStatefulSet(a, false))
+	assert.Contains(t, getControllerEnv(), corev1.EnvVar{Name: "GOMEMLIMIT", Value: "1887436800"}) // 2000Mi * 90 / 100, in bytes
+	assert.Contains(t, getControllerEnv(), corev1.EnvVar{Name: "GOMAXPROCS", Value: "2"})          // 2000m * 100 / 100000 = 2 cores
+
+	// A user-specified value in .Spec.Controller.Env should always win over the derived value, for both vars.
+	a.Spec.Controller.Env = []corev1.EnvVar{
+		{Name: "GOMEMLIMIT", Value: "999999999"},
+		{Name: "GOMAXPROCS", Value: "1"},
+	}
+	assert.NoError(t, r.reconcileApplicationControllerStatefulSet(a, false))
+	assert.Contains(t, getControllerEnv(), corev1.EnvVar{Name: "GOMEMLIMIT", Value: "999999999"})
+	assert.Contains(t, getControllerEnv(), corev1.EnvVar{Name: "GOMAXPROCS", Value: "1"})
+
+	// Disabling the feature at the operator level should remove both derived vars.
+	a.Spec.Controller.Env = nil
+	os.Setenv(common.EnableApplicationControllerGoRuntimeTuning, "false")
+
+	assert.NoError(t, r.reconcileApplicationControllerStatefulSet(a, false))
+	for _, e := range getControllerEnv() {
+		assert.NotEqual(t, "GOMEMLIMIT", e.Name, "GOMEMLIMIT should not be set when the feature is disabled at the operator level")
+		assert.NotEqual(t, "GOMAXPROCS", e.Name, "GOMAXPROCS should not be set when the feature is disabled at the operator level")
+	}
+
+	// A per-instance .Spec.Controller.GoRuntime.Enabled=true cannot override the operator-level kill switch.
+	trueVal := true
+	a.Spec.Controller.GoRuntime = &argoproj.ArgoCDGoRuntimeSpec{Enabled: &trueVal}
+	assert.NoError(t, r.reconcileApplicationControllerStatefulSet(a, false))
+	for _, e := range getControllerEnv() {
+		assert.NotEqual(t, "GOMEMLIMIT", e.Name, "GOMEMLIMIT should not be set when the operator-level kill switch is on, regardless of the per-instance field")
+		assert.NotEqual(t, "GOMAXPROCS", e.Name, "GOMAXPROCS should not be set when the operator-level kill switch is on, regardless of the per-instance field")
+	}
+
+	os.Unsetenv(common.EnableApplicationControllerGoRuntimeTuning)
+
+	// With the operator-level kill switch removed again, the per-instance Enabled=true field is a no-op (same as default-on).
+	assert.NoError(t, r.reconcileApplicationControllerStatefulSet(a, false))
+	assert.Contains(t, getControllerEnv(), corev1.EnvVar{Name: "GOMEMLIMIT", Value: "1887436800"})
+	assert.Contains(t, getControllerEnv(), corev1.EnvVar{Name: "GOMAXPROCS", Value: "2"})
+
+	// A per-instance .Spec.Controller.GoRuntime.Enabled=false opt-out should remove both derived vars.
+	falseVal := false
+	a.Spec.Controller.GoRuntime = &argoproj.ArgoCDGoRuntimeSpec{Enabled: &falseVal}
+	assert.NoError(t, r.reconcileApplicationControllerStatefulSet(a, false))
+	for _, e := range getControllerEnv() {
+		assert.NotEqual(t, "GOMEMLIMIT", e.Name, "GOMEMLIMIT should not be set when the per-instance GoRuntime.Enabled field is false")
+		assert.NotEqual(t, "GOMAXPROCS", e.Name, "GOMAXPROCS should not be set when the per-instance GoRuntime.Enabled field is false")
+	}
+
+	// A per-instance .Spec.Controller.GoRuntime.MemoryLimitPercent should override the default 90% headroom,
+	// and should not affect GOMAXPROCS (memory and CPU tuning are independent).
+	var fiftyPercent int32 = 50
+	a.Spec.Controller.GoRuntime = &argoproj.ArgoCDGoRuntimeSpec{MemoryLimitPercent: &fiftyPercent}
+	assert.NoError(t, r.reconcileApplicationControllerStatefulSet(a, false))
+	assert.Contains(t, getControllerEnv(), corev1.EnvVar{Name: "GOMEMLIMIT", Value: "1048576000"}) // 2000Mi * 50 / 100, in bytes
+	assert.Contains(t, getControllerEnv(), corev1.EnvVar{Name: "GOMAXPROCS", Value: "2"})          // unaffected, still default 100% of 2000m
+
+	// A per-instance .Spec.Controller.GoRuntime.CPULimitPercent should override the default 100% CPU headroom,
+	// and should not affect GOMEMLIMIT (memory and CPU tuning are independent). 2000m * 20 / 100000 = 0.4, floors to 1.
+	var twentyPercent int32 = 20
+	a.Spec.Controller.GoRuntime = &argoproj.ArgoCDGoRuntimeSpec{CPULimitPercent: &twentyPercent}
+	assert.NoError(t, r.reconcileApplicationControllerStatefulSet(a, false))
+	assert.Contains(t, getControllerEnv(), corev1.EnvVar{Name: "GOMAXPROCS", Value: "1"})
+	assert.Contains(t, getControllerEnv(), corev1.EnvVar{Name: "GOMEMLIMIT", Value: "1887436800"}) // unaffected, back to default 90% of 2000Mi
+
+	a.Spec.Controller.GoRuntime = nil
+}
+
+// TestReconcileAppController_GoRuntimeEnv_GOMAXPROCSFloor verifies that GOMAXPROCS is floored at 1 (never 0)
+// for sub-1-core CPU limits, through the full reconcileApplicationControllerStatefulSet path (not just the
+// getArgoApplicationControllerGoRuntimeEnv unit level).
+func TestReconcileAppController_GoRuntimeEnv_GOMAXPROCSFloor(t *testing.T) {
+	a := makeTestArgoCD(func(a *argoproj.ArgoCD) {
+		a.Spec.Controller.Resources = &corev1.ResourceRequirements{
+			Limits: corev1.ResourceList{
+				corev1.ResourceCPU: resourcev1.MustParse("250m"), // 0.25 cores
+			},
+		}
+	})
+
+	resObjs := []client.Object{a}
+	subresObjs := []client.Object{a}
+	runtimeObjs := []runtime.Object{}
+	sch := makeTestReconcilerScheme(argoproj.AddToScheme, promoter.AddToScheme)
+	cl := makeTestReconcilerClient(sch, resObjs, subresObjs, runtimeObjs)
+	r := makeTestReconciler(cl, sch, testclient.NewSimpleClientset())
+
+	assert.NoError(t, r.reconcileApplicationControllerStatefulSet(a, false))
+
+	ss := &appsv1.StatefulSet{}
+	assert.NoError(t, r.Get(
+		context.TODO(),
+		types.NamespacedName{
+			Name:      applicationControllerResourceName(a),
+			Namespace: a.Namespace,
+		},
+		ss))
+
+	// 250m * 100 / 100000 = 0.25, which floors to 1, never 0 (Go runtime panics on GOMAXPROCS=0).
+	assert.Contains(t, ss.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{Name: "GOMAXPROCS", Value: "1"})
+}
+
 func TestReconcileArgoCD_sidecarcontainer(t *testing.T) {
 	a := makeTestArgoCD(func(a *argoproj.ArgoCD) {
 		a.Spec.Controller.SidecarContainers = []corev1.Container{

argocd-operator/controllers/argocd/util.go

@@ -172,6 +172,71 @@ func getArgoApplicationControllerResources(cr *argoproj.ArgoCD) corev1.ResourceR
 	return resources
 }
 
+// isApplicationControllerGoRuntimeTuningEnabledOnOperator reports whether operator-level Go runtime
+// tuning is enabled. ENABLE_APPLICATION_CONTROLLER_GO_RUNTIME_TUNING takes precedence over
+// ENABLE_GO_RUNTIME_TUNING; both default to true.
+func isApplicationControllerGoRuntimeTuningEnabledOnOperator() bool {
+	if v, ok := os.LookupEnv(common.EnableApplicationControllerGoRuntimeTuning); ok {
+		return v != "false"
+	}
+	if v, ok := os.LookupEnv(common.EnableGoRuntimeTuning); ok {
+		return v != "false"
+	}
+	return true
+}
+
+// getArgoApplicationControllerGoRuntimeEnv returns GOMEMLIMIT and GOMAXPROCS for the application
+// controller, derived as a percentage of its memory and CPU limits respectively.
+func getArgoApplicationControllerGoRuntimeEnv(cr *argoproj.ArgoCD) []corev1.EnvVar {
+	env := make([]corev1.EnvVar, 0)
+
+	// Default-on: only skip Go runtime tuning if it has been explicitly disabled on the operator, via either
+	// the component-specific gate or the global alias (component-specific takes precedence if both are set).
+	if !isApplicationControllerGoRuntimeTuningEnabledOnOperator() {
+		return env
+	}
+
+	// Per-instance opt-out. Note that GoRuntime.IsEnabled() defaults to true when GoRuntime (or GoRuntime.Enabled)
+	// is unset, so this only short-circuits when the CR author has explicitly set enabled: false.
+	if !cr.Spec.Controller.GoRuntime.IsEnabled() {
+		return env
+	}
+
+	resources := getArgoApplicationControllerResources(cr)
+
+	// Compute GOMEMLIMIT from memory limit if configured
+	memLimit := resources.Limits.Memory()
+	if memLimit != nil && !memLimit.IsZero() {
+		memLimitPercent := cr.Spec.Controller.GoRuntime.GetMemoryLimitPercent(common.ArgoCDDefaultControllerMemLimitPercent)
+		goMemLimitBytes := memLimit.Value() * int64(memLimitPercent) / 100
+		if goMemLimitBytes > 0 {
+			env = append(env, corev1.EnvVar{
+				Name:  "GOMEMLIMIT",
+				Value: fmt.Sprintf("%d", goMemLimitBytes),
+			})
+		}
+	}
+
+	// Compute GOMAXPROCS from CPU limit if configured
+	cpuLimit := resources.Limits.Cpu()
+	if cpuLimit != nil && !cpuLimit.IsZero() {
+		cpuLimitPercent := cr.Spec.Controller.GoRuntime.GetCPULimitPercent(common.ArgoCDDefaultControllerCPULimitPercent)
+		// GOMAXPROCS = floor(millicores * percent / 100000), i.e. a truncated percentage of the
+		// CPU limit in whole cores. Truncation (like GOMEMLIMIT above) never allows more
+		// concurrency than the percentage permits. Floored to 1 because GOMAXPROCS=0 is invalid.
+		gomaxprocsValue := cpuLimit.MilliValue() * int64(cpuLimitPercent) / 100000
+		if gomaxprocsValue < 1 {
+			gomaxprocsValue = 1
+		}
+		env = append(env, corev1.EnvVar{
+			Name:  "GOMAXPROCS",
+			Value: fmt.Sprintf("%d", gomaxprocsValue),
+		})
+	}
+
+	return env
+}
+
 // getArgoApplicationControllerCommand will return the command for the ArgoCD Application Controller component.
 func getArgoApplicationControllerCommand(cr *argoproj.ArgoCD, useTLSForRedis bool) []string {
 	allowed := argoutil.IsNamespaceClusterConfigNamespace(cr.Namespace)