Commit e83b057c authored by Miciah Masters's avatar Miciah Masters Committed by Miciah Dashiel Butler Masters
Browse files

Configure router to use Power of Two Random Choices

Configure router deployments to use HAProxy's "random" load balancing
algorithm, which with its default parameters implements the "Power of Two
Random Choices" balancing algorithm.  Allow the user to revert to the
previous "Least Connections" balancing algorithm using an unsupported
config override.

See https://www.haproxy.com/blog/power-of-two-load-balancing/ for a
description and analysis of Power of Two Random Choices.

This commit resolves NE-551.

https://issues.redhat.com/browse/NE-551

* pkg/operator/controller/ingress/deployment.go
(RouterLoadBalancingAlgorithmEnvName): New const.
(desiredRouterDeployment): Check spec.unsupportedConfigOverrides, and set
ROUTER_LOAD_BALANCE_ALGORITHM to "leastconn" if the
{"loadBalancingAlgorithm":"leastconn"} override is set.
* pkg/operator/controller/ingress/deployment_test.go
(TestDesiredRouterDeployment): Verify that desiredRouterDeployment sets the
ROUTER_LOAD_BALANCE_ALGORITHM environment variable as expected and
correctly handles spec.unsupportedConfigOverrides.
* test/e2e/operator_test.go
(availableConditionsForPrivateIngressController): New variable.  Define the
expected status conditions for an ingresscontroller that specifies the
"Private" endpoint publishing strategy type and is available.
(TestTLSSecurityProfile, TestRouteAdmissionPolicy, TestSyslogLogging)
(TestContainerLogging, TestUniqueIdHeader): Use
availableConditionsForPrivateIngressController.
(TestLoadBalancingAlgorithmUnsupportedConfigOverride): New test.  Verify
that the operator configures the router with the "random"
load-balancing algorithm by default and allows the algorithm to be changed
to "leastconn" algorithm using an unsupported config override.
parent 4090f437
......@@ -2,6 +2,7 @@ package ingress
import (
"context"
"encoding/json"
"fmt"
"hash"
"hash/fnv"
......@@ -56,6 +57,8 @@ const (
RouterHeaderBufferSize = "ROUTER_BUF_SIZE"
RouterHeaderBufferMaxRewriteSize = "ROUTER_MAX_REWRITE_SIZE"
RouterLoadBalancingAlgorithmEnvName = "ROUTER_LOAD_BALANCE_ALGORITHM"
RouterDisableHTTP2EnvName = "ROUTER_DISABLE_HTTP2"
RouterDefaultEnableHTTP2Annotation = "ingress.operator.openshift.io/default-enable-http2"
......@@ -361,6 +364,24 @@ func desiredRouterDeployment(ci *operatorv1.IngressController, ingressController
env = append(env, corev1.EnvVar{Name: "ROUTER_METRICS_TLS_CERT_FILE", Value: filepath.Join(certsVolumeMountPath, "tls.crt")})
env = append(env, corev1.EnvVar{Name: "ROUTER_METRICS_TLS_KEY_FILE", Value: filepath.Join(certsVolumeMountPath, "tls.key")})
var unsupportedConfigOverrides struct {
LoadBalancingAlgorithm string `json:"loadBalancingAlgorithm"`
}
if len(ci.Spec.UnsupportedConfigOverrides.Raw) > 0 {
if err := json.Unmarshal(ci.Spec.UnsupportedConfigOverrides.Raw, &unsupportedConfigOverrides); err != nil {
return nil, fmt.Errorf("ingresscontroller %q has invalid spec.unsupportedConfigOverrides: %w", ci.Name, err)
}
}
loadBalancingAlgorithm := "random"
switch unsupportedConfigOverrides.LoadBalancingAlgorithm {
case "leastconn":
loadBalancingAlgorithm = "leastconn"
}
env = append(env, corev1.EnvVar{
Name: RouterLoadBalancingAlgorithmEnvName,
Value: loadBalancingAlgorithm,
})
if len(ci.Status.Domain) > 0 {
env = append(env, corev1.EnvVar{Name: "ROUTER_CANONICAL_HOSTNAME", Value: ci.Status.Domain})
}
......
......@@ -14,6 +14,7 @@ import (
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/intstr"
)
......@@ -211,6 +212,8 @@ func TestDesiredRouterDeployment(t *testing.T) {
deployment.Spec.Template.Spec.Tolerations)
}
checkDeploymentHasEnvVar(t, deployment, "ROUTER_LOAD_BALANCE_ALGORITHM", true, "random")
checkDeploymentHasEnvVar(t, deployment, "ROUTER_USE_PROXY_PROTOCOL", false, "")
checkDeploymentHasEnvVar(t, deployment, "ROUTER_CANONICAL_HOSTNAME", false, "")
......@@ -319,6 +322,9 @@ func TestDesiredRouterDeployment(t *testing.T) {
}
var expectedReplicas int32 = 8
ci.Spec.Replicas = &expectedReplicas
ci.Spec.UnsupportedConfigOverrides = runtime.RawExtension{
Raw: []byte(`{"loadBalancingAlgorithm":"leastconn"}`),
}
ci.Status.Domain = "example.com"
ci.Status.EndpointPublishingStrategy.Type = operatorv1.LoadBalancerServiceStrategyType
proxyNeeded, err = IsProxyProtocolNeeded(ci, infraConfig.Status.PlatformStatus)
......@@ -353,6 +359,8 @@ func TestDesiredRouterDeployment(t *testing.T) {
t.Errorf("expected empty startup probe host, got %q", deployment.Spec.Template.Spec.Containers[0].StartupProbe.Handler.HTTPGet.Host)
}
checkDeploymentHasEnvVar(t, deployment, "ROUTER_LOAD_BALANCE_ALGORITHM", true, "leastconn")
checkDeploymentHasEnvVar(t, deployment, "ROUTER_USE_PROXY_PROTOCOL", true, "true")
checkDeploymentHasEnvVar(t, deployment, "ROUTER_UNIQUE_ID_HEADER_NAME", true, "unique-id")
......@@ -381,6 +389,11 @@ func TestDesiredRouterDeployment(t *testing.T) {
checkDeploymentHasEnvVar(t, deployment, "ROUTER_IP_V4_V6_MODE", true, "v4v6")
// Any value for loadBalancingAlgorithm other than "leastconn" should be
// ignored.
ci.Spec.UnsupportedConfigOverrides = runtime.RawExtension{
Raw: []byte(`{"loadBalancingAlgorithm":"source"}`),
}
ci.Status.EndpointPublishingStrategy.LoadBalancer = &operatorv1.LoadBalancerStrategy{
Scope: operatorv1.ExternalLoadBalancer,
ProviderParameters: &operatorv1.ProviderLoadBalancerParameters{
......@@ -418,6 +431,9 @@ func TestDesiredRouterDeployment(t *testing.T) {
if len(deployment.Spec.Template.Spec.Containers[0].StartupProbe.Handler.HTTPGet.Host) != 0 {
t.Errorf("expected empty startup probe host, got %q", deployment.Spec.Template.Spec.Containers[0].StartupProbe.Handler.HTTPGet.Host)
}
checkDeploymentHasEnvVar(t, deployment, "ROUTER_LOAD_BALANCE_ALGORITHM", true, "random")
checkDeploymentDoesNotHaveEnvVar(t, deployment, "ROUTER_USE_PROXY_PROTOCOL")
checkDeploymentHasEnvVar(t, deployment, "ROUTER_UNIQUE_ID_HEADER_NAME", true, "unique-id")
......
......@@ -42,6 +42,7 @@ import (
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
......@@ -57,6 +58,11 @@ import (
)
var (
availableConditionsForPrivateIngressController = []operatorv1.OperatorCondition{
{Type: operatorv1.IngressControllerAvailableConditionType, Status: operatorv1.ConditionTrue},
{Type: operatorv1.LoadBalancerManagedIngressConditionType, Status: operatorv1.ConditionFalse},
{Type: operatorv1.DNSManagedIngressConditionType, Status: operatorv1.ConditionFalse},
}
availableConditionsForIngressControllerWithNodePort = []operatorv1.OperatorCondition{
{Type: operatorv1.IngressControllerAvailableConditionType, Status: operatorv1.ConditionTrue},
{Type: operatorv1.LoadBalancerManagedIngressConditionType, Status: operatorv1.ConditionFalse},
......@@ -801,12 +807,7 @@ func TestTLSSecurityProfile(t *testing.T) {
t.Fatalf("failed to create ingresscontroller %s: %v", name, err)
}
defer assertIngressControllerDeleted(t, kclient, ic)
conditions := []operatorv1.OperatorCondition{
{Type: operatorv1.IngressControllerAvailableConditionType, Status: operatorv1.ConditionTrue},
{Type: operatorv1.LoadBalancerManagedIngressConditionType, Status: operatorv1.ConditionFalse},
{Type: operatorv1.DNSManagedIngressConditionType, Status: operatorv1.ConditionFalse},
}
if err := waitForIngressControllerCondition(t, kclient, 5*time.Minute, name, conditions...); err != nil {
if err := waitForIngressControllerCondition(t, kclient, 5*time.Minute, name, availableConditionsForPrivateIngressController...); err != nil {
t.Fatalf("failed to observe expected conditions: %v", err)
}
......@@ -870,12 +871,7 @@ func TestRouteAdmissionPolicy(t *testing.T) {
t.Fatalf("failed to create ingresscontroller %s: %v", icName, err)
}
defer assertIngressControllerDeleted(t, kclient, ic)
conditions := []operatorv1.OperatorCondition{
{Type: operatorv1.IngressControllerAvailableConditionType, Status: operatorv1.ConditionTrue},
{Type: operatorv1.LoadBalancerManagedIngressConditionType, Status: operatorv1.ConditionFalse},
{Type: operatorv1.DNSManagedIngressConditionType, Status: operatorv1.ConditionFalse},
}
if err := waitForIngressControllerCondition(t, kclient, 5*time.Minute, icName, conditions...); err != nil {
if err := waitForIngressControllerCondition(t, kclient, 5*time.Minute, icName, availableConditionsForPrivateIngressController...); err != nil {
t.Fatalf("failed to observe expected conditions: %v", err)
}
......@@ -1192,12 +1188,7 @@ $ModLoad omstdout.so
t.Fatalf("failed to create ingresscontroller %s: %v", icName, err)
}
defer assertIngressControllerDeleted(t, kclient, ic)
conditions := []operatorv1.OperatorCondition{
{Type: operatorv1.IngressControllerAvailableConditionType, Status: operatorv1.ConditionTrue},
{Type: operatorv1.LoadBalancerManagedIngressConditionType, Status: operatorv1.ConditionFalse},
{Type: operatorv1.DNSManagedIngressConditionType, Status: operatorv1.ConditionFalse},
}
if err := waitForIngressControllerCondition(t, kclient, 5*time.Minute, icName, conditions...); err != nil {
if err := waitForIngressControllerCondition(t, kclient, 5*time.Minute, icName, availableConditionsForPrivateIngressController...); err != nil {
t.Fatalf("failed to observe expected conditions: %v", err)
}
......@@ -1256,12 +1247,7 @@ func TestContainerLogging(t *testing.T) {
t.Fatalf("failed to create ingresscontroller %s: %v", icName, err)
}
defer assertIngressControllerDeleted(t, kclient, ic)
conditions := []operatorv1.OperatorCondition{
{Type: operatorv1.IngressControllerAvailableConditionType, Status: operatorv1.ConditionTrue},
{Type: operatorv1.LoadBalancerManagedIngressConditionType, Status: operatorv1.ConditionFalse},
{Type: operatorv1.DNSManagedIngressConditionType, Status: operatorv1.ConditionFalse},
}
if err := waitForIngressControllerCondition(t, kclient, 5*time.Minute, icName, conditions...); err != nil {
if err := waitForIngressControllerCondition(t, kclient, 5*time.Minute, icName, availableConditionsForPrivateIngressController...); err != nil {
t.Fatalf("failed to observe expected conditions: %v", err)
}
}
......@@ -1671,12 +1657,7 @@ func TestUniqueIdHeader(t *testing.T) {
t.Fatalf("failed to create ingresscontroller %s: %v", icName, err)
}
defer assertIngressControllerDeleted(t, kclient, ic)
conditions := []operatorv1.OperatorCondition{
{Type: operatorv1.IngressControllerAvailableConditionType, Status: operatorv1.ConditionTrue},
{Type: operatorv1.LoadBalancerManagedIngressConditionType, Status: operatorv1.ConditionFalse},
{Type: operatorv1.DNSManagedIngressConditionType, Status: operatorv1.ConditionFalse},
}
if err := waitForIngressControllerCondition(t, kclient, 5*time.Minute, icName, conditions...); err != nil {
if err := waitForIngressControllerCondition(t, kclient, 5*time.Minute, icName, availableConditionsForPrivateIngressController...); err != nil {
t.Fatalf("failed to observe expected conditions: %v", err)
}
......@@ -1782,6 +1763,47 @@ func TestUniqueIdHeader(t *testing.T) {
}
}
// TestLoadBalancingAlgorithmUnsupportedConfigOverride verifies that the
// operator configures router pod replicas to use the "leastconn" load-balancing
// algorithm if the ingresscontroller is so configured using an unsupported
// config override.
func TestLoadBalancingAlgorithmUnsupportedConfigOverride(t *testing.T) {
icName := types.NamespacedName{Namespace: operatorNamespace, Name: "leastconn"}
domain := icName.Name + "." + dnsConfig.Spec.BaseDomain
ic := newPrivateController(icName, domain)
if err := kclient.Create(context.TODO(), ic); err != nil {
t.Fatalf("failed to create ingresscontroller: %v", err)
}
defer assertIngressControllerDeleted(t, kclient, ic)
if err := waitForIngressControllerCondition(t, kclient, 5*time.Minute, icName, availableConditionsForPrivateIngressController...); err != nil {
t.Errorf("failed to observe expected conditions: %w", err)
}
if err := kclient.Get(context.TODO(), icName, ic); err != nil {
t.Fatalf("failed to get ingresscontroller: %v", err)
}
deployment := &appsv1.Deployment{}
if err := kclient.Get(context.TODO(), controller.RouterDeploymentName(ic), deployment); err != nil {
t.Fatalf("failed to get ingresscontroller deployment: %v", err)
}
expectedAlgorithm := "random"
if err := waitForDeploymentEnvVar(t, kclient, deployment, 30*time.Second, "ROUTER_LOAD_BALANCE_ALGORITHM", expectedAlgorithm); err != nil {
t.Fatalf("expected initial deployment to use the %q algorithm: %v", expectedAlgorithm, err)
}
ic.Spec.UnsupportedConfigOverrides = runtime.RawExtension{
Raw: []byte(`{"loadBalancingAlgorithm":"leastconn"}`),
}
if err := kclient.Update(context.TODO(), ic); err != nil {
t.Fatalf("failed to update ingresscontroller: %v", err)
}
expectedAlgorithm = "leastconn"
if err := waitForDeploymentEnvVar(t, kclient, deployment, 1*time.Minute, "ROUTER_LOAD_BALANCE_ALGORITHM", expectedAlgorithm); err != nil {
t.Fatalf("expected updated deployment to use the %q algorithm: %v", expectedAlgorithm, err)
}
}
func newLoadBalancerController(name types.NamespacedName, domain string) *operatorv1.IngressController {
repl := int32(1)
return &operatorv1.IngressController{
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment