[cern] Add env vars to deployment and annotations for LB service

.gitlab-ci.yml

+include:
+  - project: 'paas-tools/infrastructure-ci'
+    file: 'docker-images-ci-templates/DockerImages.gitlab-ci.yml'

Dockerfile

-FROM registry.ci.openshift.org/ocp/builder:rhel-8-golang-1.16-openshift-4.8 AS builder
+FROM registry.ci.openshift.org/openshift/release:golang-1.16 AS builder
 WORKDIR /ingress-operator
 COPY . .
 RUN make build
 
-FROM registry.ci.openshift.org/ocp/4.8:base
+FROM registry.ci.openshift.org/origin/4.8:base
 COPY --from=builder /ingress-operator/ingress-operator /usr/bin/
 COPY manifests /manifests
 ENTRYPOINT ["/usr/bin/ingress-operator"]

README.md

+# CERN modifications
+
+Openshift's ingress operator gives us very little flexibility for configuration the HAProxy router deployments and associated services.
+
+One issue in particular made it necessary to fork the upstream component and add our own modifications: the ability to modify Loadbalancer services created by the operator (we need to set specific annotations for Openstack) and the ability to selectively enable the PROXY protocol for specific router shards.
+Details can be found in [okd4-install!703](https://gitlab.cern.ch/paas-tools/okd4-install/-/merge_requests/703).
+
+The Gitlab repository is configured to automatically mirror the [upstream cluster-ingress-operator repository](https://github.com/openshift/cluster-ingress-operator).
+
+Use the following workflow to port our custom patches to a new release:
+
+```sh
+# Make sure your repo is up-to-date
+git fetch origin
+# Check out the pristine upstream release
+git checkout origin/release-4.x
+# Push a new branch with the pristine upstream release
+git checkout -b cern-4.x
+git push origin cern-4.x
+# Create a new branch for development
+git checkout -b cern-4.x-dev
+```
+
+Then, locate the latest patches that have been made by us.
+This can be done by checking out an old release branch (e.g. `cern-4.7`) and looking at the commits prefixed with `[cern]`.
+
+```
+git log origin/cern_cloud-4.7
+git cherry-pick [CERN_PATCH_REFS]
+```
+
+At this point, you might need to resolve the merge conflicts (*fingers crossed*).
+Afterwards, commit your changes with a `[cern]` prefix and push them to the dev branch.
+Finally, create a merge request from `cern-4.x-dev` to `cern-4.x` so someone else can review your changes.
+
 # OpenShift Ingress Operator
 
 Ingress Operator is an [OpenShift](https://www.openshift.com) component which enables external access to cluster services by configuring Ingress Controllers, which route traffic as specified by OpenShift [Route](https://docs.openshift.com/container-platform/3.11/architecture/networking/routes.html) and Kubernetes [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) resources.

pkg/operator/controller/ingress/controller.go

@@ -804,6 +804,11 @@ func IsProxyProtocolNeeded(ic *operatorv1.IngressController, platform *configv1.
 	}
 	switch ic.Status.EndpointPublishingStrategy.Type {
 	case operatorv1.LoadBalancerServiceStrategyType:
+		// We always want to enable the PROXY protocol when we are using services of type loadbalancer
+		if ic.Status.EndpointPublishingStrategy.Type == operatorv1.LoadBalancerServiceStrategyType {
+			return true, nil
+		}
+
 		// For now, check if we are on AWS. This can really be done for for any external
 		// [cloud] LBs that support the proxy protocol.
 		if platform.Type == configv1.AWSPlatformType {

pkg/operator/controller/ingress/deployment.go

@@ -383,6 +383,39 @@ func desiredRouterDeployment(ci *operatorv1.IngressController, ingressController
 		{Name: "STATS_PASSWORD_FILE", Value: filepath.Join(statsVolumeMountPath, "statsPassword")},
 	}
 
+	// Enable IPv6 for HostNetwork router pods
+	// IPv6 is only enabled automatically when the entire cluster is running in Dual-Stack mode
+	// We don't do that, but our HAProxy routers should still listen on IPv6 addresses
+	// because they are running in HostNetwork mode on our ingress nodes (which have IPv6 addresses).
+	if ci.Status.EndpointPublishingStrategy.Type == operatorv1.HostNetworkStrategyType {
+		env = append(env, corev1.EnvVar{Name: "ROUTER_IP_V4_V6_MODE", Value: "v4v6"})
+	}
+
+	// Add the following environment variables only routers serving user applications
+	if ci.Name != "default" {
+		// Graceful shutdown (see https://github.com/openshift/router/pull/94)
+		// amount of time router still accepts new connections when pod enters Terminating state
+		// we need to wait 2 minutes before we can expect clients to stop connecting to a router pod
+		// shutting down:
+		// - external-dns updates the DNS every 60 seconds
+		// - the DNS record TTL is 60 seconds
+		env = append(env, corev1.EnvVar{Name: "ROUTER_GRACEFUL_SHUTDOWN_DELAY", Value: "120s"})
+		// amount of time the router gives existing active connection to terminate gracefully after router
+		// stops accepting new connections. The router shuts down earlier if all connections terminated,
+		// or terminated remaining connections at the end of this delay.
+		// NB: this cannot be more than 1h because the ingress-operator hardcodes a terminationGracePeriod
+		// of 3600 seconds for router pods.
+		// unlike the others, this value must be a raw number of seconds (e.g. 30).
+		env = append(env, corev1.EnvVar{Name: "ROUTER_MAX_SHUTDOWN_TIMEOUT", Value: "900"})
+
+		// Some clients are slow to send the request, e.g. SVN.
+		// The OKD4 defaults are too aggressive.
+		// https://github.com/openshift/router/blob/df0c5dfbc792d1af2bbcab0fc44bb9cb0d200e4e/images/router/haproxy/conf/haproxy-config.template#L148-L154
+		// Values determined from years of operating Openshift 3
+		env = append(env, corev1.EnvVar{Name: "ROUTER_DEFAULT_CLIENT_TIMEOUT", Value: "60s"})
+		env = append(env, corev1.EnvVar{Name: "ROUTER_SLOWLORIS_TIMEOUT", Value: "60s"})
+	}
+
 	// Enable prometheus metrics
 	certsSecretName := fmt.Sprintf("router-metrics-certs-%s", ci.Name)
 	certsVolumeName := "metrics-certs"

pkg/operator/controller/ingress/load_balancer_service.go

@@ -243,8 +243,13 @@ func desiredLoadBalancerService(ci *operatorv1.IngressController, deploymentRef
 		service.Annotations = map[string]string{}
 	}
 
+
+	// Add Openstack- and CERN-specific annotations to the Loadbalancer service
+	// https://github.com/kubernetes/cloud-provider-openstack/blob/e5e34df195f293c563a8f20b3ddf8a9b64b96456/docs/openstack-cloud-controller-manager/expose-applications-using-loadbalancer-type-service.md#service-annotations
+
 	if proxyNeeded {
-		service.Annotations[awsLBProxyProtocolAnnotation] = "*"
+		// Enable the PROXY protocol on the side of the Openstack cloud loadbalancer
+		service.Annotations["loadbalancer.openstack.org/proxy-protocol"] = "true"
 	}
 
 	if platform != nil {