Support cert-manager for controller tls (googleforgames#2453)

* Support cert-manager for controller tls * fix missing annotations * fix spacing * fix * fix spacing * Fix typo * miss another one * fix test * add missing annotation to APIService * update values * review * minor fix * Allow disable of ca-bundle * camelcase

Author: xxtanisxx

install/helm/agones/templates/extensions.yaml

@@ -28,18 +28,24 @@ metadata:
 chart: {{template "agones.chart" . }}
 release: {{.Release.Name }}
 heritage: {{.Release.Service }}
+{{- if .Values.agones.controller.allocationApiService.annotations }}
+annotations:
+{{- toYaml .Values.agones.controller.allocationApiService.annotations | indent 4 }}
+{{- end }}
 spec:
 group: allocation.agones.dev
 groupPriorityMinimum: 1000
 versionPriority: 15
 service:
 name: agones-controller-service
 namespace: {{.Release.Namespace }}
-{{- if .Values.agones.controller.generateTLS }}
+{{- if not .Values.agones.controller.allocationApiService.disableCaBundle }}
+{{- if .Values.agones.controller.generateTLS }}
 caBundle: {{b64enc $ca.Cert }}
-{{- else }}
+{{- else }}
 caBundle: {{default (.Files.Get "certs/server.crt") .Values.agones.controller.tlsCert | b64enc }}
-{{- end }}
+{{- end }}
+{{- end }}
 version: v1
 {{- end}}
 {{- if .Values.agones.registerWebhooks }}
@@ -48,6 +54,10 @@ apiVersion: admissionregistration.k8s.io/v1
 kind: ValidatingWebhookConfiguration
 metadata:
 name: agones-validation-webhook
+{{- if .Values.agones.controller.validatingWebhook.annotations }}
+annotations:
+{{- toYaml .Values.agones.controller.validatingWebhook.annotations | indent 4 }}
+{{- end }}
 labels:
 component: controller
 app: {{template "agones.name" . }}
@@ -65,10 +75,12 @@ webhooks:
 name: agones-controller-service
 namespace: {{.Release.Namespace }}
 path: /validate
+{{- if not .Values.agones.controller.validatingWebhook.disableCaBundle }}
 {{- if .Values.agones.controller.generateTLS }}
 caBundle: {{b64enc $ca.Cert }}
 {{- else }}
 caBundle: {{default (.Files.Get "certs/server.crt") .Values.agones.controller.tlsCert | b64enc }}
+{{- end }}
 {{- end }}
 rules:
  - apiGroups:
@@ -105,6 +117,10 @@ apiVersion: admissionregistration.k8s.io/v1
 kind: MutatingWebhookConfiguration
 metadata:
 name: agones-mutation-webhook
+{{- if .Values.agones.controller.mutatingWebhook.annotations }}
+annotations:
+{{- toYaml .Values.agones.controller.mutatingWebhook.annotations | indent 4 }}
+{{- end }}
 labels:
 component: controller
 app: {{template "agones.name" . }}
@@ -122,10 +138,12 @@ webhooks:
 name: agones-controller-service
 namespace: {{.Release.Namespace }}
 path: /mutate
+{{- if not .Values.agones.controller.mutatingWebhook.disableCaBundle }}
 {{- if .Values.agones.controller.generateTLS }}
 caBundle: {{b64enc $ca.Cert }}
 {{- else }}
 caBundle: {{default (.Files.Get "certs/server.crt") .Values.agones.controller.tlsCert | b64enc }}
+{{- end }}
 {{- end }}
 rules:
  - apiGroups:
@@ -147,6 +165,7 @@ webhooks:
  - CREATE
  - UPDATE
 {{- end }}
+{{- if not .Values.agones.controller.disableSecret }}
 ---
 apiVersion: v1
 kind: Secret
@@ -167,3 +186,4 @@ data:
 server.crt: {{default (.Files.Get "certs/server.crt") .Values.agones.controller.tlsCert | b64enc }}
 server.key: {{default (.Files.Get "certs/server.key") .Values.agones.controller.tlsKey | b64enc }}
 {{- end }}
+{{- end }}

install/helm/agones/values.yaml

@@ -67,6 +67,16 @@ agones:
 generateTLS: true
 tlsCert: ""
 tlsKey: ""
+disableSecret: false
+allocationApiService:
+annotations: {}
+disableCaBundle: false
+validatingWebhook:
+annotations: {}
+disableCaBundle: false
+mutatingWebhook:
+annotations: {}
+disableCaBundle: false
 safeToEvict: false
 persistentLogs: true
 persistentLogsSizeLimitMB: 10000

site/content/en/docs/Installation/Install Agones/helm.md

@@ -209,6 +209,13 @@ The following tables lists the configurable parameters of the Agones chart and t
 | Parameter | Description | Default |
 | --------------------------------------------------- | ----------------------------------------------------------------------------------------------- | ---------------------- |
 ||||
+|`agones.controller.disableSecret`| Disables the creation of any allocator secrets. If true, you MUST provide the `{agones.releaseName}-cert` secrets before installation. |`false`|
+|`agones.controller.allocationApiService.annotations`|[Annotations][annotations] added to the Agones apiregistration |`{}`|
+|`agones.controller.allocationApiService.disableCaBundle`| Disable ca-bundle so it can be injected by cert-manager |`false`|
+|`agones.controller.validatingWebhook.annotations`|[Annotations][annotations] added to the Agones validating webhook |`{}`|
+|`agones.controller.validatingWebhook.disableCaBundle`| Disable ca-bundle so it can be injected by cert-manager |`false`|
+|`agones.controller.mutatingWebhook.annotations`|[Annotations][annotations] added to the Agones mutating webhook |`{}`|
+|`agones.controller.mutatingWebhook.disableCaBundle`| Disable ca-bundle so it can be injected by cert-manager |`false`|
 {{% /feature %}}
 
 [toleration]: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
@@ -276,12 +283,72 @@ That means that you skipped the `--cleanup` flag and you should either delete th
 ## Controller TLS Certificates
 
 By default agones chart generates tls certificates used by the admission controller, while this is handy, it requires the agones controller to restart on each `helm upgrade` command.
+
+### Manual
+
 For most use cases the controller would have required a restart anyway (eg: controller image updated). However if you really need to avoid restarts we suggest that you turn off tls automatic generation (`agones.controller.generateTLS` to `false`) and provide your own certificates (`certs/server.crt`,`certs/server.key`).
 
 {{< alert title="Tip" color="info">}}
 You can use our script located at{{< ghlink href="install/helm/agones/certs/cert.sh" >}}cert.sh{{< /ghlink >}} to generate them.
 {{< /alert >}}
 
+{{% feature publishVersion="1.22.0" %}}
+### Cert-Manager
+
+Another approach is to use [cert-manager.io](https://cert-manager.io/) solution for cluster level certificate management.
+
+In order to use the cert-manager solution, first [install cert-manager](https://cert-manager.io/docs/installation/kubernetes/) on the cluster.
+Then, [configure](https://cert-manager.io/docs/configuration/) an `Issuer`/`ClusterIssuer` resource and
+last [configure](https://cert-manager.io/docs/usage/certificate/) a `Certificate` resource to manage controller `Secret`.
+Make sure to configure the `Certificate` based on your system's requirements, including the validity `duration`.
+
+Here is an example of using a self-signed `ClusterIssuer` for configuring controller `Secret` where secret name is `my-release-cert` or `{{template "agones.fullname" . }}-cert`:
+
+```bash
+#!/bin/bash
+# Create a self-signed ClusterIssuer
+cat <<EOF | kubectl apply -f -
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+ name: selfsigned
+spec:
+ selfSigned:{}
+EOF
+
+# Create a Certificate with IP for the my-release-cert )
+cat <<EOF | kubectl apply -f -
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+ name: my-release-cert
+ namespace: agones-system
+spec:
+ ipAddresses:
+ - agones-controller-service.agones-system.svc
+ secretName: my-release-cert
+ issuerRef:
+ name: selfsigned
+ kind: ClusterIssuer
+EOF
+```
+
+After the certificates are generated, we will want to [inject caBundle](https://cert-manager.io/docs/concepts/ca-injector/) into controller webhook and disable controller secret creation by setting the following:
+
+```bash
+helm install my-release \
+ --set agones.controller.disableSecret=true \
+ --set agones.controller.allocationApiService.annotations={'cert-manager.io/inject-ca-from': 'agones-system/my-release-cert'} \
+ --set agones.controller.allocationApiService.disableCaBundle=true \
+ --set agones.controller.validatingWebhook.annotations={'cert-manager.io/inject-ca-from': 'agones-system/my-release-cert'} \
+ --set agones.controller.validatingWebhook.disableCaBundle=true \
+ --set agones.controller.mutatingWebhook.annotations={'cert-manager.io/inject-ca-from': 'agones-system/my-release-cert'} \
+ --set agones.controller.mutatingWebhook.disableCaBundle=true \
+ --namespace agones-system --create-namespace \
+ agones/agones
+```
+{{% /feature %}}
+
 ## Reserved Allocator Load Balancer IP
 
 In order to reuse the existing load balancer IP on upgrade or install the `agones-allocator` service as a `LoadBalancer` using a reserved static IP, a user can specify the load balancer's IP with the `agones.allocator.http.loadBalancerIP` helm configuration parameter value. By setting the `loadBalancerIP` value: