New CRD StackConfigPolicy to declaratively configure multiple Elasticsearch clusters (elastic#6148)

New custom resource StackConfigPolicy to configure cluster settings, snapshot repositorires and lifecycle policies for a list of Elasticsearch clusters matching a labels selector.

```
apiVersion: stackconfigpolicy.k8s.elastic.co/v1alpha1
kind: StackConfigPolicy
metadata:
  name: config-staging
  namespace: elastic-system
spec:
  resourceSelector:
    matchLabels:
      env: staging
  elasticsearch:
    clusterSettings:
      indices.recovery.max_bytes_per_sec: "100mb"
    snapshotRepositories:
      backup:
        type: gcs
        settings:
          bucket: "gcs-bucket"
    snapshotLifecyclePolicies:
      test-snapshots:
        schedule: "0 1 2 3 4 ?"
        name: "<staging-snap-{now/d}>"
        repository: "backup"
        config:
          indices: ["*"]  
          ignore_unavailable: true
          include_global_state: false
        retention:
          expire_after: "30d"
          min_count: 1
          max_count: 50
```

The namespaces used to find Elasticsearch clusters to configure depend on the StackConfigPolicy namespace. If it is the operator namespace, all namespaces managed by the operator are used, otherwise only the StackConfigPolicy namespace.

A new FileSettings Secret `<esName>-es-file-settings` owns by Elasticsearch is created empty and mounted as data volume by the Elasticsearch controller. The StackConfigPolicy controller soft owns the Secret and only updates it. On deletion, soft owned labels are used to find Secrets to reset.

For the SecureSettings Secrets defined in the StackConfigPolicy, the StackConfigPolicy controller writes the Secret namespaces and names in an annotation of the FileSettings Secret. The ES controller watches and reads the Secrets from that and merges their content with the user-provided Secrets of the Elasticsearch resource in the existing Secret `<esName>-es-secure-settings`.

Author: thbkrkr

cmd/manager/main.go

@@ -52,6 +52,7 @@ import (
 	kbv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/kibana/v1"
 	kbv1beta1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/kibana/v1beta1"
 	emsv1alpha1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/maps/v1alpha1"
+	policyv1alpha1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/stackconfigpolicy/v1alpha1"
 	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/agent"
 	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/apmserver"
 	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/association"
@@ -79,6 +80,7 @@ import (
 	licensetrial "github.com/elastic/cloud-on-k8s/v2/pkg/controller/license/trial"
 	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/maps"
 	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/remoteca"
+	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/stackconfigpolicy"
 	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/webhook"
 	"github.com/elastic/cloud-on-k8s/v2/pkg/dev"
 	"github.com/elastic/cloud-on-k8s/v2/pkg/dev/portforward"
@@ -845,6 +847,7 @@ func registerControllers(mgr manager.Manager, params operator.Parameters, access
 		{name: "LicenseTrial", registerFunc: licensetrial.Add},
 		{name: "Agent", registerFunc: agent.Add},
 		{name: "Maps", registerFunc: maps.Add},
+		{name: "StackConfigPolicy", registerFunc: stackconfigpolicy.Add},
 	}
 
 	for _, c := range controllers {
@@ -923,13 +926,14 @@ func garbageCollectSoftOwnedSecrets(ctx context.Context, k8sClient k8s.Client) {
 	defer span.End()
 
 	if err := reconciler.GarbageCollectAllSoftOwnedOrphanSecrets(ctx, k8sClient, map[string]client.Object{
-		esv1.Kind:          &esv1.Elasticsearch{},
-		apmv1.Kind:         &apmv1.ApmServer{},
-		kbv1.Kind:          &kbv1.Kibana{},
-		entv1.Kind:         &entv1.EnterpriseSearch{},
-		beatv1beta1.Kind:   &beatv1beta1.Beat{},
-		agentv1alpha1.Kind: &agentv1alpha1.Agent{},
-		emsv1alpha1.Kind:   &emsv1alpha1.ElasticMapsServer{},
+		esv1.Kind:           &esv1.Elasticsearch{},
+		apmv1.Kind:          &apmv1.ApmServer{},
+		kbv1.Kind:           &kbv1.Kibana{},
+		entv1.Kind:          &entv1.EnterpriseSearch{},
+		beatv1beta1.Kind:    &beatv1beta1.Beat{},
+		agentv1alpha1.Kind:  &agentv1alpha1.Agent{},
+		emsv1alpha1.Kind:    &emsv1alpha1.ElasticMapsServer{},
+		policyv1alpha1.Kind: &policyv1alpha1.StackConfigPolicy{},
 	}); err != nil {
 		log.Error(err, "Orphan secrets garbage collection failed, will be attempted again at next operator restart.")
 		return

config/crds/v1/all-crds.yaml

@@ -8908,3 +8908,213 @@ spec:
         type: object
     served: false
     storage: false
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.10.0
+  creationTimestamp: null
+  name: stackconfigpolicies.stackconfigpolicy.k8s.elastic.co
+spec:
+  group: stackconfigpolicy.k8s.elastic.co
+  names:
+    categories:
+    - elastic
+    kind: StackConfigPolicy
+    listKind: StackConfigPolicyList
+    plural: stackconfigpolicies
+    shortNames:
+    - scp
+    singular: stackconfigpolicy
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: Resources configured
+      jsonPath: .status.readyCount
+      name: Ready
+      type: string
+    - jsonPath: .status.phase
+      name: Phase
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: StackConfigPolicy represents a StackConfigPolicy resource in
+          a Kubernetes cluster.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            properties:
+              elasticsearch:
+                properties:
+                  clusterSettings:
+                    description: ClusterSettings holds the Elasticsearch cluster settings
+                      (/_cluster/settings)
+                    type: object
+                    x-kubernetes-preserve-unknown-fields: true
+                  snapshotLifecyclePolicies:
+                    description: SnapshotLifecyclePolicies holds the Snapshot Lifecycle
+                      Policies settings (/_slm/policy)
+                    type: object
+                    x-kubernetes-preserve-unknown-fields: true
+                  snapshotRepositories:
+                    description: SnapshotRepositories holds the Snapshot Repositories
+                      settings (/_snapshot)
+                    type: object
+                    x-kubernetes-preserve-unknown-fields: true
+                type: object
+              resourceSelector:
+                description: A label selector is a label query over a set of resources.
+                  The result of matchLabels and matchExpressions are ANDed. An empty
+                  label selector matches all objects. A null label selector matches
+                  no objects.
+                properties:
+                  matchExpressions:
+                    description: matchExpressions is a list of label selector requirements.
+                      The requirements are ANDed.
+                    items:
+                      description: A label selector requirement is a selector that
+                        contains values, a key, and an operator that relates the key
+                        and values.
+                      properties:
+                        key:
+                          description: key is the label key that the selector applies
+                            to.
+                          type: string
+                        operator:
+                          description: operator represents a key's relationship to
+                            a set of values. Valid operators are In, NotIn, Exists
+                            and DoesNotExist.
+                          type: string
+                        values:
+                          description: values is an array of string values. If the
+                            operator is In or NotIn, the values array must be non-empty.
+                            If the operator is Exists or DoesNotExist, the values
+                            array must be empty. This array is replaced during a strategic
+                            merge patch.
+                          items:
+                            type: string
+                          type: array
+                      required:
+                      - key
+                      - operator
+                      type: object
+                    type: array
+                  matchLabels:
+                    additionalProperties:
+                      type: string
+                    description: matchLabels is a map of {key,value} pairs. A single
+                      {key,value} in the matchLabels map is equivalent to an element
+                      of matchExpressions, whose key field is "key", the operator
+                      is "In", and the values array contains only "value". The requirements
+                      are ANDed.
+                    type: object
+                type: object
+                x-kubernetes-map-type: atomic
+              secureSettings:
+                items:
+                  description: SecretSource defines a data source based on a Kubernetes
+                    Secret.
+                  properties:
+                    entries:
+                      description: Entries define how to project each key-value pair
+                        in the secret to filesystem paths. If not defined, all keys
+                        will be projected to similarly named paths in the filesystem.
+                        If defined, only the specified keys will be projected to the
+                        corresponding paths.
+                      items:
+                        description: KeyToPath defines how to map a key in a Secret
+                          object to a filesystem path.
+                        properties:
+                          key:
+                            description: Key is the key contained in the secret.
+                            type: string
+                          path:
+                            description: Path is the relative file path to map the
+                              key to. Path must not be an absolute file path and must
+                              not contain any ".." components.
+                            type: string
+                        required:
+                        - key
+                        type: object
+                      type: array
+                    secretName:
+                      description: SecretName is the name of the secret.
+                      type: string
+                  required:
+                  - secretName
+                  type: object
+                type: array
+            type: object
+          status:
+            properties:
+              errors:
+                description: Errors is the number of resources which have an incorrect
+                  configuration
+                type: integer
+              observedGeneration:
+                description: ObservedGeneration is the most recent generation observed
+                  for this StackConfigPolicy.
+                format: int64
+                type: integer
+              phase:
+                description: Phase is the phase of the StackConfigPolicy.
+                type: string
+              ready:
+                description: Ready is the number of resources successfully configured.
+                type: integer
+              readyCount:
+                description: ReadyCount is a human representation of the number of
+                  resources successfully configured.
+                type: string
+              resources:
+                description: Resources is the number of resources to be configured.
+                type: integer
+              resourcesStatuses:
+                additionalProperties:
+                  description: ResourcePolicyStatus models the status of the policy
+                    for one resource to be configured.
+                  properties:
+                    currentVersion:
+                      format: int64
+                      type: integer
+                    error:
+                      properties:
+                        message:
+                          type: string
+                        version:
+                          format: int64
+                          type: integer
+                      type: object
+                    expectedVersion:
+                      format: int64
+                      type: integer
+                    phase:
+                      type: string
+                  type: object
+                description: ResourcesStatuses holds the status for each resource
+                  to be configured.
+                type: object
+            required:
+            - resourcesStatuses
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}

config/crds/v1/bases/kustomization.yaml

@@ -7,3 +7,4 @@ resources:
   - beat.k8s.elastic.co_beats.yaml
   - agent.k8s.elastic.co_agents.yaml
   - maps.k8s.elastic.co_elasticmapsservers.yaml
+  - stackconfigpolicy.k8s.elastic.co_stackconfigpolicies.yaml