Merge pull request #4989 from cfpb/mskhelm

changes to helm chart add kafka security

Author: lchen-2101

common/src/main/resources/persistence-keyspace-sigv4.conf

@@ -99,10 +99,16 @@ kafka {
   idle-timeout = ${?KAFKA_IDLE_TIMEOUT}
   security.protocol=""
   security.protocol=${?KAFKA_SECURITY}
-  ssl.truststore.location = ""
-  ssl.truststore.location = ${?TRUSTSTORE_PATH}
-  ssl.truststore.password = ""
-  ssl.truststore.password = ${?TRUSTSTORE_PASSWORD}
-  ssl.endpoint = ""
-  ssl.endpoint = ${?KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM_CONFIG}
+  # ssl.truststore.location = ""
+  # ssl.truststore.location = ${?TRUSTSTORE_PATH}
+  # ssl.truststore.password = ""
+  # ssl.truststore.password = ${?TRUSTSTORE_PASSWORD}
+  # ssl.endpoint = ""
+  # ssl.endpoint = ${?KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM_CONFIG}
+  sasl.mechanism="AWS_MSK_IAM"
+  # sasl.mechanism=${?KAFKA_SASL_MECHANISM}
+  sasl.jaas.config="software.amazon.msk.auth.iam.IAMLoginModule required;"
+  # sasl.jaas.config="{?KAFKA_SASL_JAAS_CONFIG}"
+  sasl.client.callback.handler.class="software.amazon.msk.auth.iam.IAMClientCallbackHandler"
+  #sasl.client.callback.handler.class="{?KAFKA_SASL_CLASS}"  
 }

kubernetes/beta/hmda-platform/templates/deployment.yaml

@@ -132,6 +132,10 @@ spec:
             configMapKeyRef:
               name: timed-guards
               key: actionQ3End
+        {{- if .Values.kafka.security }}
+        - name: KAFKA_SECURITY
+          value: {{.Values.kafka.security}}
+        {{- end }}
         - name: KAFKA_INSTITUTIONS_TOPIC
           value: {{.Values.kafka.institutionsTopic}}
         - name: KAFKA_SIGN_TOPIC

kubernetes/beta/hmda-platform/values.yaml

@@ -33,6 +33,7 @@ kafka:
   analyticsTopic: beta-hmda-analytics
   signTopic: beta-hmda-sign
   emailTopic: beta-hmda-email
+  security: ""
 
 service:
   type: ClusterIP

kubernetes/beta/institutions-api/templates/deployment.yaml

@@ -83,11 +83,25 @@ spec:
                   key: postgres.ssl
             - name: INSTITUTION_PG_CREATE_SCHEMA
               value: "{{ .Values.postgresql.createSchema }}"
+            {{- if .Values.kafka.security }}
+            - name: KAFKA_SECURITY
+              value: {{.Values.kafka.security}}
+            {{- end }}
             - name: KAFKA_CLUSTER_HOSTS
               valueFrom:
                 configMapKeyRef:
                   name: kafka-configmap
                   key: kafka.hosts
+            - name: AWS_ACCESS_KEY_ID
+              valueFrom:
+                secretKeyRef:
+                  name: aws-credentials
+                  key: aws-access-key-id
+            - name: AWS_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: aws-credentials
+                  key: aws-secret-access-key
             - name: JDBC_URL
               valueFrom:
                 secretKeyRef:

kubernetes/beta/institutions-api/values.yaml

@@ -6,7 +6,7 @@ replicaCount: 1
 
 image:
   repository: hmda/institutions-api
-  tag: v2.7.2
+  tag: latest
   pullPolicy: IfNotPresent
 
 service:
@@ -20,6 +20,7 @@ currentYear: 2018
 kafka:
   institutionsGroup: beta-institution-group
   institutionsTopic: beta-institution
+  security: ""
 
 #ambassador:
 #  name: institutions-api-ambassador
@@ -68,3 +69,6 @@ affinity: {}
 postgresql:
   enabled: false
   createSchema: false
+
+ambassador_id: ambassador-beta-2
+# ambassador_host: hmda-public.cfpb.gov

kubernetes/email-service/templates/deployment.yaml

@@ -52,12 +52,12 @@ spec:
             - name: AWS_ACCESS_KEY_ID
               valueFrom:
                 secretKeyRef:
-                  name: aws-email-credentials
+                  name: aws-credentials
                   key: aws-access-key-id
             - name: AWS_SECRET_ACCESS_KEY
               valueFrom:
                 secretKeyRef:
-                  name: aws-email-credentials
+                  name: aws-credentials
                   key: aws-secret-access-key
             - name: AWS_ENVIRONMENT
               valueFrom:
@@ -75,6 +75,10 @@ spec:
               value: {{.Values.kafka.emailTopic}}
             - name: KAFKA_EMAIL_CONSUMER_GROUP_ID
               value: {{.Values.kafka.emailGroup}}
+            {{- if .Values.kafka.security }}
+            - name: KAFKA_SECURITY
+              value: {{.Values.kafka.security}}
+            {{- end }}              
             - name: EMAIL_SUBJECT
               valueFrom:
                 configMapKeyRef:

kubernetes/email-service/values.yaml

@@ -12,7 +12,8 @@ image:
 kafka:
   emailTopic: hmda-email
   emailGroup: email-consumer 
-
+  security: ""
+  
 resources:
   limits:
     memory: "2200Mi"

kubernetes/hmda-analytics/templates/deployment.yaml

@@ -49,6 +49,10 @@ spec:
               value: {{.Values.kafka.signTopic}}
             - name: KAFKA_ANALYTICS_GROUP
               value: {{.Values.kafka.analyticsGroup}}
+            {{- if .Values.kafka.security }}
+            - name: KAFKA_SECURITY
+              value: {{.Values.kafka.security}}
+            {{- end }}              
             - name: PG_HOST
               valueFrom:
                 secretKeyRef:

kubernetes/hmda-analytics/values.yaml

@@ -16,7 +16,8 @@ kafka:
   analyticsGroup: analytics-group
   analyticsTopic: hmda-analytics
   signTopic: hmda-sign
-
+  security: ""
+  
 ingress:
   enabled: false
   annotations: {}

kubernetes/hmda-platform/templates/deployment.yaml

@@ -132,6 +132,10 @@ spec:
             configMapKeyRef:
               name: timed-guards
               key: actionQ3End
+        {{- if .Values.kafka.security }}
+        - name: KAFKA_SECURITY
+          value: {{.Values.kafka.security}}
+        {{- end }}
         - name: KAFKA_INSTITUTIONS_TOPIC
           value: {{.Values.kafka.institutionsTopic}}
         - name: KAFKA_SIGN_TOPIC