Merge branch 'master' into post-filing-2025

Author: Aldrian Harjati

.github/workflows/README.md

@@ -0,0 +1,22 @@
+## Releasing Image to ECR
+
+> **_NOTE:_**  This document contains info on how to publish the `hmda-platform` image to ECR.
+> This job, located in `ecr-push.yaml`, can **only** be run locally on your machine.
+
+### Prerequisites 
+
+- `act` - version >=0.2.71
+  - https://nektosact.com/installation/index.html
+- Logged in and authenticated to AWS
+
+### Copy Job
+At the root of the repo, run:
+
+```shell
+act -j 'release_to_ecr' --env=IMAGE_TAG=v0.1.2 --env=AWS_URL=aws.com --env-file <(aws configure export-credentials --format env)
+```
+Make sure to set:
+- `IMAGE_TAG` 
+  - The tag of what image you want to copy from Dockerhub to ECR
+- `AWS_URL`
+  - The URL to our AWS instance

.github/workflows/cve-scan-pr.yml

@@ -39,13 +39,4 @@ jobs:
         with:
           name: cve-report
           path: |
-            grype-report.txt
-
-      - name: Post comment with report link
-        uses: thollander/actions-comment-pull-request@v3
-        with:
-          message: CVE scan report generated by Grype are available. Check the Actions tab to download the reports.
-
-      - name: Remove Docker image
-        run: |
-          docker rmi pr-cve-scan:latest
+            grype-report.txt

.github/workflows/dockerhub-push.yml

@@ -2,8 +2,11 @@ name: HMDA Docker Hub Image Push
 
 on:
   push:
-    branches:
-      - master
+    tags:
+      - '*'           # Push events to every tag not containing /
+
+env:
+  REGISTRY: hmda/hmda-platform
 
 jobs:
   push_to_dockerhub:
@@ -14,39 +17,21 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v4
 
-      - name: Log in to Docker Hub
-        uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+      - name: Setup sbt launcher
+        uses: sbt/setup-sbt@v1
 
       - name: Build image of HMDA Platform only
         run: |
-          sbt -batch clean hmda-platform/docker:publishLocal
-        continue-on-error: true
+          sbt "project hmda-platform" dockerPublishLocalSkipTests
 
       - name: Tag Docker image
-        run: docker tag $(docker images --filter=reference="hmda/hmda-platform:latest" --format "{{.ID}}") ${{ secrets.DOCKERHUB_USERNAME }}/hmda:latest
-
-      - name: Push image to Docker Hub
-        run: docker push ${{ secrets.DOCKERHUB_USERNAME }}/hmda:latest
+        run: docker tag $(docker images --filter=reference="hmda/hmda-platform:latest" --format "{{.ID}}") ${{ env.REGISTRY }}:${{ github.ref_name }}
 
-      - name: Run Docker Scout CVE scan
-        if: ${{ github.event_name != 'pull_request_target' }}
-        uses: docker/scout-action@v1
-        with:
-          command: cves
-          image: ${{ secrets.DOCKERHUB_USERNAME }}/hmda:latest
-          sarif-file: sarif.output.json
-          summary: true
-
-      - name: Upload CVE scan to artifact
-        if: ${{ github.event_name != 'pull_request_target' }}
-        uses: github/codeql-action/upload-sarif@v2
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
         with:
-          sarif_file: sarif.output.json
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - name: Post comment with report link
-        uses: thollander/actions-comment-pull-request@v3
-        with:
-          message: CVE scan report generated by Docker Scout are available. Check the Actions tab to download the report.
+      - name: Push image to Docker Hub
+        run: docker push ${{ env.REGISTRY }}:${{ github.ref_name }}

.github/workflows/ecr-push.yaml

@@ -0,0 +1,30 @@
+name: HMDA ECR Image Push
+
+on: workflow_dispatch
+
+jobs:
+  release_to_ecr:
+      name: Copy Docker image from Dockerhub to ECR
+      runs-on: ubuntu-latest
+      steps:
+        - name: Configure AWS credentials
+          uses: aws-actions/configure-aws-credentials@v4
+          with:
+            aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }}
+            aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }}
+            aws-region: us-east-1
+
+        - name: Login to Amazon ECR
+          id: login-ecr
+          uses: aws-actions/amazon-ecr-login@v2
+
+        - name: Copy Image to ECR
+          run: |
+            docker pull hmda/hmda-platform:${{ env.IMAGE_TAG }}
+            docker tag hmda/hmda-platform:${{ env.IMAGE_TAG }} ${{ env.AWS_URL }}/hmda/hmda-platform:${{ env.IMAGE_TAG }}
+            docker push ${{ env.AWS_URL }}/hmda/hmda-platform:${{ env.IMAGE_TAG }}
+
+        - name: Cleanup images
+          run: |
+            docker rmi hmda/hmda-platform:${{ env.IMAGE_TAG }}
+            docker rmi ${{ env.AWS_URL }}/hmda/hmda-platform:${{ env.IMAGE_TAG }}

.gitignore

@@ -85,6 +85,18 @@ src_managed/
 project/boot/
 project/plugins/project/
 
+## sbt project specifics
+check-digit/project/
+data-browser/project/
+email-service/project/
+file-proxy/project/
+hmda-reporting/project/
+institutions-api/project/
+irs-publisher/project/
+modified-lar/project/
+rate-limit/project/
+ratespread-calculator/project/
+
 ## Lightbend
 .lightbend/
 

README.md

@@ -102,53 +102,116 @@ The image below shows the cloud vendor agnostic technical architecture for the H
 
 ## Installations
 Before running the HMDA Platform, make sure to have the following installed:
-1. Homebrew - https://brew.sh/
-2. Docker - ```bash brew install docker ```
-3. Docker Desktop - https://docs.docker.com/desktop/install/mac-install/
-4. <a href="./docs/JavaInstall.md">Java (version 13.0.2) for MacOS</a>
-5. Scala (version 2.12 for compatibility issues) - ```bash brew install [email protected] ```
-6. sdk - https://sdkman.io/install
-Next, use sdk to install sbt instead of brew (it won't work with brew) (Note: before install, check what version is currently being used in project/build.properties and install that version or higher):
 
-```bash
-sdk install sbt
-```
-Clone the repo and go into the repo directory:
-```bash
-git clone https://github.com/cfpb/hmda-platform.git
-cd hmda-platform
-```
+### MacOS
+1. Install [Homebrew](https://brew.sh/):
+    ```bash
+    /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
+    ```
+2. Install [Docker](https://www.docker.com/):
+    ```bash
+    brew install docker
+    ```
+3. Install one of following docker engine implementations:
+    * [Docker Desktop](https://docs.docker.com/desktop/install/mac-install/)
+    * [Podman](https://podman.io/)
+    * [Colima](https://github.com/abiosoft/colima/)
+    * [Rancher Desktop](https://rancherdesktop.io/)
+4. Go to the following link for instructions to install OpenJDK:
+    * [Java (version 13.0.2) for MacOS](/docs/JavaInstall.md)
+5. Install Scala (version 2.12 for compatibility issues):
+    ```bash 
+    brew install [email protected]
+    ```
+6. Install [sdkman](https://sdkman.io/install):
+    ```bash
+    curl -s "https://get.sdkman.io" | bash
+    ```
+7. Install [sbt](https://www.scala-sbt.org/) using `sdkman` (not `brew`):
+    ```bash
+    sdk install sbt <VERSION>
+    ```
+    * Replace `<VERSION>` with the version used in `project/build.properties` or higher
+
+8. Clone the repo and go into the repo directory:
+    ```bash
+    git clone https://github.com/cfpb/hmda-platform.git
+    cd hmda-platform
+    ```
+
 ### Apple Silicon
 
 The current platform and specifically Cassandra have problems running on "Apple silicon" architecture. If your laptop **About This Mac** information shows an Apple M1 or later chip, this applies to you. This will cause test suites to abort.
 
 The current solution is to install, build and run with an amd64-compatible JDK.
 
-```
-$ brew install asdf
-$ arch -x86_64 asdf plugin-add java https://github.com/halcyon/asdf-java.git
-$ arch -x86_64 asdf install java openjdk-13.0.2
-$ export JAVA_HOME=$HOME/.asdf/installs/java/openjdk-13.0.2
+```bash
+brew install asdf
+arch -x86_64 asdf plugin-add java https://github.com/halcyon/asdf-java.git
+arch -x86_64 asdf install java openjdk-13.0.2
+export JAVA_HOME=$HOME/.asdf/installs/java/openjdk-13.0.2
 ```
 
-## Running with sbt
+## Running Locally
+
+### Running with sbt
 
 The HMDA Platform can run locally using [`sbt`](https://www.scala-sbt.org/) with an [embedded Cassandra](https://doc.akka.io/docs/alpakka-kafka/current/) and [embedded Kafka](https://doc.akka.io/docs/alpakka-kafka/current/). To get started:
 
+1. Export the following environment variables:
+    ```bash
+    export CASSANDRA_CLUSTER_HOSTS=localhost
+    export APP_PORT=2551
+    ```
+2. Open terminal with `hmda-platform` root as the  working directory
+3. Start sbt and run the platform with the following commands:
+    ```bash
+    sbt
+    [...]
+    sbt:hmda-root> project hmda-platform
+    sbt:hmda-platform> reStart
+    ```
+
+### Running with docker compose
+
+The platform and it's dependency services, Kafka, Cassandra and PostgreSQL, can run locally using [Docker Compose](https://docs.docker.com/compose/).
+The entire filing plaform can be spun up using a one line command. Using this locally running instance of Platform One, no authentication is needed. 
+
 ```bash
-cd hmda-platform
-export CASSANDRA_CLUSTER_HOSTS=localhost
-export APP_PORT=2551
-sbt
-[...]
-sbt:hmda-root> project hmda-platform
-sbt:hmda-platform> reStart
+# Bring up every service (e.g., "hmda-platform", "hmda-analytics", "institutions-api")
+docker compose up
+
+# Bring up a single service (e.g., "hmda-platform")
+docker compose up hmda-platform
+```
+
+Additionally, there are several environment varialbes that can be configured/changed. The platform uses sensible defaults for each one. However, if required they can be overridden:
 
+```bash
+CASSANDRA_CLUSTER_HOSTS
+CASSANDRA_CLUSTER_DC
+CASSANDRA_CLUSTER_USERNAME
+CASSANDRA_CLUSTER_PASSWORD
+CASSANDRA_JOURNAL_KEYSPACE
+CASSANDRA_SNAPSHOT_KEYSPACE
+KAFKA_CLUSTER_HOSTS
+APP_PORT
+HMDA_HTTP_PORT
+HMDA_HTTP_ADMIN_PORT
+HMDA_HTTP_PUBLIC_PORT
+MANAGEMENT_PORT
+HMDA_CASSANDRA_LOCAL_PORT
+HMDA_LOCAL_KAFKA_PORT
+HMDA_LOCAL_ZK_PORT
+WS_PORT
 ```
-## Access locally build platform
-[hmda-admin-api](http://localhost:8081)   
-[hmda-filing-api](http://localhost:8080)   
-[hmda-public-api](http://localhost:8082)   
+
+### Access locally built platform APIs
+The following API endpoints are accessible when running the platform locally using either [Running with sbt](#running-with-sbt) or [Running with docker compose](#running-with-docker-compose).
+* [hmda-admin-api](http://localhost:8081)   
+* [hmda-filing-api](http://localhost:8080)   
+* [hmda-public-api](http://localhost:8082)   
+
 ## Build hmda-platform Docker image
 
 Docker Image is build via Docker plugin utilizing [sbt-native-packager](https://sbt-native-packager.readthedocs.io/en/stable/formats/docker.html#docker-plugin)
@@ -182,43 +245,6 @@ kubernetes/hmda-platform
 
 All of the containers built by the HMDA Platform are released publicly via Docker Hub: https://hub.docker.com/u/hmda
 
-## One-line Local Development Environment (No Auth)
-
-The platform and it's dependency services, Kafka, Cassandra and PostgreSQL, can run locally using [Docker Compose](https://docs.docker.com/compose/).
-
-```shell
-# Bring up hmda-platform, hmda-analytics, institutions-api
-docker-compose up
-```
-
-The entire filing plaform can be spun up using a one line command. Using this locally running instance of Platform One, no authentication is needed. 
-
-```shell
-# Bring up the hmda-platform
-docker-compose up hmda-platform
-```
-
-Additionally, there are several environment varialbes that can be configured/changed. The platform uses sensible defaults for each one. However, if required they can be overridden:
-
-```
-CASSANDRA_CLUSTER_HOSTS
-CASSANDRA_CLUSTER_DC
-CASSANDRA_CLUSTER_USERNAME
-CASSANDRA_CLUSTER_PASSWORD
-CASSANDRA_JOURNAL_KEYSPACE
-CASSANDRA_SNAPSHOT_KEYSPACE
-KAFKA_CLUSTER_HOSTS
-APP_PORT
-HMDA_HTTP_PORT
-HMDA_HTTP_ADMIN_PORT
-HMDA_HTTP_PUBLIC_PORT
-MANAGEMENT_PORT
-HMDA_CASSANDRA_LOCAL_PORT
-HMDA_LOCAL_KAFKA_PORT
-HMDA_LOCAL_ZK_PORT
-WS_PORT
-```
-
 ## Automated Testing
 
 The HMDA Platform takes a rigorous automated testing approach. In addtion to Travis and CodeCov, we've prepared a suite of [Newman](https://github.com/cfpb/hmda-platform/tree/master/newman) test scripts that perform end-to-end testing of the APIs on a recurring basis. The testing process for Newman is containerized and runs as a Kubernetes CronJob to act as a monitoring and alerting system. The platform and microservices are also testing for load by using [Locust](https://locust.io/).

build.sbt

@@ -31,6 +31,7 @@ lazy val akkaDeps = Seq(
   akkaTestkitTyped,
   akkaStreamsTestKit,
   akkaCors,
+  mskdriver,
   akkaKafkaStreams,
   embeddedKafka,
   alpakkaS3,
@@ -46,6 +47,7 @@ lazy val akkaPersistenceDeps =
     akkaPersistenceQuery,
     akkaClusterShardingTyped,
     akkaPersistenceCassandra,
+    keyspacedriver,
     cassandraLauncher
   )
 
@@ -136,6 +138,9 @@ lazy val common = (project in file("common"))
       )
     ),
     addCompilerPlugin("com.olegpy" %% "better-monadic-for" % "0.3.1")
+    // addCompilerPlugin("com.olegpy" %% "better-monadic-for" % "0.3.1"),
+    // unmanagedJars in Compile ++= Seq(new java.io.File("/tmp/aws-msk-iam-auth-2.2.0-all.jar")).classpath,
+    // unmanagedJars in Runtime ++= Seq(new java.io.File("/tmp/aws-msk-iam-auth-2.2.0-all.jar")).classpath   
   )
   .enablePlugins(BuildInfoPlugin)
   .settings(
@@ -176,7 +181,7 @@ lazy val `hmda-platform` = (project in file("hmda"))
           val oldStrategy = (assembly / assemblyMergeStrategy).value
           oldStrategy(x)
       },
-      reStart / envVars ++= Map("CASSANDRA_CLUSTER_HOSTS" -> "localhost", "APP_PORT" -> "2551"),
+     reStart / envVars ++= Map("CASSANDRA_CLUSTER_HOSTS" -> "localhost", "APP_PORT" -> "2551"),
     ),
     dockerSettings,
     packageSettings
@@ -816,4 +821,4 @@ lazy val `hmda-quarterly-data-service` = (project in file ("hmda-quarterly-data-
     packageSettings
   )
   .dependsOn(common % "compile->compile;test->test")
-  .dependsOn(`hmda-protocol` % "compile->compile;test->test")
+  .dependsOn(`hmda-protocol` % "compile->compile;test->test")