Merge pull request #1120 from CMSgov/feature/QPPSF-7911_Slow-dynamo

QPPSF-7911: Dynamo Updates

Author: saquino0827

commons/src/main/resources/measures-data.json

@@ -5442,8 +5442,8 @@
         "eMeasureUuids": {
           "initialPopulationUuid": "3E177A28-30DE-4381-8C7A-B42CD914DA33",
           "denominatorUuid": "E7E1C3BB-730F-4020-9BF3-3AA508B914FC",
-          "numeratorUuid": "827502FD-FF19-45FB-974B-B7F32B281873",
-          "denominatorExclusionUuid": "14F9E2DB-C09F-4E06-AFBD-C1E5E178D920"
+          "numeratorUuid": "14F9E2DB-C09F-4E06-AFBD-C1E5E178D920",
+          "denominatorExclusionUuid": "827502FD-FF19-45FB-974B-B7F32B281873"
         }
       }
     ],

rest-api/src/main/java/gov/cms/qpp/conversion/api/services/internal/DbServiceImpl.java

@@ -86,18 +86,17 @@ public List<Metadata> getUnprocessedCpcPlusMetaData(String orgAttribute) {
 				Map<String, AttributeValue> valueMap = new HashMap<>();
 				valueMap.put(":cpcValue", new AttributeValue().withS(Constants.CPC_DYNAMO_PARTITION_START + partition));
 				valueMap.put(":cpcProcessedValue", new AttributeValue().withS("false#"+year));
-				valueMap.put(":createDate", new AttributeValue().withS(cpcConversionStartDate));
 
 				DynamoDBQueryExpression<Metadata> metadataQuery = new DynamoDBQueryExpression<Metadata>()
 					.withIndexName(indexName)
 					.withKeyConditionExpression(Constants.DYNAMO_CPC_ATTRIBUTE + " = :cpcValue and begins_with("
 							+ orgAttribute + ", :cpcProcessedValue)")
-					.withFilterExpression(Constants.DYNAMO_CREATE_DATE_ATTRIBUTE + " > :createDate")
 					.withExpressionAttributeValues(valueMap)
 					.withConsistentRead(false);
 
-				return mapper.get().query(Metadata.class, metadataQuery).stream();
-			}).flatMap(Function.identity()).collect(Collectors.toList());
+				return mapper.get().query(Metadata.class, metadataQuery).stream().limit(10);
+			})
+				.flatMap(Function.identity()).collect(Collectors.toList());
 		} else {
 			API_LOG.warn("Could not get unprocessed CPC+ metadata because the dynamodb mapper is absent");
 			return Collections.emptyList();
@@ -123,17 +122,15 @@ public List<Metadata> getUnprocessedPcfMetaData(String orgAttribute) {
 				Map<String, AttributeValue> valueMap = new HashMap<>();
 				valueMap.put(":pcfValue", new AttributeValue().withS(Constants.PCF_DYNAMO_PARTITION_START + partition));
 				valueMap.put(":pcfProcessedValue", new AttributeValue().withS("false#"+year));
-				valueMap.put(":createDate", new AttributeValue().withS(cpcConversionStartDate));
 
 				DynamoDBQueryExpression<Metadata> metadataQuery = new DynamoDBQueryExpression<Metadata>()
 					.withIndexName(indexName)
 					.withKeyConditionExpression(Constants.DYNAMO_PCF_ATTRIBUTE + " = :pcfValue and begins_with("
 						+ orgAttribute + ", :pcfProcessedValue)")
-					.withFilterExpression(Constants.DYNAMO_CREATE_DATE_ATTRIBUTE + " > :createDate")
 					.withExpressionAttributeValues(valueMap)
 					.withConsistentRead(false);
 
-				return mapper.get().query(Metadata.class, metadataQuery).stream();
+				return mapper.get().query(Metadata.class, metadataQuery).stream().limit(10);
 			}).flatMap(Function.identity()).collect(Collectors.toList());
 		} else {
 			API_LOG.warn("Could not get unprocessed PCF metadata because the dynamodb mapper is absent");

rest-api/src/test/java/gov/cms/qpp/conversion/api/services/internal/DbServiceImplTest.java

@@ -118,27 +118,29 @@ void testNoWriteBecauseNoAudit() {
 	void testGetUnprocessedCpcPlusMetaData() {
 		when(environment.getProperty(Constants.CPC_PLUS_UNPROCESSED_FILE_SEARCH_DATE_VARIABLE)).thenReturn("2020-01-01");
 
-		int itemsPerPartition = 2;
+		int itemsPerPartition = 11;
+		int itemLimitPerpartition = 10;
 
 		PaginatedQueryList<Metadata> mockMetadataPage = mock(PaginatedQueryList.class);
 		Answer<Stream<Metadata>> answer = (InvocationOnMock invocation) -> Stream.generate(Metadata::new).limit(itemsPerPartition);
-
+		
 		when(mockMetadataPage.stream()).thenAnswer(answer);
 		when(dbMapper.query(eq(Metadata.class), any(DynamoDBQueryExpression.class)))
 			.thenReturn(mockMetadataPage);
 
 		List<Metadata> metaDataList = underTest.getUnprocessedCpcPlusMetaData(Constants.CPC_ORG);
 
 		verify(dbMapper, times(Constants.CPC_DYNAMO_PARTITIONS)).query(eq(Metadata.class), any(DynamoDBQueryExpression.class));
-		assertThat(metaDataList).hasSize(itemsPerPartition * Constants.CPC_DYNAMO_PARTITIONS);
+		assertThat(metaDataList).hasSize(itemLimitPerpartition * Constants.CPC_DYNAMO_PARTITIONS);
 	}
 
 	@Test
 	@SuppressWarnings("unchecked")
 	void testGetUnprocessedPcfMetaData() {
 		when(environment.getProperty(Constants.CPC_PLUS_UNPROCESSED_FILE_SEARCH_DATE_VARIABLE)).thenReturn("2020-01-01");
 
-		int itemsPerPartition = 2;
+		int itemsPerPartition = 11;
+		int itemLimitPerpartition = 10;
 
 		PaginatedQueryList<Metadata> mockMetadataPage = mock(PaginatedQueryList.class);
 		Answer<Stream<Metadata>> answer = (InvocationOnMock invocation) -> Stream.generate(Metadata::new).limit(itemsPerPartition);
@@ -150,7 +152,7 @@ void testGetUnprocessedPcfMetaData() {
 		List<Metadata> metaDataList = underTest.getUnprocessedPcfMetaData(Constants.CPC_ORG);
 
 		verify(dbMapper, times(Constants.CPC_DYNAMO_PARTITIONS)).query(eq(Metadata.class), any(DynamoDBQueryExpression.class));
-		assertThat(metaDataList).hasSize(itemsPerPartition * Constants.CPC_DYNAMO_PARTITIONS);
+		assertThat(metaDataList).hasSize(itemLimitPerpartition * Constants.CPC_DYNAMO_PARTITIONS);
 	}
 
 	@Test