Merge pull request #559 from CMSgov/task-dynamodb-test

Limit DynamoDb retrieve file list

Author: sfradkin

rest-api/src/main/java/gov/cms/qpp/conversion/api/services/DbServiceImpl.java

@@ -27,6 +27,7 @@ public class DbServiceImpl extends AnyOrderActionService<Metadata, Metadata>
 		implements DbService {
 
 	private static final Logger API_LOG = LoggerFactory.getLogger(DbServiceImpl.class);
+	private static final int LIMIT = 3;
 
 	@Inject
 	private DynamoDBMapper mapper;
@@ -59,7 +60,9 @@ public CompletableFuture<Metadata> write(Metadata meta) {
 	}
 
 	/**
-	 * Scans the DynamoDB table for unprocessed {@link Metadata}
+	 * Queries the DynamoDB GSI for unprocessed {@link Metadata} with a maximum of 96 items.
+	 *
+	 * Iterates over all of the different partitions, returning a maximum of three items from each.
 	 *
 	 * @return {@link List} of unprocessed {@link Metadata}
 	 */
@@ -75,9 +78,10 @@ public List<Metadata> getUnprocessedCpcPlusMetaData() {
 				.withKeyConditionExpression(Constants.DYNAMO_CPC_ATTRIBUTE + " = :cpcValue and begins_with(" +
 					Constants.DYNAMO_CPC_PROCESSED_CREATE_DATE_ATTRIBUTE + ", :cpcProcessedValue)")
 				.withExpressionAttributeValues(valueMap)
-				.withConsistentRead(false);
+				.withConsistentRead(false)
+				.withLimit(LIMIT);
 
-			return mapper.query(Metadata.class, metadataQuery).stream();
+			return mapper.queryPage(Metadata.class, metadataQuery).getResults().stream();
 		}).flatMap(Function.identity()).collect(Collectors.toList());
 	}
 
@@ -88,6 +92,7 @@ public List<Metadata> getUnprocessedCpcPlusMetaData() {
 	 * @return Metadata found
 	 */
 	public Metadata getMetadataById(String uuid) {
+		API_LOG.info("Read item {} from DynamoDB", uuid);
 		return mapper.load(Metadata.class, uuid);
 	}
 

rest-api/src/test/java/gov/cms/qpp/conversion/api/services/DbServiceImplTest.java

@@ -1,5 +1,24 @@
 package gov.cms.qpp.conversion.api.services;
 
+import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBMapper;
+import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBQueryExpression;
+import com.amazonaws.services.dynamodbv2.datamodeling.QueryResultPage;
+import gov.cms.qpp.conversion.api.model.Constants;
+import gov.cms.qpp.conversion.api.model.Metadata;
+import gov.cms.qpp.test.MockitoExtension;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.springframework.core.env.Environment;
+import org.springframework.core.task.TaskExecutor;
+
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
 import static com.google.common.truth.Truth.assertThat;
 import static com.google.common.truth.Truth.assertWithMessage;
 import static org.mockito.ArgumentMatchers.any;
@@ -12,26 +31,6 @@
 import static org.mockito.Mockito.verifyZeroInteractions;
 import static org.mockito.Mockito.when;
 
-import java.util.List;
-import java.util.concurrent.CompletableFuture;
-import java.util.stream.Stream;
-
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.springframework.core.env.Environment;
-import org.springframework.core.task.TaskExecutor;
-
-import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBMapper;
-import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBQueryExpression;
-import com.amazonaws.services.dynamodbv2.datamodeling.PaginatedQueryList;
-
-import gov.cms.qpp.conversion.api.model.Constants;
-import gov.cms.qpp.conversion.api.model.Metadata;
-import gov.cms.qpp.test.MockitoExtension;
-
 @ExtendWith(MockitoExtension.class)
 class DbServiceImplTest {
 
@@ -92,14 +91,16 @@ void testNoWriteBecauseNoAudit() {
 
 	@Test
 	void testGetUnprocessedCpcPlusMetaData() {
-		PaginatedQueryList<Metadata> mockMetadataList = mock(PaginatedQueryList.class);
-		when(mockMetadataList.stream()).thenAnswer(invocationOnMock -> Stream.of(new Metadata(), new Metadata()));
-		when(dbMapper.query(eq(Metadata.class), any(DynamoDBQueryExpression.class))).thenReturn(mockMetadataList);
+		int itemsPerPartition = 2;
+
+		QueryResultPage<Metadata> mockMetadataPage = mock(QueryResultPage.class);
+		when(mockMetadataPage.getResults()).thenReturn(Stream.generate(Metadata::new).limit(itemsPerPartition).collect(Collectors.toList()));
+		when(dbMapper.queryPage(eq(Metadata.class), any(DynamoDBQueryExpression.class))).thenReturn(mockMetadataPage);
 
 		List<Metadata> metaDataList = underTest.getUnprocessedCpcPlusMetaData();
 
-		verify(dbMapper, times(Constants.CPC_DYNAMO_PARTITIONS)).query(any(Class.class), any(DynamoDBQueryExpression.class));
-		assertThat(metaDataList).hasSize(2 * Constants.CPC_DYNAMO_PARTITIONS);
+		verify(dbMapper, times(Constants.CPC_DYNAMO_PARTITIONS)).queryPage(any(Class.class), any(DynamoDBQueryExpression.class));
+		assertThat(metaDataList).hasSize(itemsPerPartition * Constants.CPC_DYNAMO_PARTITIONS);
 	}
 
 	@Test