BXC-4569 - Use docker for Fedora in tests (#1729)

* Add docker compose to bring up a test instance of fcrepo docker to use in integration tests (which uses auth be default). Update StorageLocationTestHelper to work with a shared path for storing binaries with fedora also allows as a external binary source. Update BinaryTransferServiceImplIT to work with this setup

* Bring up fedora in gh actions. Only running persistence tests for now

* Address config via mounted volume

* Enable logging for fedora

* Debugging config

* Use env variable

* Reenable tests and cleanup

* Check on bxc storage

* Cleanup build.yml

* Sort IDs as part of formatting member order validation messages, to resolve flapping tests which fail if the HashSet happens to return a different order

* Update readme with instructions for running integration tests

* Update all of the operations integration tests to use external fedora

* Update web-access IT, fix some incorrect assumptions and a file path issue that caused tests to flap

* Cleanup dependencies

* Updated tests in services-api app.
Running redis in docker now instead of using embedded.
Cleanup fedora and redis after tests.
Switched to using MvcTestHelpers for some methods that were being provided by AbstractAPIIT

* Add servlet-api for tests

* Change servlet-api to being both provided and test scope. Remove unneeded CommonsMultipartResolver which was requiring web dependencies in the operations module

* More fixes to servlet-api dependency. Fixed includes that were using the incorrect dependency as their source, since they failed after removing fedora dependencies

* Add helper methods for getting base location for a storage area (or generate a storage uri in the default storage area), to fix issue of tests failing when files were being written to the storage root. Get web-access-app tests to work in ghactions

* Upgrade jedis to version that supports redis 5 and 7

* Switch to using JedisPooled for managing resources. Set redis at version 7.

* More jedis fixes, and update jesque

* Utility for cleaning up repo. Updating deposit tests

* Revert "Switch to using JedisPooled for managing resources. Set redis at version 7."

This reverts commit 41385d4.

* Downgrade to jedis 3.10 since jesque does not support 4. Switch to redis 6 since that is the highest version supported by jedis 3

* Update services camel app tests

* Try using fuseki as a container

* Add fuseki to gh actions

* Update integration test module. Remove fedora server dependency everywhere

* Increase message count

* Cleanup files from the shared storage location

* Fix visibility

* Fix test that was failing due to timezones

Author: bbpennel

.github/workflows/build.yml

@@ -15,19 +15,46 @@ jobs:
 
     services:
       redis:
-        image: redis
+        image: redis:6
         ports:
-          - 6379/tcp
+          - 46380:6379
+      fedora:
+        image: fcrepo/fcrepo:5.1.1
+        ports:
+          - 48085:8080
+        volumes:
+          - ${{ github.workspace }}/it_config:/config
+          - /tmp/boxc_test_storage:/tmp/boxc_test_storage
+        env:
+          CATALINA_OPTS: "-Djava.awt.headless=true -server -Xms512m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -Dfcrepo.external.content.allowed=/config/allowed_external_paths.txt"
 
     steps:
+    - name: Change permissions of /tmp/boxc_test_storage
+      run: sudo chmod 2777 /tmp/boxc_test_storage
+
     - name: Checkout repository
       uses: actions/checkout@v2
+
+    - name: Get Container ID of fedora service
+      id: get_container_id
+      run: |
+        CONTAINER_ID=$(docker ps --format "{{.ID}}" -f "ancestor=fcrepo/fcrepo:5.1.1")
+        echo "CONTAINER_ID=$CONTAINER_ID" >> $GITHUB_ENV
+    - name: Restart fedora now that the repository has been checked out so config files are available
+      run: docker restart ${{ env.CONTAINER_ID }}
+
+    - name: Run Fuseki container as command to pass in extra dataset name argument
+      run: |
+        docker run -d --rm -p 43030:3030 atomgraph/fuseki --mem /test
+
     - name: Checkout submodules
       run: git submodule update --init --recursive
+
     - name: Set up JDK 11
       uses: actions/setup-java@v1
       with:
         java-version: 11
+
     - name: Cache Maven packages
       uses: actions/cache@v2
       with:
@@ -36,9 +63,6 @@ jobs:
         restore-keys: ${{ runner.os }}-m2
     - name: Build with Maven
       run: mvn -B -U -pl '!integration' clean install -DskipTests
-    - name: Package integration module with Maven
-      run: mvn -B -U -pl integration clean package -DskipTests
-
     - name: Set up nodejs
       uses: actions/setup-node@v2
       with:
@@ -69,3 +93,8 @@ jobs:
           ${{github.workspace}}/**/target/site/jacoco-it/jacoco.xml:jacoco
           ${{github.workspace}}/static/js/vue-cdr-access/coverage/lcov.info:lcov
           ${{github.workspace}}/static/js/admin/vue-permissions-editor/coverage/lcov.info:lcov
+
+    - name: View fedora service logs
+      if: always()
+      run: |
+        docker logs ${{ env.CONTAINER_ID }}

.gitignore

@@ -41,4 +41,5 @@ services-worker
 .sass-cache
 *.css.map
 dependency-reduced-pom.xml
-etc/solr-config/access/data/
+etc/solr-config/access/data/
+it_config/logs/*

README.md

@@ -61,12 +61,25 @@ All tests run automatically in Github Actions.
 All Java tests run automatically when building the project, unless skipped.
 JavaScript test don't run on a maven build, but can be run manually using the NPM command below.
 
+### Running java unit tests
 ```
 # Java Unit Tests (skipping tests from external modules)
 mvn -pl '!clamav-java' test 
+```
+
+### Running java unit and integration tests
+To run the integration tests locally, you will first need to start external dependencies such as Fedora. Do to this, from the root directory of this project, run the following command:
+```
+docker-compose up
+```
+And then run the integration tests from within your IDE or via the following:
+```
 # Java unit and integration tests
 mvn -pl '!clamav-java' verify 
+```
 
+### Running JavaScript tests
+```
 # JavaScript Tests
 npm --prefix static/js/admin/vue-permissions-editor run test
 npm --prefix static/js/vue-cdr-access run test

deposit-app/pom.xml

@@ -163,59 +163,6 @@
             <groupId>gov.loc</groupId>
             <artifactId>bagit</artifactId>
         </dependency>
-        <!-- integration testing -->
-        <dependency>
-            <groupId>org.fcrepo</groupId>
-            <artifactId>fcrepo-http-commons</artifactId>
-            <scope>test</scope>
-            <type>test-jar</type>
-        </dependency>
-        <dependency>
-            <groupId>org.fcrepo</groupId>
-            <artifactId>fcrepo-auth-common</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.glassfish.grizzly</groupId>
-            <artifactId>grizzly-http-servlet</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.glassfish.jersey.core</groupId>
-            <artifactId>jersey-common</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.glassfish.jersey.ext</groupId>
-            <artifactId>jersey-spring5</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.glassfish.jersey.test-framework.providers</groupId>
-            <artifactId>jersey-test-framework-provider-grizzly2</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.fcrepo</groupId>
-            <artifactId>fcrepo-kernel-modeshape</artifactId>
-            <scope>test</scope>
-            <classifier>tests</classifier>
-        </dependency>
-        <dependency>
-            <groupId>org.fcrepo</groupId>
-            <artifactId>fcrepo-http-api</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>javax.xml.bind</groupId>
-            <artifactId>jaxb-api</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>com.sun.xml.bind</groupId>
-            <artifactId>jaxb-core</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>com.sun.xml.bind</groupId>
-            <artifactId>jaxb-impl</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>it.ozimov</groupId>
-            <artifactId>embedded-redis</artifactId>
-        </dependency>
     </dependencies>
 
     <build>

deposit-app/src/test/java/edu/unc/lib/boxc/deposit/fcrepo4/AbstractFedoraDepositJobIT.java

@@ -1,26 +1,5 @@
 package edu.unc.lib.boxc.deposit.fcrepo4;
 
-import java.io.File;
-import java.io.IOException;
-import java.net.URI;
-import java.nio.file.Path;
-import java.util.List;
-import java.util.UUID;
-
-import org.apache.http.HttpStatus;
-import org.fcrepo.client.FcrepoClient;
-import org.fcrepo.client.FcrepoOperationFailedException;
-import org.fcrepo.client.FcrepoResponse;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.junit.jupiter.api.io.TempDir;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.test.context.ContextConfiguration;
-import org.springframework.test.context.ContextHierarchy;
-import org.springframework.test.context.junit.jupiter.SpringExtension;
-
 import edu.unc.lib.boxc.deposit.api.RedisWorkerConstants.DepositState;
 import edu.unc.lib.boxc.deposit.impl.model.ActivityMetricsClient;
 import edu.unc.lib.boxc.deposit.impl.model.DepositModelManager;
@@ -35,12 +14,30 @@
 import edu.unc.lib.boxc.model.fcrepo.ids.RepositoryPaths;
 import edu.unc.lib.boxc.model.fcrepo.services.RepositoryInitializer;
 import edu.unc.lib.boxc.model.fcrepo.test.TestHelper;
+import edu.unc.lib.boxc.model.fcrepo.test.TestRepositoryDeinitializer;
 import edu.unc.lib.boxc.operations.api.events.PremisLoggerFactory;
 import edu.unc.lib.boxc.persist.api.storage.StorageLocationManager;
 import edu.unc.lib.boxc.persist.api.transfer.BinaryTransferService;
+import edu.unc.lib.boxc.persist.impl.storage.StorageLocationTestHelper;
+import org.apache.http.HttpStatus;
+import org.fcrepo.client.FcrepoClient;
+import org.fcrepo.client.FcrepoOperationFailedException;
+import org.fcrepo.client.FcrepoResponse;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.io.TempDir;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.ContextHierarchy;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
 import redis.clients.jedis.Jedis;
 import redis.clients.jedis.JedisPool;
-import redis.embedded.RedisServer;
+
+import java.io.File;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.UUID;
 
 /**
  *
@@ -49,7 +46,6 @@
  */
 @ExtendWith(SpringExtension.class)
 @ContextHierarchy({
-    @ContextConfiguration("/spring-test/test-fedora-container.xml"),
     @ContextConfiguration("/spring-test/cdr-client-container.xml")
 })
 public abstract class AbstractFedoraDepositJobIT {
@@ -76,6 +72,10 @@ public abstract class AbstractFedoraDepositJobIT {
     protected BinaryTransferService binaryTransferService;
     @Autowired
     protected StorageLocationManager storageLocationManager;
+    @Autowired
+    protected StorageLocationTestHelper storageLocationTestHelper;
+    @Autowired
+    protected FcrepoClient fcrepoClient;
     @TempDir
     public Path tmpFolder;
     @Autowired
@@ -93,23 +93,6 @@ public abstract class AbstractFedoraDepositJobIT {
 
     protected ContentRootObject rootObj;
 
-    private static final RedisServer redisServer;
-
-    static {
-        try {
-            redisServer = new RedisServer(46380);
-            redisServer.start();
-        } catch (Exception e) {
-            throw new RuntimeException(e);
-        }
-
-    }
-
-    @AfterAll
-    public static void afterAll() throws Exception {
-        redisServer.stop();
-    }
-
     @BeforeEach
     public void initBase() throws Exception {
         TestHelper.setContentBase(baseAddress);
@@ -130,24 +113,12 @@ public void initBase() throws Exception {
     }
 
     @AfterEach
-    public void cleanupRedis() throws Exception {
+    public void cleanupDatastores() throws Exception {
         try (Jedis jedis = jedisPool.getResource()) {
             jedis.flushDB();
         }
-    }
-
-    protected URI createBaseContainer(String name) throws IOException, FcrepoOperationFailedException {
-        URI baseUri = URI.create(serverAddress + name);
-        // Create a parent object to put the binary into
-        try (FcrepoResponse response = client.put(baseUri).perform()) {
-            return response.getLocation();
-        } catch (FcrepoOperationFailedException e) {
-            // Eat conflict exceptions since this will run multiple times
-            if (e.getStatusCode() != HttpStatus.SC_CONFLICT) {
-                throw e;
-            }
-            return baseUri;
-        }
+        TestRepositoryDeinitializer.cleanup(fcrepoClient);
+        storageLocationTestHelper.cleanupStorageLocations();
     }
 
     protected ContentObject findContentObjectByPid(List<ContentObject> objs, final PID pid) {

deposit-app/src/test/java/edu/unc/lib/boxc/deposit/fcrepo4/IngestContentObjectsJobIT.java

@@ -44,6 +44,7 @@
 import org.jdom2.output.XMLOutputter;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -96,10 +97,6 @@ public class IngestContentObjectsJobIT extends AbstractFedoraDepositJobIT {
 
     private static final Logger log = getLogger(IngestContentObjectsJobIT.class);
 
-    static {
-        System.setProperty("fcrepo.properties.management", "relaxed");
-    }
-
     private static final String INGESTOR_PRINC = "ingestor";
     private static final String DEPOSITOR_NAME = "boxy_depositor";
 
@@ -460,10 +457,12 @@ public void ingestWorkObjectWithModsHistoryTest() throws Exception {
         Path modsPath = job.getModsPath(workPid, true);
         var originalModsPath = Path.of("src/test/resources/simpleMods.xml");
         Files.copy(originalModsPath, modsPath);
-        Path modsHistoryPath = depositDirManager.getHistoryFile(workPid, DatastreamType.MD_DESCRIPTIVE_HISTORY);
+        var modsPid = DatastreamPids.getMdDescriptivePid(workPid);
+        var modsHistoryStorageUri = storageLocationTestHelper.makeTestStorageUri(DatastreamPids.getDatastreamHistoryPid(modsPid));
+        Path modsHistoryPath = Paths.get(modsHistoryStorageUri);
         FileUtils.writeStringToFile(modsHistoryPath.toFile(), "History content", UTF_8);
         String modsHistorySha1 = getSha1(modsHistoryPath);
-        historyResc.addProperty(CdrDeposit.storageUri, modsHistoryPath.toUri().toString());
+        historyResc.addProperty(CdrDeposit.storageUri, modsHistoryStorageUri.toString());
         historyResc.addLiteral(CdrDeposit.sha1sum, modsHistorySha1);
 
         job.closeModel();
@@ -508,7 +507,8 @@ public void ingestWorkObjectWithFITSHistoryTest() throws Exception {
         PID fitsPid = DatastreamPids.getTechnicalMetadataPid(mainPid);
         PID historyPid = DatastreamPids.getDatastreamHistoryPid(fitsPid);
         Resource historyResc = DepositModelHelpers.addDatastream(mainResc, DatastreamType.TECHNICAL_METADATA_HISTORY);
-        Path historyPath = depositDirManager.getHistoryFile(fitsPid, DatastreamType.TECHNICAL_METADATA_HISTORY);
+        var historyUri = storageLocationTestHelper.makeTestStorageUri(historyPid);
+        Path historyPath = Path.of(historyUri);
         FileUtils.writeStringToFile(historyPath.toFile(), "History content", UTF_8);
         String historySha1 = getSha1(historyPath);
         historyResc.addProperty(CdrDeposit.storageUri, historyPath.toUri().toString());
@@ -714,6 +714,7 @@ public void resumeIngestFolderTest() throws Exception {
         model = job.getWritableModel();
         Resource file2Resc = model.getResource(file2Pid.getRepositoryPath());
         Resource orig2Resc = DepositModelHelpers.getDatastream(file2Resc);
+        setupStorageUriForResource(FILE2_LOC, orig2Resc, file2Resc, file2Pid);
         orig2Resc.addProperty(CdrDeposit.storageUri, Paths.get(depositDir.getAbsolutePath(),
                 FILE2_LOC).toUri().toString());
         job.closeModel();
@@ -1100,6 +1101,7 @@ public void fromMultipleDepositsTest() throws Exception {
     private final static Date CREATED_DATE = DateTimeUtil.parseUTCToDate(CREATED_STRING);
     private final static Date LAST_MODIFIED_DATE = DateTimeUtil.parseUTCToDate(LAST_MODIFIED_STRING);
 
+    @Disabled("This test only works if fedora is in 'relaxed' mode, which is not the case on our servers or in docker")
     @Test
     public void overrideTimestampsTest() throws Exception {
         Map<String, String> status = new HashMap<>();
@@ -1271,9 +1273,7 @@ private PID addFileObject(Bag parent, String stagingLocation,
 
         Resource origResc = DepositModelHelpers.addDatastream(fileResc, ORIGINAL_FILE);
         if (stagingLocation != null) {
-            origResc.addProperty(CdrDeposit.storageUri, Paths.get(depositDir.getAbsolutePath(),
-                    stagingLocation).toUri().toString());
-            fileResc.addLiteral(Cdr.storageLocation, LOC1_ID);
+            setupStorageUriForResource(stagingLocation, origResc, fileResc, filePid);
         }
         origResc.addProperty(CdrDeposit.mimetype, mimetype);
         if (sha1 != null) {
@@ -1286,15 +1286,26 @@ private PID addFileObject(Bag parent, String stagingLocation,
         parent.add(fileResc);
 
         // Create the accompanying fake premis report file
-        Path fitsPath = job.getTechMdPath(filePid, true);
+        var fitsStorageUri = storageLocationTestHelper.makeTestStorageUri(DatastreamPids.getTechnicalMetadataPid(filePid));
+        Path fitsPath = Path.of(fitsStorageUri);
         Files.createFile(fitsPath);
         Resource fitsResc = DepositModelHelpers.addDatastream(fileResc, TECHNICAL_METADATA);
-        fitsResc.addProperty(CdrDeposit.storageUri, fitsPath.toUri().toString());
+        fitsResc.addProperty(CdrDeposit.storageUri, fitsStorageUri.toString());
         fitsResc.addLiteral(CdrDeposit.sha1sum, getSha1(fitsPath));
 
         return filePid;
     }
 
+    private void setupStorageUriForResource(String stagingLocation, Resource origResc, Resource fileResc, PID filePid) throws Exception {
+        var fileStagingPath = Paths.get(depositDir.getAbsolutePath(), stagingLocation);
+        var storageUri = storageLocationTestHelper.makeTestStorageUri(DatastreamPids.getOriginalFilePid(filePid));
+        var storagePath = Path.of(storageUri);
+        Files.copy(fileStagingPath, storagePath);
+        origResc.addProperty(CdrDeposit.storageUri, storageUri.toString());
+        fileResc.addLiteral(Cdr.storageLocation, LOC1_ID);
+        fileResc.addLiteral(CdrDeposit.label, stagingLocation);
+    }
+
     private List<PID> addWorkWithFileObject(Bag parent, String stagingLocation,
             String mimetype, String sha1, String md5) throws Exception {
         Model model = parent.getModel();