Skip to content

Commit e0752d6

Browse files
adds redis to Dynamo main
renames packages bumps version to 0.3.0-SNAPSHOT
1 parent b4a795e commit e0752d6

File tree

74 files changed

+277
-113
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

74 files changed

+277
-113
lines changed

README.md

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,10 @@
33
[![Build Status](https://travis-ci.org/ub-leipzig/iiif-dynamo.png?branch=master)](https://travis-ci.org/ub-leipzig/iiif-dynamo)
44

55
An Apache Camel Jetty implementation that queries web annotations with SPARQL and dynamically constructs
6-
IIIF Manifests and Collections.
6+
IIIF Manifests and Collections. It also uses Redis to cache query results to produce instantaneous responses.
77

88
## Configuration
9-
* `application.properties`
9+
* `de.ubleipzig.dynamo.cfg`
1010
* `de.ubleipzig.webanno.Constants`
1111

1212
## Building
@@ -15,6 +15,10 @@ To build run
1515
```bash
1616
./buildtools/src/install/install-jpms.sh
1717
```
18+
## Running Docker Composition
19+
`docker-compose up`
20+
* This starts three containers `redis`, `dynamo` and `dynamo-search` on the `110_default` network (which is where the JDK10
21+
[Trellis deployment](https://github.com/trellis-ldp/trellis-deployment/tree/master/trellis-compose/trellis-app-triplestore/1.10) lives)
1822

1923
## Endpoint
2024
The test query endpoint is exposed at `http://localhost:9095/dynamo`
@@ -23,14 +27,14 @@ The test query endpoint is exposed at `http://localhost:9095/dynamo`
2327
This example requests canvases with resources that have either metadata value `1676` or metadata value `1670`
2428

2529
```bash
26-
$ http://localhost:9095/dynamic?type=meta&v1=1676&v2=1670
30+
$ http://localhost:9095/dynamo?type=meta&v1=1676&v2=1670
2731
```
2832

2933
## Example Endpoint Collection Type Query
3034
This example builds a collection of dynamic manifest identifiers by evaluating all possible metadata query values.
3135

3236
```bash
33-
$ http://workspaces.ub.uni-leipzig.de:9095/dynamo?type=collection
37+
$ http://localhost:9095/dynamo?type=collection
3438
```
3539

3640
## Dependencies

build.gradle

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ plugins {
1818

1919
ext {
2020
/* Dependencies */
21-
camelVersion = '2.20.0'
21+
camelVersion = '2.21.0'
2222
commonsRdfVersion = '0.5.0'
2323
iiifVocabularyVersion = '0.1.1'
2424
jacksonVersion = '2.9.5'
@@ -108,6 +108,17 @@ subprojects {
108108
java = 'SLASHSTAR_STYLE'
109109
}
110110
}
111+
112+
task processConfig(type: Copy) {
113+
from('src/main/cfg') {
114+
include '**/*.cfg'
115+
}
116+
into 'build/cfg/main'
117+
}
118+
119+
classes {
120+
classes.dependsOn processConfig
121+
}
111122
}
112123

113124
configure(rootProject) {
Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
#!/usr/bin/env bash
22
./gradlew --version
3-
./gradlew --stacktrace --warning-mode=all web-anno:build
4-
./gradlew --stacktrace --warning-mode=all iiif.dynamic:build
3+
./gradlew --stacktrace --warning-mode=all webanno:build
4+
./gradlew --stacktrace --warning-mode=all dynamo:build

dynamo/build.gradle

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
apply plugin: 'application'
2+
3+
mainClassName = 'de.ubleipzig.dynamo.camel.Dynamo'
4+
description = 'A Camel service that assembles manifests from RDF queries'
5+
6+
javaModule {
7+
name = 'de.ubleipzig.dynamo'
8+
hacks {
9+
patches('org.apache.camel:camel-core-osgi', 'org.apache.camel:camel-core.')
10+
patches('com.sun.xml.bind:jaxb-impl', 'javax.xml.bind:jaxb-api')
11+
patches('com.sun.xml.bind:jaxb-core', 'javax.xml.bind:jaxb-api')
12+
patches('org.apache.httpcomponents:httpcore', 'org.apache.httpcomponents:httpcore-osgi')
13+
patches('org.apache.httpcomponents:httpcore-nio', 'org.apache.httpcomponents:httpcore-osgi')
14+
patches('org.apache.httpcomponents:httpclient-osgi', 'org.apache.httpcomponents:httpclient')
15+
patches('org.slf4j:jcl.over.slf4j', 'commons-logging:commons-logging')
16+
}
17+
}
18+
19+
dependencies {
20+
patch "org.apache.camel:camel-core-osgi:${camelVersion}"
21+
patch "com.sun.xml.bind:jaxb-core:${jaxbCoreVersion}"
22+
patch "com.sun.xml.bind:jaxb-impl:${jaxbCoreVersion}"
23+
patch "org.apache.httpcomponents:httpclient-osgi:$httpClientOsgiVersion"
24+
patch "org.apache.httpcomponents:httpcore:${httpCoreOsgiVersion}"
25+
patch "org.apache.httpcomponents:httpcore-nio:${httpCoreOsgiVersion}"
26+
patch "commons-logging:commons-logging:${commonsLoggingVersion}"
27+
28+
implementation project(':webanno')
29+
30+
implementation("ch.qos.logback:logback-classic:$logbackVersion")
31+
implementation("com.fasterxml.jackson.core:jackson-annotations:$jacksonVersion") {force = true}
32+
implementation("com.fasterxml.jackson.core:jackson-core:$jacksonVersion") {force = true}
33+
implementation("com.fasterxml.jackson.core:jackson-databind:$jacksonVersion") {force = true}
34+
implementation("com.github.jsonld-java:jsonld-java:$jsonldVersion")
35+
implementation("javax.activation:javax.activation-api:$activationApiVersion")
36+
implementation("javax.xml.bind:jaxb-api:$jaxbVersion")
37+
implementation("org.apache.camel:camel-jetty9:$camelVersion")
38+
implementation("org.apache.camel:camel-http4:$camelVersion")
39+
implementation("org.apache.camel:camel-spring-redis:$camelVersion")
40+
implementation("org.apache.httpcomponents:httpclient-cache:$httpClientOsgiVersion")
41+
42+
testImplementation("org.apache.camel:camel-test:$camelVersion")
43+
testImplementation("org.apache.camel:camel-jetty9:$camelVersion")
44+
testImplementation("org.apache.camel:camel-http4:$camelVersion")
45+
testImplementation("ch.qos.logback:logback-classic:$logbackVersion")
46+
}
47+
48+
artifacts {
49+
archives (file('build/cfg/main/de.ubleipzig.dynamo.cfg')) {
50+
classifier 'configuration'
51+
type 'cfg'
52+
}
53+
}
Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
## ---------------------------------------------------------------------------
2+
## Licensed to the Apache Software Foundation (ASF) under one or more
3+
## contributor license agreements. See the NOTICE file distributed with
4+
## this work for additional information regarding copyright ownership.
5+
## The ASF licenses this file to You under the Apache License, Version 2.0
6+
## (the "License"); you may not use this file except in compliance with
7+
## the License. You may obtain a copy of the License at
8+
##
9+
## http://www.apache.org/licenses/LICENSE-2.0
10+
##
11+
## Unless required by applicable law or agreed to in writing, software
12+
## distributed under the License is distributed on an "AS IS" BASIS,
13+
## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
## See the License for the specific language governing permissions and
15+
## limitations under the License.
16+
## ---------------------------------------------------------------------------
17+
dynamo.host=dynamo
18+
dynamo.port=9095
19+
dynamo.prefix=/dynamo
20+
searchservice.host=dynamo-search
21+
searchservice.port=9097
22+
searchservice.prefix=/search
23+
triplestore.baseUrl=fuseki:3030/fuseki/trellis/query
24+
trellis.FQDN=http://some.testdomain.org
25+
redis.host=localhost
26+
redis.port=6379

iiif.dynamic/src/main/java/de/ubleipzig/iiif/dynamic/JsonLdProcessorUtils.java renamed to dynamo/src/main/java/de/ubleipzig/dynamo/JsonLdProcessorUtils.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
* limitations under the License.
1313
*/
1414

15-
package de.ubleipzig.iiif.dynamic;
15+
package de.ubleipzig.dynamo;
1616

1717
import static org.slf4j.LoggerFactory.getLogger;
1818

iiif.dynamic/src/main/java/de/ubleipzig/iiif/dynamic/PatternUtils.java renamed to dynamo/src/main/java/de/ubleipzig/dynamo/PatternUtils.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
* limitations under the License.
1313
*/
1414

15-
package de.ubleipzig.iiif.dynamic;
15+
package de.ubleipzig.dynamo;
1616

1717
import java.util.regex.Matcher;
1818
import java.util.regex.Pattern;

iiif.dynamic/src/main/java/de/ubleipzig/iiif/dynamic/QueryUtils.java renamed to dynamo/src/main/java/de/ubleipzig/dynamo/QueryUtils.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
* limitations under the License.
1313
*/
1414

15-
package de.ubleipzig.iiif.dynamic;
15+
package de.ubleipzig.dynamo;
1616

1717
import java.io.BufferedReader;
1818
import java.io.IOException;

iiif.dynamic/src/main/java/de/ubleipzig/iiif/dynamic/camel/Dynamo.java renamed to dynamo/src/main/java/de/ubleipzig/dynamo/camel/Dynamo.java

Lines changed: 87 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -12,35 +12,47 @@
1212
* limitations under the License.
1313
*/
1414

15-
package de.ubleipzig.iiif.dynamic.camel;
15+
package de.ubleipzig.dynamo.camel;
1616

17-
import static de.ubleipzig.iiif.dynamic.JsonLdProcessorUtils.toJsonLd;
18-
import static de.ubleipzig.iiif.dynamic.QueryUtils.getQuery;
1917
import static de.ubleipzig.webanno.AbstractSerializer.serialize;
2018
import static java.net.URLEncoder.encode;
2119
import static org.apache.camel.Exchange.CONTENT_TYPE;
2220
import static org.apache.camel.Exchange.HTTP_CHARACTER_ENCODING;
2321
import static org.apache.camel.Exchange.HTTP_METHOD;
22+
import static org.apache.camel.Exchange.HTTP_QUERY;
2423
import static org.apache.camel.Exchange.HTTP_RESPONSE_CODE;
2524
import static org.apache.camel.LoggingLevel.INFO;
2625

2726
import com.github.jsonldjava.core.JsonLdError;
2827

28+
import de.ubleipzig.dynamo.JsonLdProcessorUtils;
29+
import de.ubleipzig.dynamo.QueryUtils;
2930
import de.ubleipzig.webanno.CollectionBuilder;
3031
import de.ubleipzig.webanno.ManifestBuilder;
3132
import de.ubleipzig.webanno.templates.AnnotationList;
3233

3334
import java.io.IOException;
35+
import java.io.InputStream;
3436
import java.io.UncheckedIOException;
37+
import java.util.Hashtable;
38+
import java.util.Objects;
39+
import java.util.Properties;
40+
41+
import javax.naming.Context;
42+
import javax.naming.InitialContext;
3543

3644
import org.apache.camel.Exchange;
3745
import org.apache.camel.builder.RouteBuilder;
38-
import org.apache.camel.component.properties.PropertiesComponent;
46+
import org.apache.camel.impl.JndiRegistry;
3947
import org.apache.camel.main.Main;
4048
import org.apache.camel.main.MainListenerSupport;
4149
import org.apache.camel.main.MainSupport;
50+
import org.apache.camel.util.IOHelper;
4251
import org.slf4j.Logger;
4352
import org.slf4j.LoggerFactory;
53+
import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
54+
import org.springframework.data.redis.core.RedisTemplate;
55+
import org.springframework.data.redis.serializer.StringRedisSerializer;
4456

4557

4658
/**
@@ -50,6 +62,16 @@
5062
*/
5163
public class Dynamo {
5264

65+
private static final Logger LOGGER = LoggerFactory.getLogger(Dynamo.class);
66+
private static final JedisConnectionFactory CONNECTION_FACTORY = new JedisConnectionFactory();
67+
private static RedisTemplate<String, String> redisTemplate;
68+
69+
static {
70+
CONNECTION_FACTORY.setHostName("redis");
71+
CONNECTION_FACTORY.setPort(6379);
72+
CONNECTION_FACTORY.afterPropertiesSet();
73+
}
74+
5375
/**
5476
* @param args String[]
5577
* @throws Exception Exception
@@ -66,6 +88,13 @@ private void init() throws Exception {
6688
final Main main = new Main();
6789
main.addRouteBuilder(new QueryRoute());
6890
main.addMainListener(new Events());
91+
final JndiRegistry registry = new JndiRegistry(createInitialContext());
92+
redisTemplate = new RedisTemplate<>();
93+
redisTemplate.setConnectionFactory(CONNECTION_FACTORY);
94+
redisTemplate.setDefaultSerializer(new StringRedisSerializer());
95+
redisTemplate.afterPropertiesSet();
96+
Objects.requireNonNull(registry).bind("redisTemplate", redisTemplate);
97+
main.setPropertyPlaceholderLocations("file:${env:DYNAMO_HOME}/de.ubleipzig.dynamo.cfg");
6998
main.run();
7099
}
71100

@@ -90,8 +119,6 @@ public void beforeStop(final MainSupport main) {
90119
*/
91120
public static class QueryRoute extends RouteBuilder {
92121

93-
private static final Logger LOGGER = LoggerFactory.getLogger(Dynamo.class);
94-
95122
private static final String contentTypeNTriples = "application/n-triples";
96123
private static final String contentTypeJsonLd = "application/ld+json";
97124
private static final String contentTypeHTMLForm = "application/x-www-form-urlencoded";
@@ -106,10 +133,7 @@ public static class QueryRoute extends RouteBuilder {
106133
*/
107134
public void configure() {
108135

109-
final PropertiesComponent pc = getContext().getComponent("properties", PropertiesComponent.class);
110-
pc.setLocation("classpath:application.properties");
111-
112-
from("jetty:http://{{rest.host}}:{{rest.port}}{{rest.prefix}}?"
136+
from("jetty:http://{{dynamo.host}}:{{dynamo.port}}{{dynamo.prefix}}?"
113137
+ "optionsEnabled=true&matchOnUriPrefix=true&sendServerVersion=false"
114138
+ "&httpMethodRestrict=GET,OPTIONS")
115139
.routeId("Sparqler")
@@ -118,9 +142,24 @@ public void configure() {
118142
.constant("*")
119143
.choice()
120144
.when(header(HTTP_METHOD).isEqualTo("GET"))
145+
.to("direct:redis-get");
146+
from("direct:redis-get")
147+
.routeId("RedisGet")
148+
.process(e -> {
149+
final String httpquery = e.getIn().getHeader(HTTP_QUERY).toString();
150+
if (redisTemplate.opsForValue().get(httpquery) != null) {
151+
e.getIn().setBody(redisTemplate.opsForValue().get(httpquery));
152+
LOGGER.info("Getting query result from Redis Cache");
153+
} else {
154+
e.getIn().setHeader(CONTENT_TYPE, EMPTY);
155+
}
156+
})
157+
.choice()
158+
.when(header(CONTENT_TYPE).isEqualTo(EMPTY))
121159
.to("direct:sparql");
122160
from("direct:sparql")
123161
.routeId("SparqlerGet")
162+
.removeHeader(CONTENT_TYPE)
124163
.choice()
125164
.when(header(SPARQL_QUERY).isEqualTo("meta"))
126165
.to("direct:getMetaGraph")
@@ -135,7 +174,7 @@ public void configure() {
135174
.constant(contentTypeNTriples)
136175
.process(e -> e.getIn()
137176
.setBody(sparqlConstruct(
138-
getQuery("canvas-anno.sparql", getV1(e), getV2(e)))))
177+
QueryUtils.getQuery("canvas-anno.sparql", getV1(e), getV2(e)))))
139178
.to("http4:{{triplestore.baseUrl}}?useSystemProperties=true&bridgeEndpoint=true")
140179
.filter(header(HTTP_RESPONSE_CODE).isEqualTo(200))
141180
.setHeader(CONTENT_TYPE)
@@ -151,7 +190,7 @@ public void configure() {
151190
.setHeader(HTTP_ACCEPT)
152191
.constant(contentTypeNTriples)
153192
.process(e -> e.getIn()
154-
.setBody(sparqlConstruct(getQuery("collection.sparql"))))
193+
.setBody(sparqlConstruct(QueryUtils.getQuery("collection.sparql"))))
155194
.to("http4:{{triplestore.baseUrl}}?useSystemProperties=true&bridgeEndpoint=true")
156195
.filter(header(HTTP_RESPONSE_CODE).isEqualTo(200))
157196
.setHeader(CONTENT_TYPE)
@@ -206,7 +245,8 @@ public void configure() {
206245
.getBody()
207246
.toString());
208247
e.getIn().setBody(builder.build());
209-
});
248+
})
249+
.to("direct:redis-put");
210250
from("direct:buildCollection")
211251
.routeId("CollectionBuilder")
212252
.removeHeader(HTTP_ACCEPT)
@@ -220,6 +260,18 @@ public void configure() {
220260
.getBody()
221261
.toString());
222262
e.getIn().setBody(builder.build());
263+
})
264+
.to("direct:redis-put");
265+
from("direct:redis-put")
266+
.routeId("RedisPut")
267+
.log(INFO, LOGGER, "Storing query result in Redis Cache")
268+
.process(e -> {
269+
final String httpquery = e.getIn().getHeader(HTTP_QUERY).toString();
270+
final String body = e.getIn().getBody().toString();
271+
if (null == redisTemplate.opsForValue().get(httpquery)) {
272+
redisTemplate.opsForValue().set(httpquery, body);
273+
}
274+
e.getIn().setBody(redisTemplate.opsForValue().get(httpquery));
223275
});
224276
}
225277

@@ -260,12 +312,31 @@ private void processJsonLdExchange(final Exchange e, final String contextUri, fi
260312
final String body = e.getIn().getBody().toString();
261313
if ( body != null && !body.isEmpty()) {
262314
e.getIn()
263-
.setBody(toJsonLd(e.getIn()
264-
.getBody()
265-
.toString(), contextUri, frameUri));
315+
.setBody(JsonLdProcessorUtils.toJsonLd(e.getIn()
316+
.getBody()
317+
.toString(), contextUri, frameUri));
266318
} else {
267319
e.getIn().setHeader(CONTENT_TYPE, EMPTY);
268320
}
269321
}
270322
}
323+
324+
325+
/**
326+
*
327+
* @return InitialContext Context
328+
* @throws Exception Exception
329+
*/
330+
private static Context createInitialContext() throws Exception {
331+
final InputStream in = Dynamo.class
332+
.getClassLoader()
333+
.getResourceAsStream("jndi.properties");
334+
try {
335+
final Properties properties = new Properties();
336+
properties.load(in);
337+
return new InitialContext(new Hashtable<>(properties));
338+
} finally {
339+
IOHelper.close(in);
340+
}
341+
}
271342
}

0 commit comments

Comments
 (0)