diff --git a/.github/workflows/gradle-build.yml b/.github/workflows/gradle-build.yml new file mode 100644 index 00000000000..f3a5d5a73f0 --- /dev/null +++ b/.github/workflows/gradle-build.yml @@ -0,0 +1,36 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Java CI + +on: + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Login to github docker Registry + run: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} + + - name: Set the image tag + id: image_vars + run: | + TAG=latest + echo ::set-output name=tag::$(echo $TAG | awk '{print tolower($0)}') + + - uses: actions/setup-java@v2 + with: + distribution: adopt + java-version: 11 + + - name: Setup Gradle + uses: gradle/gradle-build-action@v2 + + - name: Execute Gradle build + run: ./gradlew distDocker -PdockerRegistry=ghcr.io -PdockerImagePrefix=${{ github.repository }} -PdockerImageTag=${{steps.image_vars.outputs.tag}} \ No newline at end of file diff --git a/common/scala/build.gradle b/common/scala/build.gradle index ccdfc8a567a..30f41f873ed 100644 --- a/common/scala/build.gradle +++ b/common/scala/build.gradle @@ -89,8 +89,8 @@ dependencies { compile "io.reactivex:rxjava:1.3.8" compile "io.reactivex:rxjava-reactive-streams:1.2.1" compile "com.microsoft.azure:azure-cosmosdb:2.6.2" + compile "com.sksamuel.elastic4s:elastic4s-client-esjava_${gradle.scala.depVersion}:7.10.3" - compile "com.sksamuel.elastic4s:elastic4s-http_${gradle.scala.depVersion}:6.7.4" //for mongo compile "org.mongodb.scala:mongo-scala-driver_${gradle.scala.depVersion}:2.7.0" diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/database/elasticsearch/ElasticSearchActivationStore.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/database/elasticsearch/ElasticSearchActivationStore.scala index 50504263f90..4b551e86312 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/database/elasticsearch/ElasticSearchActivationStore.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/database/elasticsearch/ElasticSearchActivationStore.scala @@ -19,17 +19,17 @@ package org.apache.openwhisk.core.database.elasticsearch import java.time.Instant import java.util.concurrent.TimeUnit - import scala.language.postfixOps import akka.actor.ActorSystem import akka.event.Logging.ErrorLevel import akka.http.scaladsl.model._ import akka.stream.scaladsl.Flow -import com.sksamuel.elastic4s.http.search.SearchHit -import com.sksamuel.elastic4s.http.{ElasticClient, ElasticProperties, NoOpRequestConfigCallback} -import com.sksamuel.elastic4s.indexes.IndexRequest -import com.sksamuel.elastic4s.searches.queries.RangeQuery -import com.sksamuel.elastic4s.searches.queries.matches.MatchPhrase +import com.sksamuel.elastic4s.http.{JavaClient, NoOpRequestConfigCallback} +import com.sksamuel.elastic4s.{ElasticClient, ElasticProperties} +import com.sksamuel.elastic4s.requests.indexes.IndexRequest +import com.sksamuel.elastic4s.requests.searches.SearchHit +import com.sksamuel.elastic4s.requests.searches.queries.RangeQuery +import com.sksamuel.elastic4s.requests.searches.queries.matches.MatchPhrase import org.apache.http.auth.{AuthScope, UsernamePasswordCredentials} import org.apache.http.impl.client.BasicCredentialsProvider import org.apache.http.impl.nio.client.HttpAsyncClientBuilder @@ -60,18 +60,16 @@ class ElasticSearchActivationStore( useBatching: Boolean = false)(implicit actorSystem: ActorSystem, override val logging: Logging) extends ActivationStore { - import com.sksamuel.elastic4s.http.ElasticDsl._ + import com.sksamuel.elastic4s.ElasticDsl._ import ElasticSearchActivationStore.{generateIndex, httpClientCallback} private implicit val executionContextExecutor: ExecutionContextExecutor = actorSystem.dispatcher + private val javaClient = JavaClient(ElasticProperties(s"${elasticSearchConfig.protocol}://${elasticSearchConfig.hosts}"), + NoOpRequestConfigCallback, + httpClientCallback) - private val client = - ElasticClient( - ElasticProperties(s"${elasticSearchConfig.protocol}://${elasticSearchConfig.hosts}"), - NoOpRequestConfigCallback, - httpClientCallback) + private val client = ElasticClient(javaClient) - private val esType = "_doc" private val maxOpenDbRequests = actorSystem.settings.config .getInt("akka.http.host-connection-pool.max-connections") / 2 private val batcher: Batcher[IndexRequest, Either[ArtifactStoreException, DocInfo]] = @@ -111,7 +109,7 @@ class ElasticSearchActivationStore( "response" -> response)) val index = generateIndex(activation.namespace.namespace) - val op = indexInto(index, esType).doc(payload.toString).id(activation.docid.asString) + val op = indexInto(index).doc(payload.toString).id(activation.docid.asString) // always use batching val res = batcher.put(op).map { @@ -185,7 +183,7 @@ class ElasticSearchActivationStore( } .map { res => if (res.status == StatusCodes.OK.intValue) { - if (res.result.hits.total == 0) { + if (res.result.hits.total.value == 0) { transid.finished(this, start, s"[GET] 'activations', document: '$activationId'; not found.") throw NoDocumentException("not found on 'get'") } else { @@ -227,7 +225,7 @@ class ElasticSearchActivationStore( val res = client .execute { - deleteByQuery(index, esType, termQuery("_id", activationId.asString)) + deleteByQuery(index, termQuery("_id", activationId.asString)) } .map { res => if (res.status == StatusCodes.OK.intValue) { diff --git a/core/scheduler/src/main/scala/org/apache/openwhisk/core/scheduler/queue/ElasticSearchDurationChecker.scala b/core/scheduler/src/main/scala/org/apache/openwhisk/core/scheduler/queue/ElasticSearchDurationChecker.scala index 88e1f2ff177..7041d28ef33 100644 --- a/core/scheduler/src/main/scala/org/apache/openwhisk/core/scheduler/queue/ElasticSearchDurationChecker.scala +++ b/core/scheduler/src/main/scala/org/apache/openwhisk/core/scheduler/queue/ElasticSearchDurationChecker.scala @@ -17,9 +17,10 @@ package org.apache.openwhisk.core.scheduler.queue import akka.actor.ActorSystem -import com.sksamuel.elastic4s.http.ElasticDsl._ -import com.sksamuel.elastic4s.http.{ElasticClient, ElasticProperties, NoOpRequestConfigCallback} -import com.sksamuel.elastic4s.searches.queries.Query +import com.sksamuel.elastic4s.ElasticDsl._ +import com.sksamuel.elastic4s.http.{JavaClient, NoOpRequestConfigCallback} +import com.sksamuel.elastic4s.{ElasticClient, ElasticProperties} +import com.sksamuel.elastic4s.requests.searches.queries.Query import com.sksamuel.elastic4s.{ElasticDate, ElasticDateMath, Seconds} import org.apache.openwhisk.common.Logging import org.apache.openwhisk.core.ConfigKeys @@ -28,7 +29,7 @@ import org.apache.openwhisk.spi.Spi import pureconfig.loadConfigOrThrow import spray.json.{JsArray, JsNumber, JsValue, RootJsonFormat, deserializationError, _} -import scala.concurrent.Future +import scala.concurrent.{ExecutionContextExecutor, Future} import scala.concurrent.duration.FiniteDuration import scala.language.implicitConversions import scala.util.{Failure, Try} @@ -45,7 +46,7 @@ object ElasticSearchDurationChecker { val FilterAggregationName = "filterAggregation" val AverageAggregationName = "averageAggregation" - implicit val serde = new ElasticSearchDurationCheckResultFormat() + implicit val serde: ElasticSearchDurationCheckResultFormat = new ElasticSearchDurationCheckResultFormat() def getFromDate(timeWindow: FiniteDuration): ElasticDateMath = ElasticDate.now minus (timeWindow.toSeconds.toInt, Seconds) @@ -58,7 +59,7 @@ class ElasticSearchDurationChecker(private val client: ElasticClient, val timeWi import ElasticSearchDurationChecker._ import org.apache.openwhisk.core.database.elasticsearch.ElasticSearchActivationStore.generateIndex - implicit val ec = actorSystem.getDispatcher + implicit val ec: ExecutionContextExecutor = actorSystem.getDispatcher override def checkAverageDuration(invocationNamespace: String, actionMetaData: WhiskActionMetaData)( callback: DurationCheckResult => DurationCheckResult): Future[DurationCheckResult] = { @@ -104,7 +105,7 @@ class ElasticSearchDurationChecker(private val client: ElasticClient, val timeWi .map(callback(_)) .andThen { case Failure(t) => - logging.error(this, s"failed to check the average duration: ${t}") + logging.error(this, s"failed to check the average duration: $t") } } } @@ -116,11 +117,11 @@ object ElasticSearchDurationCheckerProvider extends DurationCheckerProvider { implicit val as: ActorSystem = actorSystem implicit val logging: Logging = log - val elasticClient = - ElasticClient( - ElasticProperties(s"${elasticSearchConfig.protocol}://${elasticSearchConfig.hosts}"), - NoOpRequestConfigCallback, - httpClientCallback) + val javaClient = JavaClient(ElasticProperties(s"${elasticSearchConfig.protocol}://${elasticSearchConfig.hosts}"), + NoOpRequestConfigCallback, + httpClientCallback) + + val elasticClient = ElasticClient(javaClient) new ElasticSearchDurationChecker(elasticClient, durationCheckerConfig.timeWindow) } @@ -187,7 +188,7 @@ class ElasticSearchDurationCheckResultFormat extends RootJsonFormat[DurationChec "took": 0 } */ - implicit def read(json: JsValue) = { + implicit def read(json: JsValue): DurationCheckResult = { val jsObject = json.asJsObject jsObject.getFields("aggregations", "took", "hits") match { diff --git a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/ElasticSearchDurationCheckerTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/ElasticSearchDurationCheckerTests.scala index 4f6d33e2756..5c3ff3da5f2 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/ElasticSearchDurationCheckerTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/ElasticSearchDurationCheckerTests.scala @@ -17,8 +17,9 @@ package org.apache.openwhisk.core.scheduler.queue.test -import com.sksamuel.elastic4s.http.ElasticDsl._ -import com.sksamuel.elastic4s.http.{ElasticClient, ElasticProperties, NoOpRequestConfigCallback} +import com.sksamuel.elastic4s.ElasticDsl._ +import com.sksamuel.elastic4s.http.{JavaClient, NoOpRequestConfigCallback} +import com.sksamuel.elastic4s.{ElasticClient, ElasticProperties} import common._ import common.rest.WskRestOperations import org.apache.http.auth.{AuthScope, UsernamePasswordCredentials} @@ -91,11 +92,11 @@ class ElasticSearchDurationCheckerTests } } - private val client = - ElasticClient( - ElasticProperties(s"${elasticSearchConfig.protocol}://${elasticSearchConfig.hosts}"), + val javaClient = JavaClient(ElasticProperties(s"${elasticSearchConfig.protocol}://${elasticSearchConfig.hosts}"), NoOpRequestConfigCallback, httpClientCallback) + + val client = ElasticClient(javaClient) private val elasticSearchDurationChecker = new ElasticSearchDurationChecker(client, defaultDurationCheckWindow) diff --git a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueFlowTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueFlowTests.scala index 16250661218..bdbf4eb0577 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueFlowTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueFlowTests.scala @@ -3,7 +3,7 @@ package org.apache.openwhisk.core.scheduler.queue.test import akka.actor.ActorRef import akka.actor.FSM.{CurrentState, StateTimeout, SubscribeTransitionCallBack, Transition} import akka.testkit.{TestActor, TestFSMRef, TestProbe} -import com.sksamuel.elastic4s.http.{search => _} +import com.sksamuel.elastic4s.requests.{searches => _} import org.apache.openwhisk.common.GracefulShutdown import org.apache.openwhisk.core.connector.ContainerCreationError.{NonExecutableActionError, WhiskError} import org.apache.openwhisk.core.connector.ContainerCreationMessage diff --git a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTests.scala index 55ca7dc4777..4b8de2afe64 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTests.scala @@ -15,7 +15,7 @@ import com.ibm.etcd.api._ import com.ibm.etcd.client.kv.KvClient.Watch import com.ibm.etcd.client.kv.WatchUpdate import com.ibm.etcd.client.{EtcdClient => Client} -import com.sksamuel.elastic4s.http.ElasticClient +import com.sksamuel.elastic4s.ElasticClient import common.StreamLogging import org.apache.openwhisk.common.{GracefulShutdown, TransactionId} import org.apache.openwhisk.core.ack.ActiveAck diff --git a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTestsFixture.scala b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTestsFixture.scala index 2d2ead20aed..751ee816ee1 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTestsFixture.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTestsFixture.scala @@ -1,14 +1,13 @@ package org.apache.openwhisk.core.scheduler.queue.test import java.time.Instant - import akka.actor.{ActorRef, ActorSystem} import akka.testkit.{ImplicitSender, TestKit, TestProbe} -import com.sksamuel.elastic4s.http -import com.sksamuel.elastic4s.http.ElasticDsl.{avgAgg, boolQuery, matchQuery, rangeQuery, search} -import com.sksamuel.elastic4s.http._ -import com.sksamuel.elastic4s.http.search.{SearchHits, SearchResponse} -import com.sksamuel.elastic4s.searches.SearchRequest +import com.sksamuel.elastic4s.Executor +import com.sksamuel.elastic4s.ElasticDsl.{avgAgg, boolQuery, matchQuery, rangeQuery, search} +import com.sksamuel.elastic4s._ +import com.sksamuel.elastic4s.requests.common.Shards +import com.sksamuel.elastic4s.requests.searches.{SearchHits, SearchRequest, SearchResponse, Total} import common.StreamLogging import org.apache.kafka.clients.producer.RecordMetadata import org.apache.kafka.common.TopicPartition @@ -24,20 +23,9 @@ import org.apache.openwhisk.core.entity.{WhiskActivation, _} import org.apache.openwhisk.core.etcd.EtcdKV.{ContainerKeys, QueueKeys, ThrottlingKeys} import org.apache.openwhisk.core.scheduler.SchedulerEndpoints import org.apache.openwhisk.core.scheduler.grpc.GetActivation -import org.apache.openwhisk.core.scheduler.queue.ElasticSearchDurationChecker.{getFromDate, AverageAggregationName} +import org.apache.openwhisk.core.scheduler.queue.ElasticSearchDurationChecker.{AverageAggregationName, getFromDate} import org.apache.openwhisk.core.scheduler.queue._ -import org.apache.openwhisk.core.service.{ - AlreadyExist, - DeleteEvent, - Done, - InitialDataStorageResults, - PutEvent, - RegisterData, - RegisterInitialData, - UnregisterData, - UnwatchEndpoint, - WatchEndpoint -} +import org.apache.openwhisk.core.service.{AlreadyExist, DeleteEvent, Done, InitialDataStorageResults, PutEvent, RegisterData, RegisterInitialData, UnregisterData, UnwatchEndpoint, WatchEndpoint} import org.scalamock.scalatest.MockFactory import scala.concurrent.duration.DurationInt @@ -217,17 +205,18 @@ class MemoryQueueTestsFixture (mockEsClient .execute[SearchRequest, SearchResponse, Future](_: SearchRequest)( + _: Executor[Future], _: Functor[Future], - _: http.Executor[Future], _: Handler[SearchRequest, SearchResponse], - _: Manifest[SearchResponse])) - .expects(searchRequest, *, *, *, *) + _: Manifest[SearchResponse], + _: CommonRequestOptions)) + .expects(searchRequest, *, *, *, *, *) .returns( Future.successful(RequestSuccess( 200, None, Map.empty, - SearchResponse(1, false, false, Map.empty, Shards(0, 0, 0), None, Map.empty, SearchHits(0, 0, Array.empty))))) + SearchResponse(1, false, false, Map.empty, Shards(0, 0, 0), None, Map.empty, SearchHits(Total(0, "eq"), 0, Array.empty))))) .once() } diff --git a/tools/jenkins/apache/dockerhub.groovy b/tools/jenkins/apache/dockerhub.groovy index 6ff1c591d9d..f0238c6cabe 100644 --- a/tools/jenkins/apache/dockerhub.groovy +++ b/tools/jenkins/apache/dockerhub.groovy @@ -51,6 +51,5 @@ node('ubuntu') { withCredentials([string(credentialsId: 'openwhisk_slack_token', variable: 'OPENWHISK_SLACK_TOKEN')]) { sh "curl -X POST --data-urlencode 'payload={\"channel\": \"#dev\", \"username\": \"whiskbot\", \"text\": \"OpenWhisk Docker Images build and posted to https://hub.docker.com/u/openwhisk by Jenkins job ${BUILD_URL}\", \"icon_emoji\": \":openwhisk:\"}' https://hooks.slack.com/services/${OPENWHISK_SLACK_TOKEN}" } - } } diff --git a/tools/macos/README.md b/tools/macos/README.md index e669a538955..989ae336157 100644 --- a/tools/macos/README.md +++ b/tools/macos/README.md @@ -55,7 +55,7 @@ brew install gnu-tar # install pip sudo easy_install pip # install script prerequisites -pip install docker==5.0.0 ansible==4.1.0 jinja2==3.0.1 couchdb==1.2 httplib2==0.19.1 requests==2.25.1 six=1.16.0 +pip install docker==5.0.0 ansible==4.1.0 jinja2==3.0.1 couchdb==1.2 httplib2==0.19.1 requests==2.25.1 six==1.16.0 ``` Make sure you correctly configure the environment variable $JAVA_HOME.