[ZEPPELIN-4517]. Use spark-2.4 as the default profile

### What is this PR for?

Simple PR to make default profile of spark-2.4

### What type of PR is it?
[Improvement]

### Todos
* [ ] - Task

### What is the Jira issue?
* https://jira.apache.org/jira/browse/ZEPPELIN-4517

### How should this be tested?
* CI pass

### Screenshots (if appropriate)

### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No

Author: Jeff Zhang <[email protected]>

Closes #3586 from zjffdu/ZEPPELIN-4517 and squashes the following commits:

cb8f5db [Jeff Zhang] [ZEPPELIN-4517]. Use spark-2.4 as the default profile

Author: zjffdu

dev/create_release.sh

@@ -98,8 +98,8 @@ function make_binary_release() {
 
 git_clone
 make_source_package
-make_binary_release all "-Pspark-2.3 -Phadoop-2.6 -Pscala-${SCALA_VERSION}"
-make_binary_release netinst "-Pspark-2.3 -Phadoop-2.6 -Pscala-${SCALA_VERSION} -pl zeppelin-interpreter,zeppelin-zengine,:zeppelin-display_${SCALA_VERSION},:zeppelin-spark-dependencies_${SCALA_VERSION},:zeppelin-spark_${SCALA_VERSION},zeppelin-web,zeppelin-server,zeppelin-distribution -am"
+make_binary_release all "-Phadoop-2.6 -Pscala-${SCALA_VERSION}"
+make_binary_release netinst "-Phadoop-2.6 -Pscala-${SCALA_VERSION} -pl zeppelin-interpreter,zeppelin-zengine,:zeppelin-display_${SCALA_VERSION},:zeppelin-spark-dependencies_${SCALA_VERSION},:zeppelin-spark_${SCALA_VERSION},zeppelin-web,zeppelin-server,zeppelin-distribution -am"
 
 # remove non release files and dirs
 rm -rf "${WORKING_DIR}/zeppelin"

dev/publish_release.sh

@@ -46,7 +46,7 @@ if [[ $RELEASE_VERSION == *"SNAPSHOT"* ]]; then
   DO_SNAPSHOT="yes"
 fi
 
-PUBLISH_PROFILES="-Ppublish-distr -Pspark-2.1 -Phadoop-2.6 -Pr"
+PUBLISH_PROFILES="-Ppublish-distr -Phadoop-2.6 -Pr"
 PROJECT_OPTIONS="-pl !zeppelin-distribution"
 NEXUS_STAGING="https://repository.apache.org/service/local/staging"
 NEXUS_PROFILE="153446d1ac37c4"

rlang/pom.xml

@@ -37,7 +37,7 @@
     <properties>
         <interpreter.name>r</interpreter.name>
         <jsoup.version>1.12.1</jsoup.version>
-        <spark.version>2.4.3</spark.version>
+        <spark.version>2.4.4</spark.version>
         <grpc.version>1.15.0</grpc.version>
 
         <spark.archive>spark-${spark.version}</spark.archive>

rlang/src/main/java/org/apache/zeppelin/r/IRInterpreter.java

@@ -73,7 +73,7 @@ protected boolean isSparkSupported() {
    * @return
    */
   protected int sparkVersion() {
-    return 20403;
+    return 20404;
   }
 
   /**

spark/pom.xml

@@ -198,8 +198,11 @@
         <!-- profile spark-x only affect the embedded spark version in zeppelin distribution -->
         <profile>
             <id>spark-2.4</id>
+            <activation>
+                <activeByDefault>true</activeByDefault>
+            </activation>
             <properties>
-                <spark.version>2.4.3</spark.version>
+                <spark.version>2.4.4</spark.version>
                 <protobuf.version>2.5.0</protobuf.version>
                 <py4j.version>0.10.7</py4j.version>
             </properties>
@@ -216,9 +219,6 @@
 
         <profile>
             <id>spark-2.2</id>
-            <activation>
-                <activeByDefault>true</activeByDefault>
-            </activation>
             <properties>
                 <spark.version>2.2.3</spark.version>
                 <py4j.version>0.10.7</py4j.version>

spark/scala-2.11/pom.xml

@@ -32,7 +32,7 @@
   <name>Zeppelin: Spark Interpreter Scala_2.11</name>
 
   <properties>
-    <spark.version>2.4.3</spark.version>
+    <spark.version>2.4.4</spark.version>
     <spark.scala.version>2.11.12</spark.scala.version>
     <spark.scala.binary.version>2.11</spark.scala.binary.version>
     <spark.scala.compile.version>${spark.scala.version}</spark.scala.compile.version>

spark/scala-2.12/pom.xml

@@ -33,7 +33,7 @@
   <name>Zeppelin: Spark Interpreter Scala_2.12</name>
 
   <properties>
-    <spark.version>2.4.3</spark.version>
+    <spark.version>2.4.4</spark.version>
     <spark.scala.version>2.12.8</spark.scala.version>
     <spark.scala.binary.version>2.12</spark.scala.binary.version>
     <spark.scala.compile.version>${spark.scala.version}</spark.scala.compile.version>

submarine/pom.xml

@@ -57,6 +57,17 @@
       <artifactId>zeppelin-python</artifactId>
       <version>0.9.0-SNAPSHOT</version>
     </dependency>
+    <dependency>
+      <groupId>org.apache.zeppelin</groupId>
+      <artifactId>zeppelin-jupyter-interpreter</artifactId>
+      <version>${project.version}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>net.sf.py4j</groupId>
+          <artifactId>py4j</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
     <dependency>
       <groupId>org.apache.zeppelin</groupId>
       <artifactId>zeppelin-shell</artifactId>

zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest24.java

@@ -33,7 +33,7 @@ public SparkIntegrationTest24(String sparkVersion) {
   @Parameterized.Parameters
   public static List<Object[]> data() {
     return Arrays.asList(new Object[][]{
-            {"2.4.3"}
+            {"2.4.4"}
     });
   }
 

zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinSparkClusterTest24.java

@@ -33,7 +33,7 @@ public ZeppelinSparkClusterTest24(String sparkVersion) throws Exception {
   @Parameterized.Parameters
   public static List<Object[]> data() {
     return Arrays.asList(new Object[][]{
-            {"2.4.3"}
+            {"2.4.4"}
     });
   }
 }