Update sparkbench

* sparkbench/assembly/pom.xml: * Changed property name activation on `allModules` profile. * Added new profile that excludes `sparkbench-streaming` artifact. * sparkbench/pom.xml: * Changed property name activation on `allModules` profile. * Added new profile that excludes `streaming` module. * Added profile spark2.4 due spark-core_2.12 supports > 2.4.0 version. * Added profile scala 2.12. Scala < 2.12 does not compiles on java 1.11 jdk. * Added profile hadoop3.2 to propagate this variable to all spark benchmark. * sparkbecnh/streaming/pom.xml: * Added profile spark2.4 on sparkbench-streaming POM with spark-streaming-kafka-0-8_2.11 version 2.4.0. Signed-off-by: Luis Ponce <[email protected]>
Intel-bigdata · Aug 22, 2019 · 4eb1191 · 4eb1191
1 parent 40dfcce
commit 4eb1191
Show file tree

Hide file tree

Showing 3 changed files with 111 additions and 2 deletions.
diff --git a/sparkbench/assembly/pom.xml b/sparkbench/assembly/pom.xml
@@ -159,7 +159,43 @@
       </dependencies>
       <activation>
         <property>
-          <name>!modules</name>
+          <name>!exclude-streaming</name>
+        </property>
+      </activation>
+    </profile>
+
+    <profile>
+      <id>exclude-streaming</id>
+      <dependencies>
+        <dependency>
+          <groupId>com.intel.hibench.sparkbench</groupId>
+          <artifactId>sparkbench-micro</artifactId>
+          <version>${project.version}</version>
+        </dependency>
+        <dependency>
+          <groupId>com.intel.hibench.sparkbench</groupId>
+          <artifactId>sparkbench-ml</artifactId>
+          <version>${project.version}</version>
+        </dependency>
+        <dependency>
+          <groupId>com.intel.hibench.sparkbench</groupId>
+          <artifactId>sparkbench-websearch</artifactId>
+          <version>${project.version}</version>
+        </dependency>
+        <dependency>
+          <groupId>com.intel.hibench.sparkbench</groupId>
+          <artifactId>sparkbench-graph</artifactId>
+          <version>${project.version}</version>
+        </dependency>
+        <dependency>
+          <groupId>com.intel.hibench.sparkbench</groupId>
+          <artifactId>sparkbench-sql</artifactId>
+          <version>${project.version}</version>
+        </dependency>
+      </dependencies>
+      <activation>
+        <property>
+          <name>exclude-streaming</name>
         </property>
       </activation>
     </profile>

diff --git a/sparkbench/pom.xml b/sparkbench/pom.xml
@@ -117,7 +117,23 @@
       </modules>
       <activation>
         <property>
-          <name>!modules</name>
+          <name>!exclude-streaming</name>
+        </property>
+      </activation>
+    </profile>
+
+    <profile>
+      <id>no-streaming</id>
+      <modules>
+        <module>micro</module>
+        <module>ml</module>
+        <module>websearch</module>
+        <module>graph</module>
+        <module>sql</module>
+      </modules>
+      <activation>
+        <property>
+          <name>exclude-streaming</name>
         </property>
       </activation>
     </profile>
@@ -191,6 +207,20 @@
       </activation>
     </profile>
 
+    <profile>
+      <id>spark2.4</id>
+      <properties>
+        <spark.version>2.4.3</spark.version>
+        <spark.bin.version>2.4</spark.bin.version>
+      </properties>
+      <activation>
+        <property>
+          <name>spark</name>
+          <value>2.4</value>
+        </property>
+      </activation>
+    </profile>
+
     <profile>
       <id>defaultScalaVersion</id>
       <properties>
@@ -232,5 +262,31 @@
       </activation>
     </profile>
 
+    <profile>
+      <id>scala2.12</id>
+      <properties>
+        <scala.version>2.12.9</scala.version>
+        <scala.binary.version>2.12</scala.binary.version>
+      </properties>
+      <activation>
+        <property>
+          <name>scala</name>
+          <value>2.12</value>
+        </property>
+      </activation>
+    </profile>
+
+    <profile>
+      <id>hadoop3.2</id>
+      <properties>
+        <hadoop.mr2.version>3.2.0</hadoop.mr2.version>
+      </properties>
+      <activation>
+        <property>
+          <name>hadoop</name>
+          <value>3.2</value>
+        </property>
+      </activation>
+    </profile>
   </profiles>
 </project>
diff --git a/sparkbench/streaming/pom.xml b/sparkbench/streaming/pom.xml
@@ -119,5 +119,22 @@
       </activation>
     </profile>
 
+    <profile>
+      <id>spark2.4</id>
+      <dependencies>
+        <dependency>
+          <groupId>org.apache.spark</groupId>
+          <artifactId>spark-streaming-kafka-0-8_2.11</artifactId>
+          <version>2.4.0</version>
+        </dependency>
+      </dependencies>
+      <activation>
+        <property>
+          <name>spark</name>
+          <value>2.4</value>
+        </property>
+      </activation>
+    </profile>
+
   </profiles>
 </project>