update ubuntu to 20.04 (#730)

* update ubuntu to 20.04 Signed-off-by: minmingzhu <[email protected]> * update CI actions and reduce Linear dataset Signed-off-by: minmingzhu <[email protected]> * Update build_and_run_all_spark_3.1_hadoop_3.2.sh * update spark.conf Signed-off-by: minmingzhu <[email protected]> * debug CI Signed-off-by: minmingzhu <[email protected]> * debug CI Signed-off-by: minmingzhu <[email protected]> * split benchmark list Signed-off-by: minmingzhu <[email protected]> * update split benchmark list Signed-off-by: minmingzhu <[email protected]> * update setup-cluster.sh Signed-off-by: minmingzhu <[email protected]> * Move the files required by CI to the actions directory Signed-off-by: minmingzhu <[email protected]> * add benchmarks.lst Signed-off-by: minmingzhu <[email protected]> --------- Signed-off-by: minmingzhu <[email protected]>
Intel-bigdata · Jul 17, 2023 · efc86df · efc86df
1 parent 62de01b
commit efc86df
Show file tree

Hide file tree

Showing 18 changed files with 445 additions and 16 deletions.
diff --git a/...kflows/hibench_ci_spark2.4_hadoop_2.7.yml → ...hibench_ci_spark2.4_hadoop_2.7_part_1.yml b/...kflows/hibench_ci_spark2.4_hadoop_2.7.yml → ...hibench_ci_spark2.4_hadoop_2.7_part_1.yml
@@ -4,16 +4,16 @@ on: [push, pull_request]
 
 jobs:
   build:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Set up JDK 1.8
         uses: actions/setup-java@v1
         with:
           java-version: 1.8
       - name: Restore cached dependencies
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: |
             # /var/cache/apt/archives/*.deb
@@ -24,4 +24,4 @@ jobs:
             ${{ runner.os }}-
       - name: Cluster spark_2.4_hadoop_2.7_hive_0.14
         run: |
-          ${{github.workspace}}/actions/build_and_run_all_spark_2.4_hadoop_2.7_hive_0.14.sh
+          ${{github.workspace}}/actions/build_and_run_all_spark_2.4_hadoop_2.7_hive_0.14_part_1.sh
diff --git a/.github/workflows/hibench_ci_spark2.4_hadoop_2.7_part_2.yml b/.github/workflows/hibench_ci_spark2.4_hadoop_2.7_part_2.yml
@@ -0,0 +1,27 @@
+name: HiBench CI SPARK_2.4_HADOOP_2.7
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up JDK 1.8
+        uses: actions/setup-java@v1
+        with:
+          java-version: 1.8
+      - name: Restore cached dependencies
+        uses: actions/cache@v3
+        with:
+          path: |
+            # /var/cache/apt/archives/*.deb
+            ~/.m2/repository
+            ~/opt
+          key: ${{ runner.os }}_spark_2.4_hadoop_2.7
+          restore-keys: |
+            ${{ runner.os }}-
+      - name: Cluster spark_2.4_hadoop_2.7_hive_0.14
+        run: |
+          ${{github.workspace}}/actions/build_and_run_all_spark_2.4_hadoop_2.7_hive_0.14_part_2.sh
diff --git a/...kflows/hibench_ci_spark3.0_hadoop_3.2.yml → ...hibench_ci_spark3.0_hadoop_3.2_part_1.yml b/...kflows/hibench_ci_spark3.0_hadoop_3.2.yml → ...hibench_ci_spark3.0_hadoop_3.2_part_1.yml
@@ -4,16 +4,16 @@ on: [push, pull_request]
 
 jobs:
   build:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Set up JDK 1.8
         uses: actions/setup-java@v1
         with:
           java-version: 1.8
       - name: Restore cached dependencies
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: |
             # /var/cache/apt/archives/*.deb
@@ -24,4 +24,4 @@ jobs:
             ${{ runner.os }}-
       - name: Cluster spark_3.0_hadoop_3.2_hive_3.0
         run: |
-          ${{github.workspace}}/actions/build_and_run_all_spark_3.0_hadoop_3.2.sh
+          ${{github.workspace}}/actions/build_and_run_all_spark_3.0_hadoop_3.2_part_1.sh
diff --git a/.github/workflows/hibench_ci_spark3.0_hadoop_3.2_part_2.yml b/.github/workflows/hibench_ci_spark3.0_hadoop_3.2_part_2.yml
@@ -0,0 +1,27 @@
+name: HiBench CI SPARK_3.0_HADOOP_3.2
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up JDK 1.8
+        uses: actions/setup-java@v1
+        with:
+          java-version: 1.8
+      - name: Restore cached dependencies
+        uses: actions/cache@v3
+        with:
+          path: |
+            # /var/cache/apt/archives/*.deb
+            ~/.m2/repository
+            ~/opt
+          key: ${{ runner.os }}_spark_3.0_hadoop_3.2
+          restore-keys: |
+            ${{ runner.os }}-
+      - name: Cluster spark_3.0_hadoop_3.2_hive_3.0
+        run: |
+          ${{github.workspace}}/actions/build_and_run_all_spark_3.0_hadoop_3.2_part_2.sh
diff --git a/...kflows/hibench_ci_spark3.1_hadoop_3.2.yml → ...hibench_ci_spark3.1_hadoop_3.2_part_1.yml b/...kflows/hibench_ci_spark3.1_hadoop_3.2.yml → ...hibench_ci_spark3.1_hadoop_3.2_part_1.yml
@@ -4,16 +4,16 @@ on: [push, pull_request]
 
 jobs:
   build:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Set up JDK 1.8
         uses: actions/setup-java@v1
         with:
           java-version: 1.8
       - name: Restore cached dependencies
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: |
             # /var/cache/apt/archives/*.deb
@@ -24,4 +24,4 @@ jobs:
             ${{ runner.os }}-
       - name: Cluster spark_3.1_hadoop_3.2_hive_3.0
         run: |
-          ${{github.workspace}}/actions/build_and_run_all_spark_3.1_hadoop_3.2.sh
+          ${{github.workspace}}/actions/build_and_run_all_spark_3.1_hadoop_3.2_part_1.sh
diff --git a/.github/workflows/hibench_ci_spark3.1_hadoop_3.2_part_2.yml b/.github/workflows/hibench_ci_spark3.1_hadoop_3.2_part_2.yml
@@ -0,0 +1,27 @@
+name: HiBench CI SPARK_3.1_HADOOP_3.2
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up JDK 1.8
+        uses: actions/setup-java@v1
+        with:
+          java-version: 1.8
+      - name: Restore cached dependencies
+        uses: actions/cache@v3
+        with:
+          path: |
+            # /var/cache/apt/archives/*.deb
+            ~/.m2/repository
+            ~/opt
+          key: ${{ runner.os }}_spark_3.1_hadoop_3.2
+          restore-keys: |
+            ${{ runner.os }}-
+      - name: Cluster spark_3.1_hadoop_3.2_hive_3.0
+        run: |
+          ${{github.workspace}}/actions/build_and_run_all_spark_3.1_hadoop_3.2_part_2.sh
diff --git a/actions/benchmarks_part_1.lst b/actions/benchmarks_part_1.lst
@@ -0,0 +1,13 @@
+micro.sleep
+micro.sort
+micro.terasort
+micro.wordcount
+micro.repartition
+micro.dfsioe
+
+sql.aggregation
+sql.join
+sql.scan
+
+websearch.nutchindexing
+websearch.pagerank
diff --git a/actions/benchmarks_part_2.lst b/actions/benchmarks_part_2.lst
@@ -0,0 +1,16 @@
+ml.bayes
+ml.kmeans
+ml.lr
+ml.als
+ml.pca
+ml.gbt
+ml.rf
+ml.svd
+ml.linear
+ml.lda
+ml.svm
+ml.gmm
+ml.correlation
+ml.summarizer
+
+graph.nweight
diff --git a/...run_all_spark_2.4_hadoop_2.7_hive_0.14.sh → ..._spark_2.4_hadoop_2.7_hive_0.14_part_1.sh b/...run_all_spark_2.4_hadoop_2.7_hive_0.14.sh → ..._spark_2.4_hadoop_2.7_hive_0.14_part_1.sh
@@ -28,4 +28,4 @@ echo "Cluster Testing with Spark Version: $SPARK_VERSION"
 echo "========================================="
 
 # run all examples
-source $GITHUB_WORKSPACE/bin/run_all.sh
+source $GITHUB_WORKSPACE/actions/run_all_part_1.sh
diff --git a/actions/build_and_run_all_spark_2.4_hadoop_2.7_hive_0.14_part_2.sh b/actions/build_and_run_all_spark_2.4_hadoop_2.7_hive_0.14_part_2.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+# exit when any command fails
+set -e
+
+export HADOOP_VERSION=2.7.7
+export SPARK_VERSION=2.4.0
+export SPARK_BIN_VERSION=spark2.4
+export SPARK_HADOOP_VERSION=hadoop2.7
+export HIVE_VERSION=0.14.0
+
+# keep track of the last executed command
+trap 'last_command=$current_command; current_command=$BASH_COMMAND' DEBUG
+# echo an error message before exiting
+trap 'echo "\"${last_command}\" command filed with exit code $?."' EXIT
+
+# mvn build
+mvn clean package -q -Dmaven.javadoc.skip=true -Dspark=2.4 -Dscala=2.11 -Dhive=0.14 -Dhadoop=2.7
+
+# Setup cluster contain of hadoop and spark
+source $GITHUB_WORKSPACE/actions/test-cluster/setup-cluster.sh
+
+#Setup Hibench
+source $GITHUB_WORKSPACE/actions/test-cluster/setup-hibench.sh
+
+echo "========================================="
+echo "Cluster Testing with Spark Version: $SPARK_VERSION"
+echo "========================================="
+
+# run all examples
+source $GITHUB_WORKSPACE/actions/run_all_part_2.sh
diff --git a/...build_and_run_all_spark_3.0_hadoop_3.2.sh → ...nd_run_all_spark_3.0_hadoop_3.2_part_1.sh b/...build_and_run_all_spark_3.0_hadoop_3.2.sh → ...nd_run_all_spark_3.0_hadoop_3.2_part_1.sh
@@ -28,4 +28,4 @@ echo "Cluster Testing with Spark Version: $SPARK_VERSION"
 echo "========================================="
 
 # run all examples
-source $GITHUB_WORKSPACE/bin/run_all.sh
+source $GITHUB_WORKSPACE/actions/run_all_part_1.sh
diff --git a/actions/build_and_run_all_spark_3.0_hadoop_3.2_part_2.sh b/actions/build_and_run_all_spark_3.0_hadoop_3.2_part_2.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+# exit when any command fails
+set -e
+
+export HADOOP_VERSION=3.2.1
+export SPARK_VERSION=3.0.0
+export SPARK_BIN_VERSION=spark3.0
+export SPARK_HADOOP_VERSION=hadoop3.2
+export HIVE_VERSION=3.0.0
+
+# keep track of the last executed command
+trap 'last_command=$current_command; current_command=$BASH_COMMAND' DEBUG
+# echo an error message before exiting
+trap 'echo "\"${last_command}\" command filed with exit code $?."' EXIT
+
+# mvn build
+mvn clean package -q -Dmaven.javadoc.skip=true -Dspark=3.0 -Dscala=2.12
+
+# Setup cluster contain of hadoop and spark
+source $GITHUB_WORKSPACE/actions/test-cluster/setup-cluster.sh
+
+#Setup Hibench
+source $GITHUB_WORKSPACE/actions/test-cluster/setup-hibench.sh
+
+echo "========================================="
+echo "Cluster Testing with Spark Version: $SPARK_VERSION"
+echo "========================================="
+
+# run all examples
+source $GITHUB_WORKSPACE/actions/run_all_part_2.sh
diff --git a/...build_and_run_all_spark_3.1_hadoop_3.2.sh → ...nd_run_all_spark_3.1_hadoop_3.2_part_1.sh b/...build_and_run_all_spark_3.1_hadoop_3.2.sh → ...nd_run_all_spark_3.1_hadoop_3.2_part_1.sh
@@ -28,4 +28,4 @@ echo "Cluster Testing with Spark Version: $SPARK_VERSION"
 echo "========================================="
 
 # run all examples
-source $GITHUB_WORKSPACE/bin/run_all.sh
+source $GITHUB_WORKSPACE/actions/run_all_part_1.sh
diff --git a/actions/build_and_run_all_spark_3.1_hadoop_3.2_part_2.sh b/actions/build_and_run_all_spark_3.1_hadoop_3.2_part_2.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+# exit when any command fails
+set -e
+
+export HADOOP_VERSION=3.2.1
+export SPARK_VERSION=3.1.1
+export SPARK_BIN_VERSION=spark3.1
+export SPARK_HADOOP_VERSION=hadoop3.2
+export HIVE_VERSION=3.0.0
+
+# keep track of the last executed command
+trap 'last_command=$current_command; current_command=$BASH_COMMAND' DEBUG
+# echo an error message before exiting
+trap 'echo "\"${last_command}\" command filed with exit code $?."' EXIT
+
+# mvn build
+mvn clean package -q -Dmaven.javadoc.skip=true -Dspark=3.1 -Dscala=2.12
+
+# Setup cluster contain of hadoop and spark
+source $GITHUB_WORKSPACE/actions/test-cluster/setup-cluster.sh
+
+#Setup Hibench
+source $GITHUB_WORKSPACE/actions/test-cluster/setup-hibench.sh
+
+echo "========================================="
+echo "Cluster Testing with Spark Version: $SPARK_VERSION"
+echo "========================================="
+
+# run all examples
+source $GITHUB_WORKSPACE/actions/run_all_part_2.sh
diff --git a/actions/frameworks.lst b/actions/frameworks.lst
@@ -0,0 +1,2 @@
+hadoop
+spark