From efc86dfa952bd330435e4567deda0273fc8dfd00 Mon Sep 17 00:00:00 2001 From: minmingzhu <45281494+minmingzhu@users.noreply.github.com> Date: Mon, 17 Jul 2023 17:02:24 +0800 Subject: [PATCH] update ubuntu to 20.04 (#730) * update ubuntu to 20.04 Signed-off-by: minmingzhu * update CI actions and reduce Linear dataset Signed-off-by: minmingzhu * Update build_and_run_all_spark_3.1_hadoop_3.2.sh * update spark.conf Signed-off-by: minmingzhu * debug CI Signed-off-by: minmingzhu * debug CI Signed-off-by: minmingzhu * split benchmark list Signed-off-by: minmingzhu * update split benchmark list Signed-off-by: minmingzhu * update setup-cluster.sh Signed-off-by: minmingzhu * Move the files required by CI to the actions directory Signed-off-by: minmingzhu * add benchmarks.lst Signed-off-by: minmingzhu --------- Signed-off-by: minmingzhu --- ...hibench_ci_spark2.4_hadoop_2.7_part_1.yml} | 8 +- .../hibench_ci_spark2.4_hadoop_2.7_part_2.yml | 27 +++++ ...hibench_ci_spark3.0_hadoop_3.2_part_1.yml} | 8 +- .../hibench_ci_spark3.0_hadoop_3.2_part_2.yml | 27 +++++ ...hibench_ci_spark3.1_hadoop_3.2_part_1.yml} | 8 +- .../hibench_ci_spark3.1_hadoop_3.2_part_2.yml | 27 +++++ actions/benchmarks_part_1.lst | 13 ++ actions/benchmarks_part_2.lst | 16 +++ ..._spark_2.4_hadoop_2.7_hive_0.14_part_1.sh} | 2 +- ...l_spark_2.4_hadoop_2.7_hive_0.14_part_2.sh | 31 +++++ ...nd_run_all_spark_3.0_hadoop_3.2_part_1.sh} | 2 +- ...and_run_all_spark_3.0_hadoop_3.2_part_2.sh | 31 +++++ ...nd_run_all_spark_3.1_hadoop_3.2_part_1.sh} | 2 +- ...and_run_all_spark_3.1_hadoop_3.2_part_2.sh | 31 +++++ actions/frameworks.lst | 2 + actions/run_all_part_1.sh | 112 ++++++++++++++++++ actions/run_all_part_2.sh | 112 ++++++++++++++++++ conf/workloads/ml/linear.conf | 2 +- 18 files changed, 445 insertions(+), 16 deletions(-) rename .github/workflows/{hibench_ci_spark2.4_hadoop_2.7.yml => hibench_ci_spark2.4_hadoop_2.7_part_1.yml} (82%) create mode 100644 .github/workflows/hibench_ci_spark2.4_hadoop_2.7_part_2.yml rename .github/workflows/{hibench_ci_spark3.0_hadoop_3.2.yml => hibench_ci_spark3.0_hadoop_3.2_part_1.yml} (83%) create mode 100644 .github/workflows/hibench_ci_spark3.0_hadoop_3.2_part_2.yml rename .github/workflows/{hibench_ci_spark3.1_hadoop_3.2.yml => hibench_ci_spark3.1_hadoop_3.2_part_1.yml} (83%) create mode 100644 .github/workflows/hibench_ci_spark3.1_hadoop_3.2_part_2.yml create mode 100644 actions/benchmarks_part_1.lst create mode 100644 actions/benchmarks_part_2.lst rename actions/{build_and_run_all_spark_2.4_hadoop_2.7_hive_0.14.sh => build_and_run_all_spark_2.4_hadoop_2.7_hive_0.14_part_1.sh} (94%) create mode 100755 actions/build_and_run_all_spark_2.4_hadoop_2.7_hive_0.14_part_2.sh rename actions/{build_and_run_all_spark_3.0_hadoop_3.2.sh => build_and_run_all_spark_3.0_hadoop_3.2_part_1.sh} (94%) create mode 100755 actions/build_and_run_all_spark_3.0_hadoop_3.2_part_2.sh rename actions/{build_and_run_all_spark_3.1_hadoop_3.2.sh => build_and_run_all_spark_3.1_hadoop_3.2_part_1.sh} (94%) create mode 100755 actions/build_and_run_all_spark_3.1_hadoop_3.2_part_2.sh create mode 100644 actions/frameworks.lst create mode 100755 actions/run_all_part_1.sh create mode 100755 actions/run_all_part_2.sh diff --git a/.github/workflows/hibench_ci_spark2.4_hadoop_2.7.yml b/.github/workflows/hibench_ci_spark2.4_hadoop_2.7_part_1.yml similarity index 82% rename from .github/workflows/hibench_ci_spark2.4_hadoop_2.7.yml rename to .github/workflows/hibench_ci_spark2.4_hadoop_2.7_part_1.yml index 569b266de..afca003d1 100644 --- a/.github/workflows/hibench_ci_spark2.4_hadoop_2.7.yml +++ b/.github/workflows/hibench_ci_spark2.4_hadoop_2.7_part_1.yml @@ -4,16 +4,16 @@ on: [push, pull_request] jobs: build: - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up JDK 1.8 uses: actions/setup-java@v1 with: java-version: 1.8 - name: Restore cached dependencies - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: | # /var/cache/apt/archives/*.deb @@ -24,4 +24,4 @@ jobs: ${{ runner.os }}- - name: Cluster spark_2.4_hadoop_2.7_hive_0.14 run: | - ${{github.workspace}}/actions/build_and_run_all_spark_2.4_hadoop_2.7_hive_0.14.sh + ${{github.workspace}}/actions/build_and_run_all_spark_2.4_hadoop_2.7_hive_0.14_part_1.sh diff --git a/.github/workflows/hibench_ci_spark2.4_hadoop_2.7_part_2.yml b/.github/workflows/hibench_ci_spark2.4_hadoop_2.7_part_2.yml new file mode 100644 index 000000000..3b5dba846 --- /dev/null +++ b/.github/workflows/hibench_ci_spark2.4_hadoop_2.7_part_2.yml @@ -0,0 +1,27 @@ +name: HiBench CI SPARK_2.4_HADOOP_2.7 + +on: [push, pull_request] + +jobs: + build: + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v3 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Restore cached dependencies + uses: actions/cache@v3 + with: + path: | + # /var/cache/apt/archives/*.deb + ~/.m2/repository + ~/opt + key: ${{ runner.os }}_spark_2.4_hadoop_2.7 + restore-keys: | + ${{ runner.os }}- + - name: Cluster spark_2.4_hadoop_2.7_hive_0.14 + run: | + ${{github.workspace}}/actions/build_and_run_all_spark_2.4_hadoop_2.7_hive_0.14_part_2.sh diff --git a/.github/workflows/hibench_ci_spark3.0_hadoop_3.2.yml b/.github/workflows/hibench_ci_spark3.0_hadoop_3.2_part_1.yml similarity index 83% rename from .github/workflows/hibench_ci_spark3.0_hadoop_3.2.yml rename to .github/workflows/hibench_ci_spark3.0_hadoop_3.2_part_1.yml index b64f5323a..e4c545605 100644 --- a/.github/workflows/hibench_ci_spark3.0_hadoop_3.2.yml +++ b/.github/workflows/hibench_ci_spark3.0_hadoop_3.2_part_1.yml @@ -4,16 +4,16 @@ on: [push, pull_request] jobs: build: - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up JDK 1.8 uses: actions/setup-java@v1 with: java-version: 1.8 - name: Restore cached dependencies - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: | # /var/cache/apt/archives/*.deb @@ -24,4 +24,4 @@ jobs: ${{ runner.os }}- - name: Cluster spark_3.0_hadoop_3.2_hive_3.0 run: | - ${{github.workspace}}/actions/build_and_run_all_spark_3.0_hadoop_3.2.sh + ${{github.workspace}}/actions/build_and_run_all_spark_3.0_hadoop_3.2_part_1.sh diff --git a/.github/workflows/hibench_ci_spark3.0_hadoop_3.2_part_2.yml b/.github/workflows/hibench_ci_spark3.0_hadoop_3.2_part_2.yml new file mode 100644 index 000000000..e326d28d2 --- /dev/null +++ b/.github/workflows/hibench_ci_spark3.0_hadoop_3.2_part_2.yml @@ -0,0 +1,27 @@ +name: HiBench CI SPARK_3.0_HADOOP_3.2 + +on: [push, pull_request] + +jobs: + build: + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v3 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Restore cached dependencies + uses: actions/cache@v3 + with: + path: | + # /var/cache/apt/archives/*.deb + ~/.m2/repository + ~/opt + key: ${{ runner.os }}_spark_3.0_hadoop_3.2 + restore-keys: | + ${{ runner.os }}- + - name: Cluster spark_3.0_hadoop_3.2_hive_3.0 + run: | + ${{github.workspace}}/actions/build_and_run_all_spark_3.0_hadoop_3.2_part_2.sh diff --git a/.github/workflows/hibench_ci_spark3.1_hadoop_3.2.yml b/.github/workflows/hibench_ci_spark3.1_hadoop_3.2_part_1.yml similarity index 83% rename from .github/workflows/hibench_ci_spark3.1_hadoop_3.2.yml rename to .github/workflows/hibench_ci_spark3.1_hadoop_3.2_part_1.yml index f89f1f78a..313add397 100644 --- a/.github/workflows/hibench_ci_spark3.1_hadoop_3.2.yml +++ b/.github/workflows/hibench_ci_spark3.1_hadoop_3.2_part_1.yml @@ -4,16 +4,16 @@ on: [push, pull_request] jobs: build: - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up JDK 1.8 uses: actions/setup-java@v1 with: java-version: 1.8 - name: Restore cached dependencies - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: | # /var/cache/apt/archives/*.deb @@ -24,4 +24,4 @@ jobs: ${{ runner.os }}- - name: Cluster spark_3.1_hadoop_3.2_hive_3.0 run: | - ${{github.workspace}}/actions/build_and_run_all_spark_3.1_hadoop_3.2.sh + ${{github.workspace}}/actions/build_and_run_all_spark_3.1_hadoop_3.2_part_1.sh diff --git a/.github/workflows/hibench_ci_spark3.1_hadoop_3.2_part_2.yml b/.github/workflows/hibench_ci_spark3.1_hadoop_3.2_part_2.yml new file mode 100644 index 000000000..db78a7b52 --- /dev/null +++ b/.github/workflows/hibench_ci_spark3.1_hadoop_3.2_part_2.yml @@ -0,0 +1,27 @@ +name: HiBench CI SPARK_3.1_HADOOP_3.2 + +on: [push, pull_request] + +jobs: + build: + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v3 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Restore cached dependencies + uses: actions/cache@v3 + with: + path: | + # /var/cache/apt/archives/*.deb + ~/.m2/repository + ~/opt + key: ${{ runner.os }}_spark_3.1_hadoop_3.2 + restore-keys: | + ${{ runner.os }}- + - name: Cluster spark_3.1_hadoop_3.2_hive_3.0 + run: | + ${{github.workspace}}/actions/build_and_run_all_spark_3.1_hadoop_3.2_part_2.sh diff --git a/actions/benchmarks_part_1.lst b/actions/benchmarks_part_1.lst new file mode 100644 index 000000000..8d7a55bd2 --- /dev/null +++ b/actions/benchmarks_part_1.lst @@ -0,0 +1,13 @@ +micro.sleep +micro.sort +micro.terasort +micro.wordcount +micro.repartition +micro.dfsioe + +sql.aggregation +sql.join +sql.scan + +websearch.nutchindexing +websearch.pagerank diff --git a/actions/benchmarks_part_2.lst b/actions/benchmarks_part_2.lst new file mode 100644 index 000000000..b3f49d7b8 --- /dev/null +++ b/actions/benchmarks_part_2.lst @@ -0,0 +1,16 @@ +ml.bayes +ml.kmeans +ml.lr +ml.als +ml.pca +ml.gbt +ml.rf +ml.svd +ml.linear +ml.lda +ml.svm +ml.gmm +ml.correlation +ml.summarizer + +graph.nweight diff --git a/actions/build_and_run_all_spark_2.4_hadoop_2.7_hive_0.14.sh b/actions/build_and_run_all_spark_2.4_hadoop_2.7_hive_0.14_part_1.sh similarity index 94% rename from actions/build_and_run_all_spark_2.4_hadoop_2.7_hive_0.14.sh rename to actions/build_and_run_all_spark_2.4_hadoop_2.7_hive_0.14_part_1.sh index cf31655a7..92905c382 100755 --- a/actions/build_and_run_all_spark_2.4_hadoop_2.7_hive_0.14.sh +++ b/actions/build_and_run_all_spark_2.4_hadoop_2.7_hive_0.14_part_1.sh @@ -28,4 +28,4 @@ echo "Cluster Testing with Spark Version: $SPARK_VERSION" echo "=========================================" # run all examples -source $GITHUB_WORKSPACE/bin/run_all.sh +source $GITHUB_WORKSPACE/actions/run_all_part_1.sh diff --git a/actions/build_and_run_all_spark_2.4_hadoop_2.7_hive_0.14_part_2.sh b/actions/build_and_run_all_spark_2.4_hadoop_2.7_hive_0.14_part_2.sh new file mode 100755 index 000000000..ec37b00ae --- /dev/null +++ b/actions/build_and_run_all_spark_2.4_hadoop_2.7_hive_0.14_part_2.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# exit when any command fails +set -e + +export HADOOP_VERSION=2.7.7 +export SPARK_VERSION=2.4.0 +export SPARK_BIN_VERSION=spark2.4 +export SPARK_HADOOP_VERSION=hadoop2.7 +export HIVE_VERSION=0.14.0 + +# keep track of the last executed command +trap 'last_command=$current_command; current_command=$BASH_COMMAND' DEBUG +# echo an error message before exiting +trap 'echo "\"${last_command}\" command filed with exit code $?."' EXIT + +# mvn build +mvn clean package -q -Dmaven.javadoc.skip=true -Dspark=2.4 -Dscala=2.11 -Dhive=0.14 -Dhadoop=2.7 + +# Setup cluster contain of hadoop and spark +source $GITHUB_WORKSPACE/actions/test-cluster/setup-cluster.sh + +#Setup Hibench +source $GITHUB_WORKSPACE/actions/test-cluster/setup-hibench.sh + +echo "=========================================" +echo "Cluster Testing with Spark Version: $SPARK_VERSION" +echo "=========================================" + +# run all examples +source $GITHUB_WORKSPACE/actions/run_all_part_2.sh diff --git a/actions/build_and_run_all_spark_3.0_hadoop_3.2.sh b/actions/build_and_run_all_spark_3.0_hadoop_3.2_part_1.sh similarity index 94% rename from actions/build_and_run_all_spark_3.0_hadoop_3.2.sh rename to actions/build_and_run_all_spark_3.0_hadoop_3.2_part_1.sh index cb92ef114..4ae930bbd 100755 --- a/actions/build_and_run_all_spark_3.0_hadoop_3.2.sh +++ b/actions/build_and_run_all_spark_3.0_hadoop_3.2_part_1.sh @@ -28,4 +28,4 @@ echo "Cluster Testing with Spark Version: $SPARK_VERSION" echo "=========================================" # run all examples -source $GITHUB_WORKSPACE/bin/run_all.sh +source $GITHUB_WORKSPACE/actions/run_all_part_1.sh diff --git a/actions/build_and_run_all_spark_3.0_hadoop_3.2_part_2.sh b/actions/build_and_run_all_spark_3.0_hadoop_3.2_part_2.sh new file mode 100755 index 000000000..aeaad64ff --- /dev/null +++ b/actions/build_and_run_all_spark_3.0_hadoop_3.2_part_2.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# exit when any command fails +set -e + +export HADOOP_VERSION=3.2.1 +export SPARK_VERSION=3.0.0 +export SPARK_BIN_VERSION=spark3.0 +export SPARK_HADOOP_VERSION=hadoop3.2 +export HIVE_VERSION=3.0.0 + +# keep track of the last executed command +trap 'last_command=$current_command; current_command=$BASH_COMMAND' DEBUG +# echo an error message before exiting +trap 'echo "\"${last_command}\" command filed with exit code $?."' EXIT + +# mvn build +mvn clean package -q -Dmaven.javadoc.skip=true -Dspark=3.0 -Dscala=2.12 + +# Setup cluster contain of hadoop and spark +source $GITHUB_WORKSPACE/actions/test-cluster/setup-cluster.sh + +#Setup Hibench +source $GITHUB_WORKSPACE/actions/test-cluster/setup-hibench.sh + +echo "=========================================" +echo "Cluster Testing with Spark Version: $SPARK_VERSION" +echo "=========================================" + +# run all examples +source $GITHUB_WORKSPACE/actions/run_all_part_2.sh diff --git a/actions/build_and_run_all_spark_3.1_hadoop_3.2.sh b/actions/build_and_run_all_spark_3.1_hadoop_3.2_part_1.sh similarity index 94% rename from actions/build_and_run_all_spark_3.1_hadoop_3.2.sh rename to actions/build_and_run_all_spark_3.1_hadoop_3.2_part_1.sh index 42d3f86a3..bcdbd273a 100755 --- a/actions/build_and_run_all_spark_3.1_hadoop_3.2.sh +++ b/actions/build_and_run_all_spark_3.1_hadoop_3.2_part_1.sh @@ -28,4 +28,4 @@ echo "Cluster Testing with Spark Version: $SPARK_VERSION" echo "=========================================" # run all examples -source $GITHUB_WORKSPACE/bin/run_all.sh +source $GITHUB_WORKSPACE/actions/run_all_part_1.sh diff --git a/actions/build_and_run_all_spark_3.1_hadoop_3.2_part_2.sh b/actions/build_and_run_all_spark_3.1_hadoop_3.2_part_2.sh new file mode 100755 index 000000000..bd9a45177 --- /dev/null +++ b/actions/build_and_run_all_spark_3.1_hadoop_3.2_part_2.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# exit when any command fails +set -e + +export HADOOP_VERSION=3.2.1 +export SPARK_VERSION=3.1.1 +export SPARK_BIN_VERSION=spark3.1 +export SPARK_HADOOP_VERSION=hadoop3.2 +export HIVE_VERSION=3.0.0 + +# keep track of the last executed command +trap 'last_command=$current_command; current_command=$BASH_COMMAND' DEBUG +# echo an error message before exiting +trap 'echo "\"${last_command}\" command filed with exit code $?."' EXIT + +# mvn build +mvn clean package -q -Dmaven.javadoc.skip=true -Dspark=3.1 -Dscala=2.12 + +# Setup cluster contain of hadoop and spark +source $GITHUB_WORKSPACE/actions/test-cluster/setup-cluster.sh + +#Setup Hibench +source $GITHUB_WORKSPACE/actions/test-cluster/setup-hibench.sh + +echo "=========================================" +echo "Cluster Testing with Spark Version: $SPARK_VERSION" +echo "=========================================" + +# run all examples +source $GITHUB_WORKSPACE/actions/run_all_part_2.sh diff --git a/actions/frameworks.lst b/actions/frameworks.lst new file mode 100644 index 000000000..32ecfe2af --- /dev/null +++ b/actions/frameworks.lst @@ -0,0 +1,2 @@ +hadoop +spark \ No newline at end of file diff --git a/actions/run_all_part_1.sh b/actions/run_all_part_1.sh new file mode 100755 index 000000000..b7e8966ea --- /dev/null +++ b/actions/run_all_part_1.sh @@ -0,0 +1,112 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -u + +current_dir=`dirname "$0"` +root_dir=`cd "${current_dir}/.."; pwd` + +. ${root_dir}/bin/functions/color.sh + +for benchmark in `cat $root_dir/actions/benchmarks_part_1.lst`; do + if [[ $benchmark == \#* ]]; then + continue + fi + + echo -e "${UYellow}${BYellow}Prepare ${Yellow}${UYellow}${benchmark} ${BYellow}...${Color_Off}" + benchmark="${benchmark/.//}" + + WORKLOAD=$root_dir/bin/workloads/${benchmark} + echo -e "${BCyan}Exec script: ${Cyan}${WORKLOAD}/prepare/prepare.sh${Color_Off}" + "${WORKLOAD}/prepare/prepare.sh" + + result=$? + if [ $result -ne 0 ] + then + echo "ERROR: ${benchmark} prepare failed!" + exit $result + fi + + for framework in `cat $root_dir/actions/frameworks.lst`; do + if [[ $framework == \#* ]]; then + continue + fi + + if [ $benchmark == "micro/dfsioe" ] && [ $framework == "spark" ]; then + continue + fi + if [ $benchmark == "micro/repartition" ] && [ $framework == "hadoop" ]; then + continue + fi + if [ $benchmark == "websearch/nutchindexing" ] && [ $framework == "spark" ]; then + continue + fi + if [ $benchmark == "graph/nweight" ] && [ $framework == "hadoop" ]; then + continue + fi + if [ $benchmark == "graph/pagerank" ] && [ $framework == "hadoop" ]; then + continue + fi + if [ $benchmark == "ml/lr" ] && [ $framework == "hadoop" ]; then + continue + fi + if [ $benchmark == "ml/als" ] && [ $framework == "hadoop" ]; then + continue + fi + if [ $benchmark == "ml/svm" ] && [ $framework == "hadoop" ]; then + continue + fi + if [ $benchmark == "ml/pca" ] && [ $framework == "hadoop" ]; then + continue + fi + if [ $benchmark == "ml/gbt" ] && [ $framework == "hadoop" ]; then + continue + fi + if [ $benchmark == "ml/rf" ] && [ $framework == "hadoop" ]; then + continue + fi + if [ $benchmark == "ml/svd" ] && [ $framework == "hadoop" ]; then + continue + fi + if [ $benchmark == "ml/linear" ] && [ $framework == "hadoop" ]; then + continue + fi + if [ $benchmark == "ml/lda" ] && [ $framework == "hadoop" ]; then + continue + fi + if [ $benchmark == "ml/gmm" ] && [ $framework == "hadoop" ]; then + continue + fi + if [ $benchmark == "ml/correlation" ] && [ $framework == "hadoop" ]; then + continue + fi + if [ $benchmark == "ml/summarizer" ] && [ $framework == "hadoop" ]; then + continue + fi + + echo -e "${UYellow}${BYellow}Run ${Yellow}${UYellow}${benchmark}/${framework}${Color_Off}" + echo -e "${BCyan}Exec script: ${Cyan}$WORKLOAD/${framework}/run.sh${Color_Off}" + $WORKLOAD/${framework}/run.sh + + result=$? + if [ $result -ne 0 ] + then + echo -e "${On_IRed}ERROR: ${benchmark}/${framework} failed to run successfully.${Color_Off}" + exit $result + fi + done +done + +echo "Run all done!" diff --git a/actions/run_all_part_2.sh b/actions/run_all_part_2.sh new file mode 100755 index 000000000..4607707de --- /dev/null +++ b/actions/run_all_part_2.sh @@ -0,0 +1,112 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -u + +current_dir=`dirname "$0"` +root_dir=`cd "${current_dir}/.."; pwd` + +. ${root_dir}/bin/functions/color.sh + +for benchmark in `cat $root_dir/actions/benchmarks_part_2.lst`; do + if [[ $benchmark == \#* ]]; then + continue + fi + + echo -e "${UYellow}${BYellow}Prepare ${Yellow}${UYellow}${benchmark} ${BYellow}...${Color_Off}" + benchmark="${benchmark/.//}" + + WORKLOAD=$root_dir/bin/workloads/${benchmark} + echo -e "${BCyan}Exec script: ${Cyan}${WORKLOAD}/prepare/prepare.sh${Color_Off}" + "${WORKLOAD}/prepare/prepare.sh" + + result=$? + if [ $result -ne 0 ] + then + echo "ERROR: ${benchmark} prepare failed!" + exit $result + fi + + for framework in `cat $root_dir/actions/frameworks.lst`; do + if [[ $framework == \#* ]]; then + continue + fi + + if [ $benchmark == "micro/dfsioe" ] && [ $framework == "spark" ]; then + continue + fi + if [ $benchmark == "micro/repartition" ] && [ $framework == "hadoop" ]; then + continue + fi + if [ $benchmark == "websearch/nutchindexing" ] && [ $framework == "spark" ]; then + continue + fi + if [ $benchmark == "graph/nweight" ] && [ $framework == "hadoop" ]; then + continue + fi + if [ $benchmark == "graph/pagerank" ] && [ $framework == "hadoop" ]; then + continue + fi + if [ $benchmark == "ml/lr" ] && [ $framework == "hadoop" ]; then + continue + fi + if [ $benchmark == "ml/als" ] && [ $framework == "hadoop" ]; then + continue + fi + if [ $benchmark == "ml/svm" ] && [ $framework == "hadoop" ]; then + continue + fi + if [ $benchmark == "ml/pca" ] && [ $framework == "hadoop" ]; then + continue + fi + if [ $benchmark == "ml/gbt" ] && [ $framework == "hadoop" ]; then + continue + fi + if [ $benchmark == "ml/rf" ] && [ $framework == "hadoop" ]; then + continue + fi + if [ $benchmark == "ml/svd" ] && [ $framework == "hadoop" ]; then + continue + fi + if [ $benchmark == "ml/linear" ] && [ $framework == "hadoop" ]; then + continue + fi + if [ $benchmark == "ml/lda" ] && [ $framework == "hadoop" ]; then + continue + fi + if [ $benchmark == "ml/gmm" ] && [ $framework == "hadoop" ]; then + continue + fi + if [ $benchmark == "ml/correlation" ] && [ $framework == "hadoop" ]; then + continue + fi + if [ $benchmark == "ml/summarizer" ] && [ $framework == "hadoop" ]; then + continue + fi + + echo -e "${UYellow}${BYellow}Run ${Yellow}${UYellow}${benchmark}/${framework}${Color_Off}" + echo -e "${BCyan}Exec script: ${Cyan}$WORKLOAD/${framework}/run.sh${Color_Off}" + $WORKLOAD/${framework}/run.sh + + result=$? + if [ $result -ne 0 ] + then + echo -e "${On_IRed}ERROR: ${benchmark}/${framework} failed to run successfully.${Color_Off}" + exit $result + fi + done +done + +echo "Run all done!" diff --git a/conf/workloads/ml/linear.conf b/conf/workloads/ml/linear.conf index b5c5621da..a28f56993 100644 --- a/conf/workloads/ml/linear.conf +++ b/conf/workloads/ml/linear.conf @@ -1,5 +1,5 @@ hibench.linear.tiny.examples 50000 -hibench.linear.tiny.features 10000 +hibench.linear.tiny.features 1000 hibench.linear.small.examples 100000 hibench.linear.small.features 20000