-
Notifications
You must be signed in to change notification settings - Fork 16
Es qat support only #10
base: master
Are you sure you want to change the base?
Changes from 20 commits
f0769a6
d3cc105
9d84deb
996798b
ac62155
5492c0f
d3f11b2
c4ad734
24552f1
67d5a5e
cc2d8bd
3a4d88b
282e7b2
bacaeb1
12d0f87
a35f856
7852409
6ddf108
85996a9
14e02b6
0c64080
c457064
87e4e02
07c813c
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -39,3 +39,4 @@ columnar_format_qat_wrapper/target/ | |
javah/ | ||
jni-header/ | ||
target/ | ||
common/ |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,134 @@ | ||
#How to deploy the Elasticsearch with QAT | ||
|
||
##I. Install the QATzip and QAT Drivers | ||
|
||
1. Update | ||
|
||
$ sudo yum update | ||
|
||
2. Create new file intel-qat.repo in /etc/yum.repos.d, the content is as follows: | ||
|
||
[intel-qat] | ||
name=Intel QAT | ||
baseurl=https://download.01.org/QAT/repo | ||
gpgcheck=0 | ||
3. Install | ||
|
||
$ sudo yum install QAT | ||
$ sudo yum install QATzip | ||
|
||
4. Configure huge page related setting | ||
|
||
$ sudo cat /etc/sysctl.conf | ||
ensure the line vm.nr_hugepages=512 exists in the file | ||
|
||
$ sudo sysctl -p | ||
$ sudo cat /proc/meminfo | grep -i hugepages_total | ||
HugePages_Total: 512 | ||
|
||
5. Config QAT Codec | ||
|
||
$ chmod 777 $ICP_ROOT/build | ||
$ chmod 777 $ICP_ROOT/build/libusdm_drv_s.so | ||
$ chmod 777 /dev/qat_adf_ctl | ||
$ chmod 777 /dev/qat_dev_processes | ||
$ chmod 777 /dev/uio* | ||
$ chmod 777 /dev/usdm_drv | ||
$ chmod -R 777 /dev/hugepages | ||
|
||
The $ICP_ROOT/ is in /opt/intel/QAT by default, and the $DZ_ROOT is in | ||
/opt/intel/QATzip by default | ||
6. Start the qat service | ||
|
||
$ sudo service qat_service start | ||
7. Run the following command to check if the QATzip is set up correctly for compressing or decompressing files. | ||
|
||
$ qzip -k $your_input_file | ||
##II. Copy the .so files to the /lib64 | ||
$ sudo cp libQatCodecEs.so /lib64 | ||
$ sudo cp libqatcodec.so /lib64 | ||
|
||
##III. Unzip the Elasticsearch binary file | ||
$ tar -zcvf elasticsearch-8.0.0-SNAPSHOT-linux-x86_64.tar.gz | ||
##IV. Start the Elaticsearch service | ||
1. Config the elasticsearch settings | ||
|
||
$ vim config/elasticsearch.yml | ||
2. Start the Elasticsearch service | ||
|
||
$ bin/elasticsearch | ||
|
||
#How to build | ||
## I. Set the environments | ||
1. Install JDK | ||
|
||
we need jdk12 for Elasticsearch 8.0.0 and jdk8 for IntelQatCodec | ||
|
||
1. $ sudo yum install -y curl | ||
2. $ curl -O https://download.java.net/java/GA/jdk12.0.1/69cfe15208a647278a19ef0990eea691/12/GPL/openjdk-12.0.1_linux-x64_bin.tar.gz | ||
3. $ tar xvf openjdk-12.0.1_linux-x64_bin.tar.gz | ||
4. $ sudo mv jdk-12.0.1 /opt/ | ||
5. Configure the java environment | ||
export JAVA_HOME=/root/jdk-12.0.1 | ||
export PATH=$JAVA_HOME/bin:$PATH | ||
6. Confirm the java version | ||
$ java -version | ||
|
||
2.Install Gradle | ||
|
||
1. Download Gradle | ||
$ wget https://services.gradle.org/distributions/gradle-5.2.1-bin.zip -P /tmp | ||
2. Install Gradle | ||
$ sudo unzip -d /opt/gradle /tmp/gradle-*.zip | ||
3. Setup environment variables | ||
export GRADLE_HOME=/opt/gradle/gradle-5.2.1 | ||
export PATH=${GRADLE_HOME}/bin:${PATH} | ||
4. Check if the Gradle install was successful. | ||
$gradle -v | ||
3.Install maven | ||
|
||
1. $ sudo yum install maven | ||
2. $ mvn -version | ||
|
||
## II. Build IntelQATCodec | ||
|
||
$ cd /path/to/IntelQatCodec | ||
$ mvn clean install -Dqatzip.libs=/opt/intel/QATzip -Dqatzip.src=/opt/intel/QATzip -DskipTests | ||
Then we can get the following files that we need. | ||
|
||
path/to/IntelQatCodec/lucene_qat_wrapper/target/lucene_qat_wrapper.jar | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Rename |
||
path/to/IntelQatCodec/lucene_qat_wrapper/target/libqatcodec.so | ||
path/to/IntelQatCodec/es_qat_wrapper/target/es_qat_wrapper.jar | ||
path/to/IntelQatCodec/es_qat_wrapper/target/classes/com/intel/qat/native/lib/Linux/amd64/libQatCodecEs.so | ||
|
||
We need to copy these files to other places: | ||
|
||
$ sudo cp libqatcodec.so libQatCodecEs.so /lib64 | ||
$ cp lucene_qat_wrapper.jar es_qat_wrapper/8.0.0/lucene-8.2.0/lucene/lib/ | ||
$ cp es_qat_wrapper.jar es_qat_wrapper/8.0.0/elasticsearch/buildSrc/libs/ | ||
|
||
## III. Apply the lucene patch | ||
|
||
$ cd /path/to/IntelQATCodec/elasticsearch_qat_wrapper/8.0.0 | ||
$ ./apply_lucene_jars.sh 8.2.0 /path/to/IntelQATCodec | ||
|
||
##IV. Build the lucene in target folder | ||
|
||
$ cd /path/to/IntelQATCodec/elasticsearch_qat_wrapper/8.0.0/target/LUCENE | ||
$ ant -autoproxy clean compile | ||
Then we need to copy the jars to the ./elasticsearch_qat_wrapper/8.0.0/elasticsearch/buildSrc/libs/ | ||
|
||
lucene-core-8.2.0-SNAPSHOT.jar | ||
lucene_qat_wrapper.jar | ||
##V. Apply the elastcsearch patch | ||
$ cd /path/to/IntelQATCodec/elasticsearch_qat_wrapper/8.0.0 | ||
$ ./apply_es_jars.sh 8.0.0 /path/to/IntelQATCodec | ||
|
||
##VI. Build the elasticsearch in target folder | ||
|
||
$ java -version | ||
To make sure the java version is 11+.(We use jdk12) | ||
|
||
$ cd /path/to/IntelQATCodec/elasticsearch_qat_wrapper/8.0.0/target/elasticsearch | ||
$ ./gradlew :distribution:archives:linux-tar:assemble --parallel | ||
and then we can get the binary files in the ./distribution/archives/linux-tar/build/distributions/ |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,89 @@ | ||
#/** | ||
# * Licensed to the Apache Software Foundation (ASF) under one | ||
# * or more contributor license agreements. See the NOTICE file | ||
# * distributed with this work for additional information | ||
# * regarding copyright ownership. The ASF licenses this file | ||
# * to you under the Apache License, Version 2.0 (the | ||
# * "License"); you may not use this file except in compliance | ||
# * with the License. You may obtain a copy of the License at | ||
# * | ||
# * http://www.apache.org/licenses/LICENSE-2.0 | ||
# * | ||
# * Unless required by applicable law or agreed to in writing, software | ||
# * distributed under the License is distributed on an "AS IS" BASIS, | ||
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# * See the License for the specific language governing permissions and | ||
# * limitations under the License. | ||
# */ | ||
#!/bin/bash | ||
|
||
declare -a supported_Elasticsearch_versions=("8.0.0") | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It seems 8.0.0 has not been released. We should use a released version. https://www.elastic.co/guide/en/elasticsearch/reference/current/es-release-notes.html |
||
declare -A es_lucene_version_m=(["8.0.0"]="8.2.0") | ||
|
||
# Repo Address | ||
ES_REPO=https://github.com/Intel-bigdata/elasticsearch.git | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can we use directly source code from ES official website? |
||
ES_version_base="8.0.0" | ||
ES_baseCommitID=dc6e29a949d6a68cbc57d71c9fd2a3328ed71f17 | ||
|
||
|
||
ES_version=$1 | ||
QATCodec_SRC_DIR=$2 | ||
|
||
ES_QAT_DIR=$QATCodec_SRC_DIR/es_qat_wrapper/${ES_version_base}/elasticsearch | ||
TARGET_DIR=$QATCodec_SRC_DIR/es_qat_wrapper/${ES_version_base}/target | ||
ES_SRC_DIR=$TARGET_DIR/elasticsearch | ||
|
||
|
||
function clone_repo(){ | ||
echo "Clone from Repo $1" | ||
git clone $1 $ES_SRC_DIR | ||
} | ||
|
||
function checkout_branch(){ | ||
pushd $ES_SRC_DIR | ||
Branch_name=VERSION-${ES_version_base} | ||
git checkout -b $Branch_name $ES_baseCommitID | ||
popd | ||
} | ||
|
||
function usage(){ | ||
printf "Usage: sh build_es_jars.sh es_version [PATH/TO/QAT_Codec_SRC]\n (e.g. sh build_es_jars.sh 8.0.0 /home/user/workspace/QATCodec)\n" | ||
exit 1 | ||
} | ||
|
||
function check_ES_version(){ | ||
valid_version=false | ||
for v in $supported_Elasticsearch_versions | ||
do | ||
if [ "$v" = "$1" ]; then | ||
valid_version=true | ||
break; | ||
fi | ||
done | ||
if ! $valid_version ; then | ||
printf "Unsupported elasticsearch version $1, current supported versions include: ${supported_Elasticsearch_versions[@]} \n" | ||
exit 1 | ||
fi | ||
} | ||
|
||
apply_patch_to_es(){ | ||
pushd $TARGET_DIR | ||
clone_repo $ES_REPO | ||
checkout_branch | ||
echo yes | cp -rf $ES_QAT_DIR/ $TARGET_DIR/ | ||
popd | ||
} | ||
if [ "$#" -ne 2 ]; then | ||
usage | ||
fi | ||
|
||
|
||
check_ES_version $ES_version | ||
|
||
if [ -d $TARGET_DIR ]; then | ||
echo "$TARGET_DIR is not clean" | ||
else | ||
mkdir -p $TARGET_DIR | ||
fi | ||
|
||
apply_patch_to_es |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,91 @@ | ||
#/** | ||
# * Licensed to the Apache Software Foundation (ASF) under one | ||
# * or more contributor license agreements. See the NOTICE file | ||
# * distributed with this work for additional information | ||
# * regarding copyright ownership. The ASF licenses this file | ||
# * to you under the Apache License, Version 2.0 (the | ||
# * "License"); you may not use this file except in compliance | ||
# * with the License. You may obtain a copy of the License at | ||
# * | ||
# * http://www.apache.org/licenses/LICENSE-2.0 | ||
# * | ||
# * Unless required by applicable law or agreed to in writing, software | ||
# * distributed under the License is distributed on an "AS IS" BASIS, | ||
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# * See the License for the specific language governing permissions and | ||
# * limitations under the License. | ||
# */ | ||
#!/bin/bash | ||
|
||
declare -a supported_Lucene_versions=("8.2.0") | ||
#declare -A es_lucene_version_m=(["8.0.0"]="8.2.0") | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. remove this. |
||
|
||
|
||
# Repo Address | ||
LUCENE_REPO=https://github.com/apache/lucene-solr.git | ||
ES_version_base="8.0.0" | ||
Lucene_version_base="8.2.0" | ||
Lucene_baseCommitID=e8494222be8a44678b43f907b47accbda3ebd401 | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Please use git release tag instead of commit ID. |
||
|
||
|
||
LUCENE_version=$1 | ||
QATCodec_SRC_DIR=$2 | ||
|
||
LUCENE_QAT_DIR=$QATCodec_SRC_DIR/es_qat_wrapper/${ES_version_base}/lucene-${Lucene_version_base}/lucene | ||
TARGET_DIR=$QATCodec_SRC_DIR/es_qat_wrapper/${ES_version_base}/target | ||
LUCENE_SRC_DIR=$TARGET_DIR/LUCENE | ||
|
||
|
||
function clone_repo(){ | ||
echo "Clone from Repo $1" | ||
git clone $1 $LUCENE_SRC_DIR | ||
} | ||
|
||
function checkout_branch(){ | ||
pushd $LUCENE_SRC_DIR | ||
Branch_name=VERSION-${Lucene_version_base} | ||
git checkout -b $Branch_name $Lucene_baseCommitID | ||
popd | ||
} | ||
|
||
function usage(){ | ||
printf "Usage: sh build_lucene_jars.sh lucene_version [PATH/TO/QAT_Codec_SRC]\n (e.g. sh build_lucene_jars.sh 8.2.0 /home/user/workspace/QATCodec)\n" | ||
exit 1 | ||
} | ||
|
||
function check_LUCENE_version(){ | ||
valid_version=false | ||
for v in $supported_Lucene_versions | ||
do | ||
if [ "$v" = "$1" ]; then | ||
valid_version=true | ||
break; | ||
fi | ||
done | ||
if ! $valid_version ; then | ||
printf "Unsupported Lucene version $1, current supported versions include: ${supported_Lucene_versions[@]} \n" | ||
exit 1 | ||
fi | ||
} | ||
|
||
apply_patch_to_lucene(){ | ||
pushd $TARGET_DIR | ||
clone_repo $LUCENE_REPO | ||
checkout_branch | ||
echo yes | cp -rf $LUCENE_QAT_DIR $LUCENE_SRC_DIR/ | ||
popd | ||
} | ||
if [ "$#" -ne 2 ]; then | ||
usage | ||
fi | ||
|
||
|
||
check_LUCENE_version $LUCENE_version | ||
|
||
if [ -d $TARGET_DIR ]; then | ||
echo "$TARGET_DIR is not clean" | ||
else | ||
mkdir -p $TARGET_DIR | ||
fi | ||
|
||
apply_patch_to_lucene |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We can keep it in the doc. But I would suggest to change it to bash script.