Es qat support only

.gitignore

@@ -39,3 +39,4 @@ columnar_format_qat_wrapper/target/
 javah/
 jni-header/
 target/
+common/

es_qat_wrapper/8.0.0/README.md

@@ -0,0 +1,134 @@
+#How to deploy the Elasticsearch with QAT
+
+##I. Install the QATzip and QAT Drivers
+
+1. Update 
+
+       $ sudo yum update
+
+2. Create new file intel-qat.repo  in  /etc/yum.repos.d, the content is as follows: 
+
+        [intel-qat]
+        name=Intel QAT
+        baseurl=https://download.01.org/QAT/repo
+        gpgcheck=0
+3. Install
+
+        $ sudo yum install QAT
+        $ sudo yum install QATzip
+
+4. Configure huge page related setting
+
+       $ sudo cat  /etc/sysctl.conf    
+      ensure the line vm.nr_hugepages=512 exists in the file  	      
+
+       $ sudo sysctl -p
+       $ sudo cat /proc/meminfo | grep -i hugepages_total
+                 HugePages_Total: 512 
+
+5. Config QAT Codec
+
+       $ chmod 777 $ICP_ROOT/build
+       $ chmod 777 $ICP_ROOT/build/libusdm_drv_s.so 
+       $ chmod 777 /dev/qat_adf_ctl
+       $ chmod 777 /dev/qat_dev_processes
+       $ chmod 777 /dev/uio*
+       $ chmod 777 /dev/usdm_drv
+       $ chmod -R 777 /dev/hugepages
+
+     The $ICP_ROOT/  is in  /opt/intel/QAT by default, and the $DZ_ROOT is in
+/opt/intel/QATzip  by default
+6. Start the qat service 
+
+       $ sudo service qat_service start 
+7. Run the following command to check if the QATzip is set up correctly for compressing or decompressing files.
+
+       $ qzip -k $your_input_file
+##II. Copy the .so files to the /lib64   
+       $ sudo cp libQatCodecEs.so /lib64
+       $ sudo cp libqatcodec.so /lib64 
+
+##III. Unzip the Elasticsearch binary file
+        $ tar -zcvf elasticsearch-8.0.0-SNAPSHOT-linux-x86_64.tar.gz
+##IV. Start the Elaticsearch service
+1. Config the elasticsearch settings
+
+       $ vim config/elasticsearch.yml
+2. Start the Elasticsearch service
+
+       $ bin/elasticsearch
+
+#How to build 
+## I. Set the environments
+1. Install JDK
+
+we need jdk12 for Elasticsearch 8.0.0 and jdk8 for IntelQatCodec
+
+        1. $ sudo yum install -y curl
+        2. $ curl -O https://download.java.net/java/GA/jdk12.0.1/69cfe15208a647278a19ef0990eea691/12/GPL/openjdk-12.0.1_linux-x64_bin.tar.gz  
+        3. $ tar xvf openjdk-12.0.1_linux-x64_bin.tar.gz
+        4. $ sudo mv jdk-12.0.1 /opt/
+        5. Configure the java environment
+	         export JAVA_HOME=/root/jdk-12.0.1
+             export PATH=$JAVA_HOME/bin:$PATH
+        6. Confirm the java version
+   	         $ java -version
+
+2.Install Gradle
+
+    1. Download Gradle
+         $ wget https://services.gradle.org/distributions/gradle-5.2.1-bin.zip -P /tmp
+    2. Install Gradle
+         $ sudo unzip -d /opt/gradle /tmp/gradle-*.zip
+    3. Setup environment variables
+         export GRADLE_HOME=/opt/gradle/gradle-5.2.1
+         export PATH=${GRADLE_HOME}/bin:${PATH}
+    4. Check if the Gradle install was successful.
+         $gradle -v
+3.Install maven
+
+    1. $ sudo yum install maven
+    2. $ mvn -version
+
+## II. Build IntelQATCodec
+
+    $ cd /path/to/IntelQatCodec
+    $ mvn clean install -Dqatzip.libs=/opt/intel/QATzip -Dqatzip.src=/opt/intel/QATzip -DskipTests
+ Then we can get the following files that we need.
+
+      path/to/IntelQatCodec/lucene_qat_wrapper/target/lucene_qat_wrapper.jar 
+      path/to/IntelQatCodec/lucene_qat_wrapper/target/libqatcodec.so
+      path/to/IntelQatCodec/es_qat_wrapper/target/es_qat_wrapper.jar
+      path/to/IntelQatCodec/es_qat_wrapper/target/classes/com/intel/qat/native/lib/Linux/amd64/libQatCodecEs.so
+
+ We need to copy these files to other places:
+
+      $ sudo cp libqatcodec.so libQatCodecEs.so /lib64
+      $ cp lucene_qat_wrapper.jar es_qat_wrapper/8.0.0/lucene-8.2.0/lucene/lib/
+      $ cp es_qat_wrapper.jar es_qat_wrapper/8.0.0/elasticsearch/buildSrc/libs/
+
+## III. Apply the lucene patch     
+
+      $ cd /path/to/IntelQATCodec/elasticsearch_qat_wrapper/8.0.0
+      $ ./apply_lucene_jars.sh 8.2.0 /path/to/IntelQATCodec
+
+##IV. Build the lucene in target folder
+
+    $ cd /path/to/IntelQATCodec/elasticsearch_qat_wrapper/8.0.0/target/LUCENE
+    $ ant -autoproxy clean compile 
+Then we need to copy the jars to the ./elasticsearch_qat_wrapper/8.0.0/elasticsearch/buildSrc/libs/
+
+     lucene-core-8.2.0-SNAPSHOT.jar
+     lucene_qat_wrapper.jar
+##V. Apply the elastcsearch patch
+      $ cd /path/to/IntelQATCodec/elasticsearch_qat_wrapper/8.0.0
+      $ ./apply_es_jars.sh 8.0.0 /path/to/IntelQATCodec
+
+##VI. Build the  elasticsearch in target folder
+
+      $ java -version
+To make sure the java version is 11+.(We use jdk12)
+
+      $ cd /path/to/IntelQATCodec/elasticsearch_qat_wrapper/8.0.0/target/elasticsearch
+      $ ./gradlew :distribution:archives:linux-tar:assemble --parallel
+and then we can get the binary files in the ./distribution/archives/linux-tar/build/distributions/

es_qat_wrapper/8.0.0/apply_es_jars.sh

@@ -0,0 +1,89 @@
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *     http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+#!/bin/bash
+
+declare -a supported_Elasticsearch_versions=("8.0.0")
+declare -A es_lucene_version_m=(["8.0.0"]="8.2.0")
+
+# Repo Address
+ES_REPO=https://github.com/Intel-bigdata/elasticsearch.git
+ES_version_base="8.0.0"
+ES_baseCommitID=dc6e29a949d6a68cbc57d71c9fd2a3328ed71f17
+
+
+ES_version=$1
+QATCodec_SRC_DIR=$2
+
+ES_QAT_DIR=$QATCodec_SRC_DIR/es_qat_wrapper/${ES_version_base}/elasticsearch
+TARGET_DIR=$QATCodec_SRC_DIR/es_qat_wrapper/${ES_version_base}/target
+ES_SRC_DIR=$TARGET_DIR/elasticsearch
+
+
+function clone_repo(){
+  echo "Clone from Repo $1"
+  git clone $1 $ES_SRC_DIR
+}
+
+function checkout_branch(){
+  pushd $ES_SRC_DIR
+  Branch_name=VERSION-${ES_version_base}
+  git checkout -b $Branch_name $ES_baseCommitID
+  popd
+}
+
+function usage(){
+  printf "Usage: sh build_es_jars.sh es_version [PATH/TO/QAT_Codec_SRC]\n (e.g. sh build_es_jars.sh 8.0.0 /home/user/workspace/QATCodec)\n"
+  exit 1
+}
+
+function check_ES_version(){
+  valid_version=false
+  for v in $supported_Elasticsearch_versions
+  do
+  	if [ "$v" =  "$1" ]; then
+		valid_version=true
+    		break;
+	fi
+  done
+  if ! $valid_version ; then
+  	printf "Unsupported elasticsearch version $1, current supported versions include: ${supported_Elasticsearch_versions[@]} \n"
+  	exit 1
+  fi
+}
+
+apply_patch_to_es(){
+  pushd $TARGET_DIR
+  clone_repo $ES_REPO
+  checkout_branch
+  echo yes | cp -rf $ES_QAT_DIR/ $TARGET_DIR/
+  popd
+}
+if [ "$#" -ne 2 ]; then
+  usage
+fi
+
+
+check_ES_version $ES_version
+
+if [ -d $TARGET_DIR ]; then
+  echo "$TARGET_DIR is not clean"
+else
+  mkdir -p $TARGET_DIR
+fi
+
+apply_patch_to_es

es_qat_wrapper/8.0.0/apply_lucene_jars.sh

@@ -0,0 +1,91 @@
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *     http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+#!/bin/bash
+
+declare -a supported_Lucene_versions=("8.2.0")
+#declare -A es_lucene_version_m=(["8.0.0"]="8.2.0")
+
+
+# Repo Address
+LUCENE_REPO=https://github.com/apache/lucene-solr.git
+ES_version_base="8.0.0"
+Lucene_version_base="8.2.0"
+Lucene_baseCommitID=e8494222be8a44678b43f907b47accbda3ebd401
+
+
+LUCENE_version=$1
+QATCodec_SRC_DIR=$2
+
+LUCENE_QAT_DIR=$QATCodec_SRC_DIR/es_qat_wrapper/${ES_version_base}/lucene-${Lucene_version_base}/lucene
+TARGET_DIR=$QATCodec_SRC_DIR/es_qat_wrapper/${ES_version_base}/target
+LUCENE_SRC_DIR=$TARGET_DIR/LUCENE
+
+
+function clone_repo(){
+  echo "Clone from Repo $1"
+  git clone $1 $LUCENE_SRC_DIR
+}
+
+function checkout_branch(){
+  pushd $LUCENE_SRC_DIR
+  Branch_name=VERSION-${Lucene_version_base}
+  git checkout -b $Branch_name $Lucene_baseCommitID
+  popd
+}
+
+function usage(){
+  printf "Usage: sh build_lucene_jars.sh lucene_version [PATH/TO/QAT_Codec_SRC]\n (e.g. sh build_lucene_jars.sh 8.2.0 /home/user/workspace/QATCodec)\n"
+  exit 1
+}
+
+function check_LUCENE_version(){
+  valid_version=false
+  for v in $supported_Lucene_versions
+  do
+  	if [ "$v" =  "$1" ]; then
+		valid_version=true
+    		break;
+	fi
+  done
+  if ! $valid_version ; then
+  	printf "Unsupported Lucene version $1, current supported versions include: ${supported_Lucene_versions[@]} \n"
+  	exit 1
+  fi
+}
+
+apply_patch_to_lucene(){
+  pushd $TARGET_DIR
+  clone_repo $LUCENE_REPO
+  checkout_branch
+  echo yes | cp -rf $LUCENE_QAT_DIR $LUCENE_SRC_DIR/
+  popd
+}
+if [ "$#" -ne 2 ]; then
+  usage
+fi
+
+
+check_LUCENE_version $LUCENE_version
+
+if [ -d $TARGET_DIR ]; then
+  echo "$TARGET_DIR is not clean"
+else
+  mkdir -p $TARGET_DIR
+fi
+
+apply_patch_to_lucene