merge from apache master

.asf.yaml

@@ -0,0 +1,42 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.  
+#
+
+notifications:
+  commits: [email protected]
+  issues: [email protected]
+  pullrequests: [email protected]
+
+github:
+  description: High performance data store solution
+  homepage: carbondata.apache.org
+  labels:
+    - big-data
+    - data-format
+    - scala
+    - java
+    - carbondata
+    - apache
+    - spark
+    - hadoop
+  features:
+    issues: true
+    projects: true
+    wiki: true
+  enabled_merge_buttons:
+    squash:  true
+    merge:   true
+    rebase:  false

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,21 +1,15 @@
-Be sure to do all of the following to help us incorporate your contribution
-quickly and easily:
-
- - [ ] Make sure the PR title is formatted like:
-   `[CARBONDATA-<Jira issue #>] Description of pull request`
- - [ ] Make sure tests pass via `mvn clean verify`. (Even better, enable
-       Travis-CI on your fork and ensure the whole test matrix passes).
- - [ ] Replace `<Jira issue #>` in the title with the actual Jira issue
-       number, if there is one.
- - [ ] If this contribution is large, please file an Apache
-       [Individual Contributor License Agreement](https://www.apache.org/licenses/icla.txt).
- - [ ] Testing done
+ ### Why is this PR needed?
+
 
-        Please provide details on 
-        - Whether new unit test cases have been added or why no new tests are required?
-        - What manual testing you have done?
-        - Any additional information to help reviewers in testing this change.
-
- - [ ] For large changes, please consider breaking it into sub-tasks under an umbrella JIRA. 
-
----
+ ### What changes were proposed in this PR?
+
+
+ ### Does this PR introduce any user interface change?
+ - No
+ - Yes. (please explain the change and update document)
+
+ ### Is any new testcase added?
+ - No
+ - Yes
+
+

.github/workflows/build-test.yml

@@ -0,0 +1,49 @@
+name: Build Test
+
+on: [push, pull_request]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        spark-profile: [ 'spark-2.3','spark-2.4','spark-3.1' ]
+    name: Build with spark-version ${{ matrix.spark-profile }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up JDK
+        uses: actions/setup-java@v4
+        with:
+          java-version: 8
+          distribution: temurin
+      - name: Cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.m2/repository
+            /usr/local/bin/thrift
+          key: ${{ runner.os }}-maven
+          restore-keys: |
+            ${{ runner.os }}-maven
+      - name: setup-thrift
+        run: |
+          if [ ! -f "/usr/local/bin/thrift" ];then
+            echo "build thrift binary"
+            sudo apt-get update -qq
+            sudo apt-get install -qq protobuf-compiler
+            sudo apt-get install -qq libboost-dev libboost-test-dev libboost-program-options-dev libevent-dev automake libtool flex bison             pkg-config g++ libssl-dev
+            wget -qO- https://archive.apache.org/dist/thrift/0.20.0/thrift-0.20.0.tar.gz | tar zxf -
+            cd thrift-0.20.0/
+            chmod +x ./configure
+            ./configure --disable-libs
+            sudo make -j4 install
+          else
+            echo "use cache thrift binary"
+          fi
+      - name: install
+        run: |
+          mvn clean install -DskipTests -Pbuild-with-format -P${{ matrix.spark-profile }}

.github/workflows/maven-publish.yml

@@ -0,0 +1,34 @@
+# This workflow will build a package using Maven and then publish it to GitHub packages when a release is created
+# For more information see: https://github.com/actions/setup-java/blob/main/docs/advanced-usage.md#apache-maven-with-a-settings-path
+
+name: Maven Package
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up JDK 18
+      uses: actions/setup-java@v3
+      with:
+        java-version: '18'
+        distribution: 'temurin'
+        server-id: github # Value of the distributionManagement/repository/id field of the pom.xml
+        settings-path: ${{ github.workspace }} # location for the settings.xml file
+
+    - name: Build with Maven
+      run: mvn -B package --file pom.xml
+
+    - name: Publish to GitHub Packages Apache Maven
+      run: mvn deploy -s $GITHUB_WORKSPACE/settings.xml
+      env:
+        GITHUB_TOKEN: ${{ github.token }}

.gitignore

@@ -12,7 +12,18 @@
 .settings
 .cache
 target/
+store/CSDK/cmake-build-debug/*
 .project
 .classpath
+.DS_Store
 metastore_db/
-derby.log
+derby.log
+python/.idea/
+*/.cache-main
+*/.cache-tests
+*/*/.cache-main
+*/*/.cache-tests
+*/*/*/.cache-main
+*/*/*/.cache-tests
+*.flattened-pom.xml
+python/pycarbon/.pylintrc

LICENSE

@@ -200,3 +200,16 @@
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
+
+
+   ---------------------------------------------------------------------------
+   This product bundles various third-party components under other open source licenses.
+   This section summarizes those components and their licenses. See licenses-binary/
+   for text of these licenses.
+
+   BSD 2-Clause
+   ------------
+
+   com.github.luben:zstd-jni
+
+   com.github.paul-hammant:paranamer

NOTICE

@@ -1,5 +1,5 @@
-Apache CarbonData (incubating)
-Copyright 2016-2017 The Apache Software Foundation
+Apache CarbonData
+Copyright 2016 and onwards The Apache Software Foundation.
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).

README.md

@@ -1,67 +1,106 @@
 <!--
-    Licensed to the Apache Software Foundation (ASF) under one
-    or more contributor license agreements.  See the NOTICE file
-    distributed with this work for additional information
-    regarding copyright ownership.  The ASF licenses this file
-    to you under the Apache License, Version 2.0 (the
-    "License"); you may not use this file except in compliance
-    with the License.  You may obtain a copy of the License at
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to you under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
 
       http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing,
-    software distributed under the License is distributed on an
-    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-    KIND, either express or implied.  See the License for the
-    specific language governing permissions and limitations
-    under the License.
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
 -->
 
 <img src="/docs/images/CarbonData_logo.png" width="200" height="40">
 
-Apache CarbonData(incubating) is an indexed columnar data format for fast analytics on big data platform, e.g.Apache Hadoop, Apache Spark, etc.
+Apache CarbonData is an indexed columnar data store solution for fast analytics on big data platform, e.g. Apache Hadoop, Apache Spark, etc.
 
 You can find the latest CarbonData document and learn more at:
-[http://carbondata.incubator.apache.org](http://carbondata.incubator.apache.org/)
+[https://carbondata.apache.org](https://carbondata.apache.org/)
 
 [CarbonData cwiki](https://cwiki.apache.org/confluence/display/CARBONDATA/)
 
 ## Status
-[![Build Status](https://travis-ci.org/apache/incubator-carbondata.svg?branch=master)](https://travis-ci.org/apache/incubator-carbondata)
+Spark2.4:
+[![Coverage Status](https://coveralls.io/repos/github/apache/carbondata/badge.svg?branch=master)](https://coveralls.io/github/apache/carbondata?branch=master)
+<a href="https://scan.coverity.com/projects/carbondata">
+  <img alt="Coverity Scan Build Status"
+       src="https://scan.coverity.com/projects/13444/badge.svg"/>
+</a>
 
 ## Features
-CarbonData file format is a columnar store in HDFS, it has many features that a modern columnar format has, such as splittable, compression schema ,complex data type etc, and CarbonData has following unique features:
-* Stores data along with index: it can significantly accelerate query performance and reduces the I/O scans and CPU resources, where there are filters in the query.  CarbonData index consists of multiple level of indices, a processing framework can leverage this index to reduce the task it needs to schedule and process, and it can also do skip scan in more finer grain unit (called blocklet) in task side scanning instead of scanning the whole file. 
-* Operable encoded data :Through supporting efficient compression and global encoding schemes, can query on compressed/encoded data, the data can be converted just before returning the results to the users, which is "late materialized". 
-* Supports for various use cases with one single Data format : like interactive OLAP-style query, Sequential Access (big scan), Random Access (narrow scan). 
+CarbonData file format is a columnar store in HDFS, it has many features that a modern columnar format has, such as splittable, compression schema, complex data type etc, and CarbonData has following unique features:
+* Stores data along with index: it can significantly accelerate query performance and reduces the I/O scans and CPU resources, where there are filters in the query.  CarbonData index consists of multiple level of indices, a processing framework can leverage this index to reduce the task it needs to schedule and process, and it can also do skip scan in more finer grain unit (called blocklet) in task side scanning instead of scanning the whole file.
+* Operable encoded data: through supporting efficient compression and global encoding schemes, can query on compressed/encoded data, the data can be converted just before returning the results to the users, which is "late materialized".
+* Supports for various use cases with one single Data format : like interactive OLAP-style query, Sequential Access (big scan), Random Access (narrow scan).
 
 ## Building CarbonData
-CarbonData is built using Apache Maven, to [build CarbonData](https://github.com/apache/incubator-carbondata/blob/master/build)
+CarbonData is built using Apache Maven, to [build CarbonData](https://github.com/apache/carbondata/blob/master/build)
 
 ## Online Documentation
-* [Quick Start](https://github.com/apache/incubator-carbondata/blob/master/docs/quick-start-guide.md)
-* [CarbonData File Structure](https://github.com/apache/incubator-carbondata/blob/master/docs/file-structure-of-carbondata.md)
-* [Data Types](https://github.com/apache/incubator-carbondata/blob/master/docs/supported-data-types-in-carbondata.md)
-* [Data Management](https://github.com/apache/incubator-carbondata/blob/master/docs/data-management.md)
-* [DDL Operations on CarbonData](https://github.com/apache/incubator-carbondata/blob/master/docs/ddl-operation-on-carbondata.md) 
-* [DML Operations on CarbonData](https://github.com/apache/incubator-carbondata/blob/master/docs/dml-operation-on-carbondata.md)  
-* [Cluster Installation and Deployment](https://github.com/apache/incubator-carbondata/blob/master/docs/installation-guide.md)
-* [FAQ](https://github.com/apache/incubator-carbondata/blob/master/docs/faq.md)
-* [Trouble Shooting](https://github.com/apache/incubator-carbondata/blob/master/docs/troubleshooting.md)
-* [Useful Tips](https://github.com/apache/incubator-carbondata/blob/master/docs/useful-tips-on-carbondata.md)
+* [What is CarbonData](https://github.com/apache/carbondata/blob/master/docs/introduction.md)
+* [Quick Start](https://github.com/apache/carbondata/blob/master/docs/quick-start-guide.md)
+* [Use Cases](https://github.com/apache/carbondata/blob/master/docs/usecases.md)
+* [Language Reference](https://github.com/apache/carbondata/blob/master/docs/language-manual.md)
+ * [CarbonData Data Definition Language](https://github.com/apache/carbondata/blob/master/docs/ddl-of-carbondata.md) 
+ * [CarbonData Data Manipulation Language](https://github.com/apache/carbondata/blob/master/docs/dml-of-carbondata.md) 
+ * [CarbonData Streaming Ingestion](https://github.com/apache/carbondata/blob/master/docs/streaming-guide.md) 
+ * [Configuring CarbonData](https://github.com/apache/carbondata/blob/master/docs/configuration-parameters.md) 
+ * [Index Developer Guide](https://github.com/apache/carbondata/blob/master/docs/index-developer-guide.md) 
+ * [Data Types](https://github.com/apache/carbondata/blob/master/docs/supported-data-types-in-carbondata.md) 
+* [CarbonData Index Management](https://github.com/apache/carbondata/blob/master/docs/index/index-management.md) 
+ * [CarbonData BloomFilter Index](https://github.com/apache/carbondata/blob/master/docs/index/bloomfilter-index-guide.md) 
+ * [CarbonData Lucene Index](https://github.com/apache/carbondata/blob/master/docs/index/lucene-index-guide.md)
+ * [CarbonData MV](https://github.com/apache/carbondata/blob/master/docs/mv-guide.md)
+* [Carbondata Secondary Index](https://github.com/apache/carbondata/blob/master/docs/index/secondary-index-guide.md)
+* [Heterogeneous format segments in carbondata](https://github.com/apache/carbondata/blob/master/docs/addsegment-guide.md)
+* [SDK Guide](https://github.com/apache/carbondata/blob/master/docs/sdk-guide.md) 
+* [C++ SDK Guide](https://github.com/apache/carbondata/blob/master/docs/csdk-guide.md)
+* [Performance Tuning](https://github.com/apache/carbondata/blob/master/docs/performance-tuning.md) 
+* [S3 Storage](https://github.com/apache/carbondata/blob/master/docs/s3-guide.md)
+* [Distributed Index Server](https://github.com/apache/carbondata/blob/master/docs/index-server.md)
+* [CDC and SCD](https://github.com/apache/carbondata/blob/master/docs/scd-and-cdc-guide.md)
+* [Carbon as Spark's Datasource](https://github.com/apache/carbondata/blob/master/docs/carbon-as-spark-datasource-guide.md) 
+* [FAQs](https://github.com/apache/carbondata/blob/master/docs/faq.md) 
+
+## Experimental Features
+
+Some features are marked as experimental because the syntax/implementation might change in the future.
+1. Hybrid format table using Add Segment.
+2. Accelerating performance using MV on parquet/orc.
+3. Merge API for Spark DataFrame.
+4. Hive write for non-transactional table.
+5. Secondary Index as a Coarse Grain Index in query processing.
+
+##  Integration
+* [Hive](https://github.com/apache/carbondata/blob/master/docs/hive-guide.md)
+* [Presto](https://github.com/apache/carbondata/blob/master/docs/prestodb-guide.md)
+* [Alluxio](https://github.com/apache/carbondata/blob/master/docs/alluxio-guide.md)
+* [Flink](https://github.com/apache/carbondata/blob/master/docs/flink-integration-guide.md)
 
 ## Other Technical Material
-[Apache CarbonData meetup material](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=66850609)
+* [Apache CarbonData meetup material](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=66850609)
+* [Use Case Articles](https://cwiki.apache.org/confluence/display/CARBONDATA/CarbonData+Articles)
 
 ## Fork and Contribute
 This is an active open source project for everyone, and we are always open to people who want to use this system or contribute to it. 
-This guide document introduce [how to contribute to CarbonData](https://github.com/apache/incubator-carbondata/blob/master/docs/How-to-contribute-to-Apache-CarbonData.md).
+This guide document introduces [how to contribute to CarbonData](https://github.com/apache/carbondata/blob/master/docs/how-to-contribute-to-apache-carbondata.md).
 
 ## Contact us
 To get involved in CarbonData:
 
-* First join by emailing to [[email protected]](mailto:[email protected]),then you can discuss issues by emailing to [[email protected]](mailto:[email protected]) or visit http://apache-carbondata-mailing-list-archive.1130556.n5.nabble.com
-* Report issues on [Apache Jira](https://issues.apache.org/jira/browse/CARBONDATA).
+* First join by emailing to [[email protected]](mailto:[email protected]), then you can discuss issues by emailing to [[email protected]](mailto:[email protected]). 
+  You can also directly visit [[email protected]](https://lists.apache.org/[email protected]). 
+  Or you can visit [Apache CarbonData Dev Mailing List archive](https://lists.apache.org/[email protected]). 
+
+* Report issues on [github issues](https://github.com/apache/carbondata/issues).
+
+* You can also slack to get in touch with the community. After we invite you, you can use this [Slack Link](https://carbondataworkspace.slack.com/) to sign in to CarbonData.
+
 
 ## About
 Apache CarbonData is an open source project of The Apache Software Foundation (ASF).