build-common.xml

<?xml version="1.0"?>

<!--
   Licensed to the Apache Software Foundation (ASF) under one or more
   contributor license agreements.  See the NOTICE file distributed with
   this work for additional information regarding copyright ownership.
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
-->


<project xmlns:ivy="antlib:org.apache.ivy.ant"
         xmlns:artifact="urn:maven-artifact-ant"
         name="hivecommon" default="jar">

  <property name="hive.root" location="${basedir}/.."/>
  <property file="${hive.root}/build.properties"/>
  <property file="${user.home}/build.properties" />
  <property file="${basedir}/build.properties" />

  <property environment="env"/>

  <property name="hive.conf.dir" value="${hive.root}/conf"/>
  <property name="dist.dir" location="${hive.root}"/>

  <property name="build.dir.hive" location="${hive.root}/build"/>
  <property name="build.dir.hadoop" location="${build.dir.hive}/hadoopcore"/>
  <property name="build.dir" location="${build.dir.hive}/${ant.project.name}"/>
  <property name="build.classes" location="${build.dir}/classes"/>
  <property name="build.encoding" value="ISO-8859-1"/>

  <property name="thrift.args" value="-I ${thrift.home} --gen java:beans --gen cpp --gen php --gen py --gen rb"/>

  <property name="hadoop.conf.dir" location="${hadoop.root}/conf"/>

  <!-- configuration needed for tests -->
  <property name="test.src.dir" value="${basedir}/src/test"/>
  <property name="test.src.data.dir" value="${hive.root}/data"/>
  <property name="test.build.dir" value="${build.dir}/test"/>
  <property name="test.log.dir" value="${test.build.dir}/logs"/>
  <property name="test.data.dir" value="${test.build.dir}/data"/>
  <property name="test.build.src" value="${test.build.dir}/src"/>
  <property name="test.build.classes" value="${test.build.dir}/classes"/>
  <property name="test.include" value="Test*"/>
  <property name="test.classpath.id" value="test.classpath"/>
  <property name="test.output" value="true"/>
  <property name="test.timeout" value="18200000"/>
  <property name="test.junit.output.format" value="xml"/>
  <property name="test.junit.output.usefile" value="true"/>
  <property name="minimr.query.files" value="input16_cc.q,scriptfile1.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q"/>
  <property name="test.silent" value="true"/>
  <property name="hadoopVersion" value="${hadoop.version.ant-internal}"/>
  <property name="test.serialize.qplan" value="false"/>

  <path id="test.classpath">
    <pathelement location="${test.build.classes}" />
    <pathelement location="" />
    <pathelement location="${test.src.data.dir}/conf"/>
    <pathelement location="${hive.conf.dir}"/>
    <fileset dir="${hive.root}" includes="testlibs/*.jar"/>
    <path refid="classpath"/>
  </path>

  <taskdef resource="net/sf/antcontrib/antcontrib.properties">
    <classpath>
      <pathelement location="${hive.root}/testlibs/ant-contrib-1.0b3.jar"/>
    </classpath>
  </taskdef>

  <!-- IVY properties set here -->
  <property name="build.ivy.dir" location="${build.dir.hive}/ivy"/>
  <property name="build.ivy.lib.dir" location="${build.ivy.dir}/lib"/>
  <property name="build.ivy.report.dir" location="${build.ivy.dir}/report"/>
  <property name="build.ivy.maven.dir" location="${build.ivy.dir}/maven"/>
  <property name="ivy.conf.dir" location="${hive.root}/ivy"/>
  <loadproperties srcfile="${ivy.conf.dir}/libraries.properties"/>
  <property name="ivy.jar" location="${build.ivy.lib.dir}/ivy-${ivy.version}.jar"/>
  <property name="ivysettings.xml" location="${ivy.conf.dir}/ivysettings.xml" />
  <property name="mvn.repo" value="http://repo2.maven.org/maven2"/>
  <property name="ivy_repo_url" value="${mvn.repo}/org/apache/ivy/ivy/${ivy.version}/ivy-${ivy.version}.jar"/>

  <!-- Maven properties set here -->
  <property name="hive.ivy.org" value="org.apache.hive"/>
  <property name="mvn.publish.repo" value="snapshots"/>
  <property name="mvn.publish.repo.id" value="apache.${mvn.publish.repo}.https" />
  <property name="mvn.publish.repo.url" value="https://repository.apache.org/content/repositories/${mvn.publish.repo}"/>

  <condition property="offline">
    <istrue value="${is-offline}"/>
  </condition>
  <condition property="ivy.cache.name" value="offline" else="online">
    <isset property="offline"/>
  </condition>
  <condition property="ivy.skip">
    <and>
      <isset property="offline"/>
      <available file="${build.dir.hadoop}/hadoop-${hadoop.version.ant-internal}.installed"/>
    </and>
  </condition>

  <!--this is the naming policy for artifacts we want pulled down-->
  <property name="ivy.artifact.retrieve.pattern" value="[conf]/[artifact]-[revision](-[classifier]).[ext]"/>

  <target name="ivy-init-dirs">
    <mkdir dir="${build.ivy.dir}" />
    <mkdir dir="${build.ivy.lib.dir}" />
    <mkdir dir="${build.ivy.report.dir}" />
    <mkdir dir="${build.ivy.maven.dir}" />
  </target>

  <target name="ivy-probe-antlib" >
    <condition property="ivy.found">
      <typefound uri="antlib:org.apache.ivy.ant" name="cleancache"/>
    </condition>
  </target>

  <target name="ivy-download" depends="ivy-init-dirs"
          description="To download ivy" unless="offline">
    <get src="${ivy_repo_url}" dest="${ivy.jar}" usetimestamp="true"/>
  </target>

  <!--
  To avoid Ivy leaking things across big projects, always load Ivy in the same classloader.
  Also note how we skip loading Ivy if it is already there, just to make sure all is well.
  -->
  <target name="ivy-init-antlib" depends="ivy-download,ivy-init-dirs,ivy-probe-antlib" unless="ivy.found">
    <typedef uri="antlib:org.apache.ivy.ant" onerror="fail"
      loaderRef="ivyLoader">
      <classpath>
        <pathelement location="${ivy.jar}"/>
      </classpath>
    </typedef>
    <fail >
      <condition >
        <not>
          <typefound uri="antlib:org.apache.ivy.ant" name="cleancache"/>
        </not>
      </condition>
      You need Apache Ivy 2.1 or later from http://ant.apache.org/
      It could not be loaded from ${ivy_repo_url}
    </fail>
  </target>


  <property name="ivyresolvelog" value="download-only"/>
  <property name="ivyretrievelog" value="quite"/>

  <target name="ivy-init" depends="ivy-init-antlib" >
    <!--Configure Ivy by reading in the settings file
        If anyone has already read in a settings file into this settings ID, it gets priority
    -->
    <ivy:settings id="${ant.project.name}.ivy.settings" file="${ivysettings.xml}"/>
  </target>

  <target name="ivy-resolve" depends="ivy-init" unless="offline">
    <ivy:resolve settingsRef="${ant.project.name}.ivy.settings"
      log="${ivyresolvelog}"/>
  </target>

  <target name="ivy-retrieve" depends="ivy-resolve"
    description="Retrieve Ivy-managed artifacts">
    <ivy:retrieve settingsRef="${ant.project.name}.ivy.settings"
      pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}"
      log="${ivyresolvelog}"/>
  </target>

  <target name="ivy-resolve-checkstyle" depends="ivy-init">
    <ivy:resolve settingsRef="${ant.project.name}.ivy.settings" conf="checkstyle"
      log="${ivyresolvelog}"/>
  </target>

  <target name="ivy-retrieve-checkstyle" depends="ivy-resolve-checkstyle"
    description="Retrieve Ivy-managed artifacts for the checkstyle configurations">
    <ivy:retrieve settingsRef="${ant.project.name}.ivy.settings"
      pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}"
      log="${ivyresolvelog}"/>
    <ivy:cachepath pathid="checkstyle-classpath" conf="checkstyle"/>
  </target>

  <target name="ivy-resolve-maven-ant-tasks" depends="ivy-init">
    <ivy:resolve settingsRef="${ant.project.name}.ivy.settings" conf="maven"
      log="${ivyresolvelog}"/>
  </target>

  <target name="ivy-retrieve-maven-ant-tasks" depends="ivy-resolve-maven-ant-tasks"
    description="Retrieve Ivy-managed artifacts for the maven-ant-tasks configurations">
    <ivy:retrieve settingsRef="${ant.project.name}.ivy.settings"
      pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}"
      log="${ivyresolvelog}"/>
    <ivy:cachepath pathid="maven-ant-tasks.classpath" conf="maven"/>
  </target>

  <target name="ivy-retrieve-hadoop-source" depends="ivy-init"
    description="Retrieve Ivy-managed Hadoop source artifacts" unless="ivy.skip">
    <ivy:retrieve settingsRef="${ant.project.name}.ivy.settings"
      pattern="${build.dir.hadoop}/[artifact]-[revision].[ext]"/>
  </target>

  <target name="ivy-docs" depends="ivy-init"
    description="Resolve, Retrieve Ivy-managed artifacts for docs configuration">
    <ivy:resolve settingsRef="${ant.project.name}.ivy.settings" conf="docs"/>
    <ivy:retrieve settingsRef="${ant.project.name}.ivy.settings"
                  pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}" conf="docs"/>
    <ivy:cachepath pathid="docs-classpath" conf="docs"/>
  </target>

  <available property="hadoopcore.${hadoop.version.ant-internal}.install.done"
    file="${build.dir.hadoop}/hadoop-${hadoop.version.ant-internal}.installed"/>

  <target name="install-hadoopcore-internal" depends="ivy-retrieve-hadoop-source"
    unless="hadoopcore.${hadoop.version.ant-internal}.install.done">
    <untar src="${build.dir.hadoop}/hadoop-${hadoop.version.ant-internal}.tar.gz" dest="${build.dir.hadoop}" compression="gzip"/>
    <chmod file="${hadoop.root}/bin/hadoop" perm="+x"/>
    <touch file="${build.dir.hadoop}/hadoop-${hadoop.version.ant-internal}.installed"/>
  </target>

  <target name="install-hadoopcore-default" unless="hadoop.root.nondefault">
    <antcall target="install-hadoopcore-internal"/>
  </target>

  <target name="install-hadoopcore">
    <condition property="hadoop.root.nondefault">
      <not>
        <equals arg1="${hadoop.root}" arg2="${hadoop.root.default}"/>
      </not>
    </condition>
    <antcall target="install-hadoopcore-default"/>
  </target>

  <!-- the normal classpath -->
  <path id="common-classpath">
    <pathelement location="${hadoop.oldstyle-name.jar}"/>
    <pathelement location="${hadoop.oldstyle-name.tools.jar}"/>
    <pathelement location="${hadoop.newstyle-name.jar}"/>
    <pathelement location="${hadoop.newstyle-name.tools.jar}"/>
    <pathelement location="${build.dir.hive}/classes"/>
    <fileset dir="${build.dir.hive}" includes="*/*.jar"/>
    <fileset dir="${hive.root}/lib" includes="*.jar"/>
    <fileset dir="${hive.root}/ql/lib" includes="*.jar"/>
    <fileset dir="${build.ivy.lib.dir}/default" includes="*.jar" excludes="*hadoop*.jar"
             erroronmissingdir="false"/>
  </path>

  <path id="classpath">
    <pathelement location="${build.dir.hive}/service/classes"/>
    <pathelement location="${build.dir.hive}/common/classes"/>
    <pathelement location="${build.dir.hive}/serde/classes"/>
    <pathelement location="${build.dir.hive}/metastore/classes"/>
    <pathelement location="${build.dir.hive}/ql/classes"/>
    <pathelement location="${build.dir.hive}/cli/classes"/>
    <pathelement location="${build.dir.hive}/shims/classes"/>
    <pathelement location="${build.dir.hive}/hwi/classes"/>
    <path refid="common-classpath"/>
    <fileset dir="${basedir}" includes="lib/*.jar"/>
  </path>

  <target name="create-dirs">
    <mkdir dir="${build.dir.hive}"/>
    <mkdir dir="${build.dir}"/>
    <mkdir dir="${build.classes}"/>
    <mkdir dir="${build.dir.hive}/jexl/classes"/>
    <mkdir dir="${build.dir.hadoop}"/>
    <mkdir dir="${test.build.dir}"/>
    <mkdir dir="${test.build.src}"/>
    <mkdir dir="${test.build.classes}"/>
  </target>

  <target name="init" depends="create-dirs, deploy-ant-tasks"/>

  <target name="test-init">
    <mkdir dir="${test.data.dir}"/>
    <mkdir dir="${test.log.dir}/clientpositive"/>
    <mkdir dir="${test.log.dir}/clientnegative"/>
    <mkdir dir="${test.log.dir}/positive"/>
    <mkdir dir="${test.log.dir}/negative"/>
    <mkdir dir="${test.data.dir}/warehouse"/>
    <mkdir dir="${test.data.dir}/metadb"/>
  </target>

  <target name="setup"/>

  <target name="compile" depends="init, setup">
    <echo message="Compiling: ${ant.project.name}"/>
    <javac
     encoding="${build.encoding}"
     srcdir="${src.dir}"
     includes="**/*.java"
     destdir="${build.classes}"
     debug="${javac.debug}"
     deprecation="${javac.deprecation}">
      <compilerarg line="${javac.args} ${javac.args.warnings}" />
      <classpath refid="classpath"/>
    </javac>
    <copy todir="${build.classes}" failonerror="false">
      <fileset dir="${src.dir}/conf"/>
    </copy>
  </target>

  <target name="jar" depends="compile">
    <echo message="Jar: ${ant.project.name}"/>
    <jar
      jarfile="${build.dir}/hive-${ant.project.name}-${version}.jar"
      basedir="${build.classes}">
      <manifest>
        <!-- Not putting these in their own manifest section, since that inserts
        a new-line, which breaks the reading of the attributes. -->
        <attribute name="Implementation-Title" value="Hive"/>
        <attribute name="Implementation-Version" value="${version}"/>
        <attribute name="Implementation-Vendor" value="Apache"/>
      </manifest>
      <metainf dir="${hive.root}" includes="LICENSE,NOTICE"/>
    </jar>
  </target>

  <!-- target to compile tests -->
  <target name="compile-test" depends="compile">
    <javac
     encoding="${build.encoding}"
     srcdir="${test.src.dir}"
     includes="org/apache/hadoop/**/*.java"
     excludes="**/TestSerDe.java"
     destdir="${test.build.classes}"
     debug="${javac.debug}"
     optimize="${javac.optimize}"
     target="${javac.version}"
     source="${javac.version}"
     deprecation="${javac.deprecation}">
      <compilerarg line="${javac.args} ${javac.args.warnings}" />
      <classpath refid="test.classpath"/>
    </javac>
    <javac
     encoding="${build.encoding}"
     srcdir="${test.build.src}"
     includes="org/apache/hadoop/**/*.java"
     destdir="${test.build.classes}"
     debug="${javac.debug}"
     optimize="${javac.optimize}"
     target="${javac.version}"
     source="${javac.version}"
     deprecation="${javac.deprecation}">
      <compilerarg line="${javac.args} ${javac.args.warnings}" />
      <classpath refid="test.classpath"/>
    </javac>
  </target>

  <target name="test-jar" depends="compile-test">
    <delete file="${test.build.dir}/test-udfs.jar"/>
    <jar jarfile="${test.build.dir}/test-udfs.jar">
        <fileset dir="${test.build.classes}" includes="**/udf/*.class"/>
        <fileset dir="${test.build.classes}" includes="**/udf/generic/*.class"/>
    </jar>
  </target>

  <target name="test-conditions">

    <condition property="qfile" value="">
      <not>
        <isset property="qfile"/>
      </not>
    </condition>

    <condition property="qfile_regex" value="">
      <not>
        <isset property="qfile_regex"/>
      </not>
    </condition>

    <condition property="overwrite" value="false">
      <not>
        <isset property="overwrite"/>
      </not>
    </condition>

    <condition property="standalone" value="false">
      <not>
        <isset property="standalone"/>
      </not>
    </condition>

    <condition property="clustermode" value="">
      <not>
        <isset property="clustermode"/>
      </not>
    </condition>

    <condition property="run_disabled" value="false">
      <not>
        <isset property="run_disabled"/>
      </not>
    </condition>

  </target>

  <!-- target to deploy anttasks -->

  <target name="compile-ant-tasks">
    <subant target="compile">
      <fileset dir=".." includes="ant/build.xml"/>
    </subant>
  </target>

  <target name="deploy-ant-tasks" depends="compile-ant-tasks">
    <subant target="jar">
      <fileset dir=".." includes="ant/build.xml"/>
    </subant>

    <taskdef name="getversionpref" classname="org.apache.hadoop.hive.ant.GetVersionPref"
             classpath="${build.dir.hive}/anttasks/hive-anttasks-${version}.jar"/>

  </target>

  <target name="gen-test"/>

  <!-- use pfile:/// as warehouse file system in 20 for non miniMR runs -->
  <condition property="test.warehouse.scheme" value="pfile://" else="">
    <not>
      <equals arg1="${clustermode}" arg2="miniMR" />
    </not>
  </condition>

  <property name="test.warehouse.dir" value="${test.warehouse.scheme}${build.dir}/test/data/warehouse"/>
  <property name="mapred.job.tracker" value="local"/>
  <property name="fs.default.name" value="file:///"/>

  <!-- target to run the tests -->
  <target name="test"
  	depends="test-conditions,gen-test,compile-test,test-jar,test-init">
    <!--<property name="testcp" refid="test.classpath"/>-->
    <!--<echo message="test.classpath: ${testcp}"/>-->
    <junit showoutput="${test.output}" printsummary="yes" haltonfailure="no"
           fork="yes" maxmemory="512m" dir="${basedir}" timeout="${test.timeout}"
           errorProperty="tests.failed" failureProperty="tests.failed" filtertrace="off">
      <!--
      <jvmarg value="-Xdebug"/>
      <jvmarg value="-Xrunjdwp:transport=dt_socket,address=8000,server=y,suspend=y"/> -->
      <env key="HADOOP_HOME" value="${hadoop.root}"/>
      <env key="HADOOP_CLASSPATH" value="${test.src.data.dir}/conf:${build.dir.hive}/dist/lib/derby.jar:${build.dir.hive}/dist/lib/javaewah-0.2.jar"/>
      <env key="TZ" value="US/Pacific"/>
      <sysproperty key="test.output.overwrite" value="${overwrite}"/>
      <sysproperty key="test.service.standalone.server" value="${standalone}"/>
      <sysproperty key="log4j.configuration" value="file://${test.src.data.dir}/conf/hive-log4j.properties"/>
      <sysproperty key="derby.stream.error.file" value="${test.build.dir}/derby.log"/>
      <sysproperty key="hive.aux.jars.path" value="file://${test.build.dir}/test-udfs.jar"/>
      <sysproperty key="ql.test.query.clientpositive.dir" value="${ql.test.query.clientpositive.dir}"/>
      <sysproperty key="ql.test.results.clientpositive.dir" value="${ql.test.results.clientpositive.dir}"/>
      <sysproperty key="test.log.dir" value="${test.log.dir}"/>
      <sysproperty key="hadoop.log.dir" value="${test.log.dir}"/>
      <sysproperty key="test.silent" value="${test.silent}"/>
      <sysproperty key="test.tmp.dir" value="${build.dir}/tmp"/>
      <sysproperty key="test.warehouse.dir" value="${test.warehouse.dir}"/>
      <sysproperty key="mapred.job.tracker" value="${mapred.job.tracker}"/>
      <sysproperty key="fs.default.name" value="${fs.default.name}"/>
      <sysproperty key="build.dir" value="${build.dir}"/>
      <sysproperty key="build.dir.hive" value="${build.dir.hive}"/>
      <sysproperty key="hive.version" value="${version}"/>

      <classpath refid="${test.classpath.id}"/>
      <formatter type="${test.junit.output.format}" usefile="${test.junit.output.usefile}" />
      <batchtest todir="${test.build.dir}" unless="testcase">
        <fileset dir="${test.build.classes}"
                 includes="**/${test.include}.class"
                 excludes="**/TestSerDe.class,**/TestHiveMetaStore.class,**/*$*.class" />
      </batchtest>
      <batchtest todir="${test.build.dir}" if="testcase">
        <fileset dir="${test.build.classes}" includes="**/${testcase}.class"/>
      </batchtest>
      <assertions>
        <enable />
      </assertions>
    </junit>
    <fail if="tests.failed">Tests failed!</fail>
  </target>
  <target name="test-shims">
     <subant target="test">
       <property name="hadoop.version" value="${hadoop.security.version}"/>
       <property name="hadoop.security.version" value="${hadoop.security.version}"/>
       <fileset dir="${hive.root}/shims" includes="build.xml"/>
     </subant>
  </target>

  <target name="clean-test">
    <delete dir="${test.build.dir}"/>
    <delete dir="${build.dir.hive}/ql/tmp"/>
    <delete dir="${build.dir.hive}/test"/>
  </target>

  <target name="clean">
    <echo message="Cleaning: ${ant.project.name}"/>
    <delete dir="${build.dir}"/>
  </target>

  <target name="check-thrift-home">
    <condition property="thrift.home.defined">
      <or>
        <not>
          <isset property="thrift.home"/>
        </not>
        <equals arg1="${thrift.home}" arg2="$${thrift.home}" trim="true"/>
      </or>
    </condition>
  </target>

  <target name="check-ivy" depends="ivy-init">
   <available file="${basedir}/ivy.xml" property="ivy.present"/>
  </target>

  <target name="make-pom" if="ivy.present" depends="check-ivy, jar">
    <echo> Writing POM to ${build.dir}/pom.xml</echo>

    <ivy:makepom ivyfile="${basedir}/ivy.xml" pomfile="${build.dir}/pom.xml">
      <mapping conf="default" scope="compile" />
      <mapping conf="runtime" scope="runtime" />
    </ivy:makepom>
    <replace file="${build.dir}/pom.xml">
      <replacetoken>&lt;dependencies&gt;</replacetoken>
      <replacevalue>
        &lt;description&gt;Hive is a data warehouse infrastructure built on top of Hadoop see
        http://wiki.apache.org/hadoop/Hive &lt;/description&gt;
        &lt;licenses&gt;
        &lt;license&gt;
        &lt;name&gt;The Apache Software License, Version 2.0&lt;/name&gt;
        &lt;url&gt;http://www.apache.org/licenses/LICENSE-2.0.txt&lt;/url&gt;
        &lt;distribution&gt;repo&lt;/distribution&gt;
        &lt;/license&gt;
        &lt;/licenses&gt;
        &lt;scm&gt;
        &lt;url&gt;http://svn.apache.org/repos/asf/hive/&lt;/url&gt;
        &lt;/scm&gt;
        &lt;dependencies&gt;
      </replacevalue>
    </replace>
  </target>

  <target name="mvn-taskdef" depends="ivy-retrieve-maven-ant-tasks">
    <typedef resource="org/apache/maven/artifact/ant/antlib.xml"
	     uri="urn:maven-artifact-ant" classpathref="maven-ant-tasks.classpath" />
  </target>

  <!-- Deploy a single artifact to the maven repository -->
  <target name="maven-publish-artifact" depends="mvn-taskdef">
    <artifact:pom
       file="${build.dir.hive}/maven/poms/hive-${hive.project}-${version}.pom"
       id="hive.project.pom" />

    <artifact:install-provider artifactId="wagon-http" version="1.0-beta-2" />
    <artifact:deploy
       file="${build.dir.hive}/maven/jars/hive-${hive.project}-${version}.jar">
      <remoteRepository id="${mvn.publish.repo.id}" url="${mvn.publish.repo.url}" />
      <pom refid="hive.project.pom" />
    </artifact:deploy>
  </target>

</project>