From 5b5feaaeda7c1f1e30dec63b5321a001acb86887 Mon Sep 17 00:00:00 2001 From: Nisarg Thakkar Date: Fri, 8 Sep 2023 17:27:27 -0700 Subject: [PATCH] [build] Fix JDK8 builds and set up CI (#627) Previously, builds using JDK8 would fail. Also, cleaned up the alpn-boot loading process for HTTP/2 on JDK 8 tests. --- .../gh-ci-alpini-parametrized-flow.txt | 4 +- .../rawWorkflows/gh-ci-parameterized-flow.txt | 4 +- .github/workflows/gh-ci-pulsar-tests.yml | 4 +- .github/workflows/gh-ci.yml | 96 +++++++++---------- build.gradle | 64 +++++++------ gradle.properties | 14 +-- 6 files changed, 89 insertions(+), 97 deletions(-) diff --git a/.github/rawWorkflows/gh-ci-alpini-parametrized-flow.txt b/.github/rawWorkflows/gh-ci-alpini-parametrized-flow.txt index 4c305827f8..a5f2957471 100644 --- a/.github/rawWorkflows/gh-ci-alpini-parametrized-flow.txt +++ b/.github/rawWorkflows/gh-ci-alpini-parametrized-flow.txt @@ -2,7 +2,7 @@ strategy: fail-fast: false matrix: - jdk: [11, 17] + jdk: [8, 11, 17] runs-on: ubuntu-latest timeout-minutes: $TimeOut outputs: @@ -28,7 +28,7 @@ if: steps.check_alpini_files_changed.outputs.alpini == 'true' with: java-version: ${{ matrix.jdk }} - distribution: 'microsoft' + distribution: 'temurin' cache: 'gradle' # - name: Allow Deprecated TLS versions for Alpini tests # run: | diff --git a/.github/rawWorkflows/gh-ci-parameterized-flow.txt b/.github/rawWorkflows/gh-ci-parameterized-flow.txt index 58b422851e..c027c62778 100644 --- a/.github/rawWorkflows/gh-ci-parameterized-flow.txt +++ b/.github/rawWorkflows/gh-ci-parameterized-flow.txt @@ -2,7 +2,7 @@ strategy: fail-fast: false matrix: - jdk: [11, 17] + jdk: [8, 11, 17] runs-on: ubuntu-latest needs: $Dependency if: $Conditional @@ -15,7 +15,7 @@ uses: actions/setup-java@v3 with: java-version: ${{ matrix.jdk }} - distribution: 'microsoft' + distribution: 'temurin' cache: 'gradle' - shell: bash run: | diff --git a/.github/workflows/gh-ci-pulsar-tests.yml b/.github/workflows/gh-ci-pulsar-tests.yml index 4ba813fd95..e52327a89c 100644 --- a/.github/workflows/gh-ci-pulsar-tests.yml +++ b/.github/workflows/gh-ci-pulsar-tests.yml @@ -8,7 +8,7 @@ jobs: strategy: fail-fast: false matrix: - jdk: [11, 17] + jdk: [8, 11, 17] runs-on: ubuntu-latest timeout-minutes: 120 steps: @@ -19,7 +19,7 @@ jobs: uses: actions/setup-java@v3 with: java-version: ${{ matrix.jdk }} - distribution: 'microsoft' + distribution: 'temurin' cache: 'gradle' - shell: bash run: | diff --git a/.github/workflows/gh-ci.yml b/.github/workflows/gh-ci.yml index 8043eec14b..6299065c49 100644 --- a/.github/workflows/gh-ci.yml +++ b/.github/workflows/gh-ci.yml @@ -29,7 +29,7 @@ jobs: strategy: fail-fast: false matrix: - jdk: [11, 17] + jdk: [8, 11, 17] runs-on: ubuntu-latest timeout-minutes: 20 steps: @@ -40,7 +40,7 @@ jobs: uses: actions/setup-java@v3 with: java-version: ${{ matrix.jdk }} - distribution: 'microsoft' + distribution: 'temurin' cache: 'gradle' - shell: bash run: | @@ -71,7 +71,7 @@ jobs: strategy: fail-fast: false matrix: - jdk: [11, 17] + jdk: [8, 11, 17] runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -82,7 +82,7 @@ jobs: uses: actions/setup-java@v3 with: java-version: ${{ matrix.jdk }} - distribution: 'microsoft' + distribution: 'temurin' cache: 'gradle' - shell: bash run: | @@ -113,7 +113,7 @@ jobs: strategy: fail-fast: false matrix: - jdk: [11, 17] + jdk: [8, 11, 17] runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -124,7 +124,7 @@ jobs: uses: actions/setup-java@v3 with: java-version: ${{ matrix.jdk }} - distribution: 'microsoft' + distribution: 'temurin' cache: 'gradle' - shell: bash run: | @@ -155,7 +155,7 @@ jobs: strategy: fail-fast: false matrix: - jdk: [11, 17] + jdk: [8, 11, 17] runs-on: ubuntu-latest timeout-minutes: 120 steps: @@ -166,7 +166,7 @@ jobs: uses: actions/setup-java@v3 with: java-version: ${{ matrix.jdk }} - distribution: 'microsoft' + distribution: 'temurin' cache: 'gradle' - shell: bash run: | @@ -197,7 +197,7 @@ jobs: strategy: fail-fast: false matrix: - jdk: [11, 17] + jdk: [8, 11, 17] runs-on: ubuntu-latest timeout-minutes: 120 steps: @@ -208,7 +208,7 @@ jobs: uses: actions/setup-java@v3 with: java-version: ${{ matrix.jdk }} - distribution: 'microsoft' + distribution: 'temurin' cache: 'gradle' - shell: bash run: | @@ -239,7 +239,7 @@ jobs: strategy: fail-fast: false matrix: - jdk: [11, 17] + jdk: [8, 11, 17] runs-on: ubuntu-latest timeout-minutes: 120 steps: @@ -250,7 +250,7 @@ jobs: uses: actions/setup-java@v3 with: java-version: ${{ matrix.jdk }} - distribution: 'microsoft' + distribution: 'temurin' cache: 'gradle' - shell: bash run: | @@ -281,7 +281,7 @@ jobs: strategy: fail-fast: false matrix: - jdk: [11, 17] + jdk: [8, 11, 17] runs-on: ubuntu-latest timeout-minutes: 120 steps: @@ -292,7 +292,7 @@ jobs: uses: actions/setup-java@v3 with: java-version: ${{ matrix.jdk }} - distribution: 'microsoft' + distribution: 'temurin' cache: 'gradle' - shell: bash run: | @@ -323,7 +323,7 @@ jobs: strategy: fail-fast: false matrix: - jdk: [11, 17] + jdk: [8, 11, 17] runs-on: ubuntu-latest timeout-minutes: 120 steps: @@ -334,7 +334,7 @@ jobs: uses: actions/setup-java@v3 with: java-version: ${{ matrix.jdk }} - distribution: 'microsoft' + distribution: 'temurin' cache: 'gradle' - shell: bash run: | @@ -365,7 +365,7 @@ jobs: strategy: fail-fast: false matrix: - jdk: [11, 17] + jdk: [8, 11, 17] runs-on: ubuntu-latest timeout-minutes: 120 steps: @@ -376,7 +376,7 @@ jobs: uses: actions/setup-java@v3 with: java-version: ${{ matrix.jdk }} - distribution: 'microsoft' + distribution: 'temurin' cache: 'gradle' - shell: bash run: | @@ -407,7 +407,7 @@ jobs: strategy: fail-fast: false matrix: - jdk: [11, 17] + jdk: [8, 11, 17] runs-on: ubuntu-latest timeout-minutes: 120 steps: @@ -418,7 +418,7 @@ jobs: uses: actions/setup-java@v3 with: java-version: ${{ matrix.jdk }} - distribution: 'microsoft' + distribution: 'temurin' cache: 'gradle' - shell: bash run: | @@ -449,7 +449,7 @@ jobs: strategy: fail-fast: false matrix: - jdk: [11, 17] + jdk: [8, 11, 17] runs-on: ubuntu-latest timeout-minutes: 120 steps: @@ -460,7 +460,7 @@ jobs: uses: actions/setup-java@v3 with: java-version: ${{ matrix.jdk }} - distribution: 'microsoft' + distribution: 'temurin' cache: 'gradle' - shell: bash run: | @@ -491,7 +491,7 @@ jobs: strategy: fail-fast: false matrix: - jdk: [11, 17] + jdk: [8, 11, 17] runs-on: ubuntu-latest timeout-minutes: 120 steps: @@ -502,7 +502,7 @@ jobs: uses: actions/setup-java@v3 with: java-version: ${{ matrix.jdk }} - distribution: 'microsoft' + distribution: 'temurin' cache: 'gradle' - shell: bash run: | @@ -533,7 +533,7 @@ jobs: strategy: fail-fast: false matrix: - jdk: [11, 17] + jdk: [8, 11, 17] runs-on: ubuntu-latest timeout-minutes: 120 steps: @@ -544,7 +544,7 @@ jobs: uses: actions/setup-java@v3 with: java-version: ${{ matrix.jdk }} - distribution: 'microsoft' + distribution: 'temurin' cache: 'gradle' - shell: bash run: | @@ -575,7 +575,7 @@ jobs: strategy: fail-fast: false matrix: - jdk: [11, 17] + jdk: [8, 11, 17] runs-on: ubuntu-latest timeout-minutes: 120 steps: @@ -586,7 +586,7 @@ jobs: uses: actions/setup-java@v3 with: java-version: ${{ matrix.jdk }} - distribution: 'microsoft' + distribution: 'temurin' cache: 'gradle' - shell: bash run: | @@ -617,7 +617,7 @@ jobs: strategy: fail-fast: false matrix: - jdk: [11, 17] + jdk: [8, 11, 17] runs-on: ubuntu-latest timeout-minutes: 120 steps: @@ -628,7 +628,7 @@ jobs: uses: actions/setup-java@v3 with: java-version: ${{ matrix.jdk }} - distribution: 'microsoft' + distribution: 'temurin' cache: 'gradle' - shell: bash run: | @@ -659,7 +659,7 @@ jobs: strategy: fail-fast: false matrix: - jdk: [11, 17] + jdk: [8, 11, 17] runs-on: ubuntu-latest timeout-minutes: 120 steps: @@ -670,7 +670,7 @@ jobs: uses: actions/setup-java@v3 with: java-version: ${{ matrix.jdk }} - distribution: 'microsoft' + distribution: 'temurin' cache: 'gradle' - shell: bash run: | @@ -701,7 +701,7 @@ jobs: strategy: fail-fast: false matrix: - jdk: [11, 17] + jdk: [8, 11, 17] runs-on: ubuntu-latest timeout-minutes: 120 steps: @@ -712,7 +712,7 @@ jobs: uses: actions/setup-java@v3 with: java-version: ${{ matrix.jdk }} - distribution: 'microsoft' + distribution: 'temurin' cache: 'gradle' - shell: bash run: | @@ -743,7 +743,7 @@ jobs: strategy: fail-fast: false matrix: - jdk: [11, 17] + jdk: [8, 11, 17] runs-on: ubuntu-latest timeout-minutes: 120 steps: @@ -754,7 +754,7 @@ jobs: uses: actions/setup-java@v3 with: java-version: ${{ matrix.jdk }} - distribution: 'microsoft' + distribution: 'temurin' cache: 'gradle' - shell: bash run: | @@ -785,7 +785,7 @@ jobs: strategy: fail-fast: false matrix: - jdk: [11, 17] + jdk: [8, 11, 17] runs-on: ubuntu-latest timeout-minutes: 120 steps: @@ -796,7 +796,7 @@ jobs: uses: actions/setup-java@v3 with: java-version: ${{ matrix.jdk }} - distribution: 'microsoft' + distribution: 'temurin' cache: 'gradle' - shell: bash run: | @@ -827,7 +827,7 @@ jobs: strategy: fail-fast: false matrix: - jdk: [11, 17] + jdk: [8, 11, 17] runs-on: ubuntu-latest timeout-minutes: 120 steps: @@ -838,7 +838,7 @@ jobs: uses: actions/setup-java@v3 with: java-version: ${{ matrix.jdk }} - distribution: 'microsoft' + distribution: 'temurin' cache: 'gradle' - shell: bash run: | @@ -869,7 +869,7 @@ jobs: strategy: fail-fast: false matrix: - jdk: [11, 17] + jdk: [8, 11, 17] runs-on: ubuntu-latest timeout-minutes: 120 steps: @@ -880,7 +880,7 @@ jobs: uses: actions/setup-java@v3 with: java-version: ${{ matrix.jdk }} - distribution: 'microsoft' + distribution: 'temurin' cache: 'gradle' - shell: bash run: | @@ -911,7 +911,7 @@ jobs: strategy: fail-fast: false matrix: - jdk: [11, 17] + jdk: [8, 11, 17] runs-on: ubuntu-latest timeout-minutes: 120 outputs: @@ -937,7 +937,7 @@ jobs: if: steps.check_alpini_files_changed.outputs.alpini == 'true' with: java-version: ${{ matrix.jdk }} - distribution: 'microsoft' + distribution: 'temurin' cache: 'gradle' # - name: Allow Deprecated TLS versions for Alpini tests # run: | @@ -981,7 +981,7 @@ jobs: strategy: fail-fast: false matrix: - jdk: [11, 17] + jdk: [8, 11, 17] runs-on: ubuntu-latest timeout-minutes: 120 outputs: @@ -1007,7 +1007,7 @@ jobs: if: steps.check_alpini_files_changed.outputs.alpini == 'true' with: java-version: ${{ matrix.jdk }} - distribution: 'microsoft' + distribution: 'temurin' cache: 'gradle' # - name: Allow Deprecated TLS versions for Alpini tests # run: | @@ -1051,7 +1051,7 @@ jobs: strategy: fail-fast: false matrix: - jdk: [11, 17] + jdk: [8, 11, 17] runs-on: ubuntu-latest needs: [ValidateGradleWrapper, StaticAnalysis, UnitTestsAndCodeCoverage, AvroCompatibilityTests, IntegrationTestsA, IntegrationTestsB, IntegrationTestsC, IntegrationTestsD, IntegrationTestsE, IntegrationTestsF, IntegrationTestsG, IntegrationTestsH, IntegrationTestsI, IntegrationTestsJ, IntegrationTestsK, IntegrationTestsL, IntegrationTestsM, IntegrationTestsN, IntegrationTestsO, IntegrationTestsP, IntegrationTestsQ, IntegrationTestsZ, AlpiniUnitTests, AlpiniFunctionalTests] timeout-minutes: 120 @@ -1063,7 +1063,7 @@ jobs: uses: actions/setup-java@v3 with: java-version: ${{ matrix.jdk }} - distribution: 'microsoft' + distribution: 'temurin' cache: 'gradle' - shell: bash run: | diff --git a/build.gradle b/build.gradle index 3d85ff248d..c125763322 100644 --- a/build.gradle +++ b/build.gradle @@ -1,5 +1,4 @@ import com.github.spotbugs.snom.SpotBugsTask -import org.apache.tools.ant.taskdefs.condition.Os import org.gradle.internal.logging.text.StyledTextOutput.Style import org.gradle.internal.logging.text.StyledTextOutputFactory @@ -47,15 +46,18 @@ def log4j2Version = '2.17.1' def pegasusVersion = '29.31.0' def protobufVersion = '3.21.7' def jacksonVersion = '2.13.3' - +def pulsarGroup = 'org.apache.pulsar' +def pulsarVersion = '2.10.3' +def alpnAgentVersion = '2.0.10' ext.libraries = [ - avro: 'org.apache.avro:avro:' + avroVersion, - avroCompiler: 'org.apache.avro:avro-compiler:' + avroVersion, - avroMapred: 'org.apache.avro:avro-mapred:' + avroVersion, - avroUtilBuilder: 'com.linkedin.avroutil1:builder:' + avroUtilVersion, - avroUtilCompatHelper: 'com.linkedin.avroutil1:helper-all:' + avroUtilVersion, - avroUtilFastserde: 'com.linkedin.avroutil1:avro-fastserde:' + avroUtilVersion, + alpnAgent: "org.mortbay.jetty.alpn:jetty-alpn-agent:${alpnAgentVersion}", + avro: "org.apache.avro:avro:${avroVersion}", + avroCompiler: "org.apache.avro:avro-compiler:${avroVersion}", + avroMapred: "org.apache.avro:avro-mapred:${avroVersion}", + avroUtilBuilder: "com.linkedin.avroutil1:builder:${avroUtilVersion}", + avroUtilCompatHelper: "com.linkedin.avroutil1:helper-all:${avroUtilVersion}", + avroUtilFastserde: "com.linkedin.avroutil1:avro-fastserde:${avroUtilVersion}", avroUtilSpotbugsPlugin: 'com.linkedin.avroutil1:spotbugs-plugin:0.2.69', bouncyCastle: 'org.bouncycastle:bcprov-jdk15on:1.55', caffeine: 'com.github.ben-manes.caffeine:caffeine:2.8.5', @@ -66,7 +68,7 @@ ext.libraries = [ commonsCli: 'commons-cli:commons-cli:1.5.0', commonsLang: 'commons-lang:commons-lang:2.6', conscrypt: 'org.conscrypt:conscrypt-openjdk-uber:2.5.2', - d2: 'com.linkedin.pegasus:d2:' + pegasusVersion, + d2: "com.linkedin.pegasus:d2:${pegasusVersion}", failsafe: 'net.jodah:failsafe:2.4.0', fastUtil: 'it.unimi.dsi:fastutil:8.3.0', hadoopCommon: 'org.apache.hadoop:hadoop-common:2.3.0', @@ -77,20 +79,20 @@ ext.libraries = [ httpCore5H2: 'org.apache.httpcomponents.core5:httpcore5-h2:5.2.2', httpClient: 'org.apache.httpcomponents:httpclient:4.5.2', httpCore: 'org.apache.httpcomponents:httpcore:4.4.5', - jacksonCore: 'com.fasterxml.jackson.core:jackson-core:' + jacksonVersion, - jacksonAnnotations: 'com.fasterxml.jackson.core:jackson-annotations:' + jacksonVersion, - jacksonDatabind: 'com.fasterxml.jackson.core:jackson-databind:' + jacksonVersion, + jacksonCore: "com.fasterxml.jackson.core:jackson-core:${jacksonVersion}", + jacksonAnnotations: "com.fasterxml.jackson.core:jackson-annotations:${jacksonVersion}", + jacksonDatabind: "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}", javax: 'javax.servlet:javax.servlet-api:3.1.0', javaxActivation: 'com.sun.activation:javax.activation:1.2.0', jdom: 'org.jdom:jdom:1.1', jna: 'net.java.dev.jna:jna:4.5.1', jsr305: 'com.google.code.findbugs:jsr305:3.0.2', joptSimple: 'net.sf.jopt-simple:jopt-simple:3.2', - kafka: kafkaGroup + ':kafka_2.12:' + kafkaVersion, - kafkaClients: kafkaGroup + ':kafka-clients:' + kafkaVersion, - kafkaClientsTest: kafkaGroup + ':kafka-clients:' + kafkaVersion + ':test', - log4j2api: 'org.apache.logging.log4j:log4j-api:' + log4j2Version, - log4j2core: 'org.apache.logging.log4j:log4j-core:' + log4j2Version, + kafka: "${kafkaGroup}:kafka_2.12:${kafkaVersion}", + kafkaClients: "${kafkaGroup}:kafka-clients:${kafkaVersion}", + kafkaClientsTest: "${kafkaGroup}:kafka-clients:${kafkaVersion}:test", + log4j2api: "org.apache.logging.log4j:log4j-api:${log4j2Version}", + log4j2core: "org.apache.logging.log4j:log4j-core:${log4j2Version}", mail: 'javax.mail:mail:1.4.4', mapreduceClientCore: 'org.apache.hadoop:hadoop-mapreduce-client-core:2.3.0', mapreduceClientJobClient: 'org.apache.hadoop:hadoop-mapreduce-client-jobclient:2.3.0', @@ -100,8 +102,8 @@ ext.libraries = [ pulsarClient: "${pulsarGroup}:pulsar-client:${pulsarVersion}", pulsarIoCore: "${pulsarGroup}:pulsar-io-core:${pulsarVersion}", pulsarIoCommon: "${pulsarGroup}:pulsar-io-common:${pulsarVersion}", - r2: 'com.linkedin.pegasus:r2:' + pegasusVersion, - restliCommon: 'com.linkedin.pegasus:restli-common:' + pegasusVersion, + r2: "com.linkedin.pegasus:r2:${pegasusVersion}", + restliCommon: "com.linkedin.pegasus:restli-common:${pegasusVersion}", rocksdbjni: 'org.rocksdb:rocksdbjni:7.9.2', samzaApi: 'org.apache.samza:samza-api:1.5.1', slf4j: 'org.slf4j:slf4j:1.7.36', @@ -119,9 +121,9 @@ ext.libraries = [ zkclient: 'com.101tec:zkclient:0.7', // For Kafka AdminUtils zookeeper: 'org.apache.zookeeper:zookeeper:3.5.9', zstd: 'com.github.luben:zstd-jni:1.5.2-3', - grpcNettyShaded: 'io.grpc:grpc-netty-shaded:' + grpcVersion, - grpcProtobuf: 'io.grpc:grpc-protobuf:' + grpcVersion, - grpcStub: 'io.grpc:grpc-stub:' + grpcVersion, + grpcNettyShaded: "io.grpc:grpc-netty-shaded:${grpcVersion}", + grpcProtobuf: "io.grpc:grpc-protobuf:${grpcVersion}", + grpcStub: "io.grpc:grpc-stub:${grpcVersion}", tomcatAnnotations: 'org.apache.tomcat:annotations-api:6.0.53', ] @@ -155,6 +157,15 @@ def removeSquareBrackets(String testName) { return testName.replaceFirst('\\[[0-9]+\\]', '') } +configurations { + alpnAgent { + } +} + +dependencies { + alpnAgent libraries.alpnAgent +} + subprojects { //apply group and version to all submodules group = rootProject.group @@ -375,14 +386,7 @@ subprojects { if (!project.ext.has("skipAlpnBoot") || !project.ext.skipAlpnBoot) { // The ALPN version should match the JVM version if (JavaVersion.current() < JavaVersion.VERSION_11) { - if (Os.isFamily(Os.FAMILY_MAC)) { - jvmArgs '-Xbootclasspath/p:/Library/Java/Boot/1_8_0_172/alpn-boot-8.1.12.v20180117.jar' - } else if (Os.isFamily(Os.FAMILY_UNIX)) { - jvmArgs '-Xbootclasspath/p:/export/apps/jdkboot/1_8_0_172/alpn-boot-8.1.12.v20180117.jar' - } else { - def osName = System.getProperty('os.name').toLowerCase(Locale.ENGLISH) - throw new GradleException("$osName is not supported since ALPN only works for either Mac or Linux right now.") - } + jvmArgs "-javaagent:${rootProject.configurations.alpnAgent.asPath}" } } diff --git a/gradle.properties b/gradle.properties index a8d8a3d856..91ddb2bb5e 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,14 +1,2 @@ org.gradle.daemon=true -org.gradle.caching=true - -pulsarGroup = org.apache.pulsar -pulsarVersion = 2.10.3 - -org.gradle.jvmargs= \ ---add-exports=java.base/sun.nio.ch=ALL-UNNAMED \ ---add-opens=java.base/java.lang=ALL-UNNAMED \ ---add-opens=java.base/java.lang.reflect=ALL-UNNAMED \ ---add-opens=java.base/java.io=ALL-UNNAMED \ ---add-opens=java.base/sun.net=ALL-UNNAMED \ ---add-exports=jdk.unsupported/sun.misc=ALL-UNNAMED - +org.gradle.caching=true \ No newline at end of file