Merge branch 'master' into v1.10.5-patch-struct

amazon-ion · Nov 1, 2023 · af7f3f1 · af7f3f1
2 parents 54b93c4 + 9a74c6a
commit af7f3f1
Show file tree

Hide file tree

Showing 85 changed files with 11,387 additions and 11,826 deletions.
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -37,11 +37,11 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@6a28655e3dcb49cb0840ea372fd6d17733edd8a4 # v2.21.8
+        uses: github/codeql-action/init@0116bc2df50751f9724a2e35ef1f24d22f90e4e1 # v2.22.3
         with:
           languages: ${{ matrix.language }}
           # If you wish to specify custom queries, you can do so here or in a config file.
@@ -52,7 +52,7 @@ jobs:
       # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
       # If this step fails, then you should remove it and run the build manually (see below)
       - name: Autobuild
-        uses: github/codeql-action/autobuild@6a28655e3dcb49cb0840ea372fd6d17733edd8a4 # v2.21.8
+        uses: github/codeql-action/autobuild@0116bc2df50751f9724a2e35ef1f24d22f90e4e1 # v2.22.3
 
       # ℹ️ Command-line programs to run using the OS shell.
       # 📚 https://git.io/JvXDl
@@ -66,4 +66,4 @@ jobs:
       #   make release
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@6a28655e3dcb49cb0840ea372fd6d17733edd8a4 # v2.21.8
+        uses: github/codeql-action/analyze@0116bc2df50751f9724a2e35ef1f24d22f90e4e1 # v2.22.3
diff --git a/.github/workflows/ion-java-performance-regression-detector.yml b/.github/workflows/ion-java-performance-regression-detector.yml
@@ -6,145 +6,118 @@ name: Ion Java performance regression detector
 on:
   pull_request:
     paths:
-      - 'src/*'
+      - 'src/**'
 
 permissions:
   contents: read
 
+env:
+  data_size: '50000'
+  test_data_id: 'generated-test-data'
+
 jobs:
-  detect-regression:
-    name: Detect Regression
+  generate-test-data:
+    name: Generate Test Data
     runs-on: ubuntu-latest
-
     steps:
-      - name: Set up JDK 11
-        uses: actions/setup-java@cd89f46ac9d01407894225f350157564c9c7cee2 # v3.12.0
-        with:
-          distribution: 'corretto'
-          java-version: 11
-
-      - name: Checkout ion-java from the new commit.
-        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-          path: ion-java-new
-
-      - name: Build ion-java from the new commit
-        run: cd ion-java-new && git submodule init && git submodule update && ./gradlew clean publishToMavenLocal
-
-      - name: Checkout ion-java-benchmark-cli
-        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
-        with:
-          repository: amazon-ion/ion-java-benchmark-cli
-          ref: master
-          path: ion-java-benchmark-cli
-
-      - name: Build ion-java-benchmark-cli
-        run: cd ion-java-benchmark-cli && mvn clean install
-
       - name: Checkout ion-data-generator
-        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
           repository: amazon-ion/ion-data-generator
           ref: main
-          path: ion-data-generator
 
       - name: Build ion-data-generator
-        run: cd ion-data-generator && mvn clean install
+        run: mvn clean install
 
-      - name: Check the version of ion-java.
-        run: java -jar ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar --version
-
-      - name: Generate test Ion Data
+      - name: Generate test data
+        env:
+          jar_file: target/ion-data-generator-1.0-SNAPSHOT.jar
+          schema_dir: tst/com/amazon/ion/workflow
         run: |
           mkdir -p testData
-          java -jar ion-data-generator/target/ion-data-generator-1.0-SNAPSHOT.jar  generate -S 50000 --input-ion-schema ion-data-generator/tst/com/amazon/ion/workflow/nestedStruct.isl testData/testStructs.10n
-          java -jar ion-data-generator/target/ion-data-generator-1.0-SNAPSHOT.jar  generate -S 50000 --input-ion-schema ion-data-generator/tst/com/amazon/ion/workflow/nestedList.isl testData/testLists.10n
-          java -jar ion-data-generator/target/ion-data-generator-1.0-SNAPSHOT.jar  generate -S 50000 --input-ion-schema ion-data-generator/tst/com/amazon/ion/workflow/sexp.isl testData/testSexps.10n
-          java -jar ion-data-generator/target/ion-data-generator-1.0-SNAPSHOT.jar  generate -S 50000 --input-ion-schema ion-data-generator/tst/com/amazon/ion/workflow/realWorldDataSchema01.isl testData/realWorldData01.10n
-          java -jar ion-data-generator/target/ion-data-generator-1.0-SNAPSHOT.jar  generate -S 50000 --input-ion-schema ion-data-generator/tst/com/amazon/ion/workflow/realWorldDataSchema02.isl testData/realWorldData02.10n
-          java -jar ion-data-generator/target/ion-data-generator-1.0-SNAPSHOT.jar  generate -S 50000 --input-ion-schema ion-data-generator/tst/com/amazon/ion/workflow/realWorldDataSchema03.isl testData/realWorldData03.10n
+          for test in nestedStruct nestedList sexp realWorldDataSchema01 realWorldDataSchema02 realWorldDataSchema03
+          do
+          java -jar $jar_file generate -S ${{env.data_size}} --input-ion-schema $schema_dir/${test}.isl testData/${test}.10n
+          done
 
       - name: Upload test Ion Data to artifacts
-        uses: actions/upload-artifact@82c141cc518b40d92cc801eee768e7aafc9c2fa2 # v2.3.1
+        uses: actions/upload-artifact@v3
         with:
-          name: test Ion Data
+          name: ${{env.test_data_id}}
           path: testData
 
-      - name: Benchmark ion-java from the new commit
-        run: |
-          mkdir -p benchmarkResults
-          cd ion-java-benchmark-cli && java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar run-suite --test-ion-data /home/runner/work/ion-java/ion-java/testData --benchmark-options-combinations tst/com/amazon/ion/benchmark/optionsCombinations.ion /home/runner/work/ion-java/ion-java/benchmarkResults
+  detect-regression:
+    name: Detect Regression
+    runs-on: ubuntu-latest
+    needs: generate-test-data
+    strategy:
+      matrix:
+        test-data: ['nestedStruct', 'nestedList', 'sexp', 'realWorldDataSchema01', 'realWorldDataSchema02', 'realWorldDataSchema03']
+        usage-patterns: [' read --mode AverageTime --forks 3 --warmups 2 --iterations 10 --api streaming ',' read --mode AverageTime --forks 3 --warmups 2 --iterations 10 --api dom ', ' write --mode AverageTime --forks 3 --warmups 2 --iterations 10 --api streaming --io-type buffer --ion-length-preallocation 1 ', ' write --mode AverageTime --forks 3 --warmups 2 --iterations 10 --api dom --io-type buffer --ion-length-preallocation 1 ']
+      fail-fast: false
 
-      - name: Upload benchmark results to artifacts
-        uses: actions/upload-artifact@82c141cc518b40d92cc801eee768e7aafc9c2fa2 # v2.3.1
+    steps:
+      - name: Set up JDK 11
+        uses: actions/setup-java@v3
         with:
-          name: Benchmark result
-          path: benchmarkResults
-
-      - name: Clean maven dependencies repository
-        run : rm -r /home/runner/.m2
+          distribution: 'corretto'
+          java-version: 11
 
-      - name: Checkout the current commit
-        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+      - name: Checkout ion-java-benchmark-cli
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
-          repository: amazon-ion/ion-java
+          repository: amazon-ion/ion-java-benchmark-cli
           ref: master
-          path: ion-java
-
-      - name: Build ion-java from the previous commit
-        run: cd ion-java && git submodule init && git submodule update && ./gradlew clean publishToMavenLocal
-
-      - name: Build ion-java-benchmark-cli
-        run: cd ion-java-benchmark-cli && mvn clean install
+          path: ion-java-benchmark-cli
 
-      - name: Check the version of ion-java
-        run: java -jar ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar --version
+      - name: Checkout ion-java from the previous commit
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          ref: ${{ github.base_ref }}
+          submodules: recursive
+          path: baseline
 
-      - name: Create directories for test data and benchmark results
-        run: |
-          mkdir -p benchmarkResults
-          mkdir -p testData
+      - name: Checkout ion-java from the new commit.
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+          ref: ${{ github.head_ref }}
+          submodules: recursive
+          path: new
 
       - name: Download test Ion Data from artifacts
-        uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # v2.1.1
+        uses: actions/download-artifact@v3
         with:
-          name: test Ion Data
+          name: generated-test-data
           path: testData
 
-      - name: Download benchmark results of ion-java from the new commit from artifacts
-        uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # v2.1.1
-        with:
-          name: Benchmark result
-          path: benchmarkResults
-
-      - name: Benchmark ion-java from the previous commit and add the generated benchmark results to the existing directories
-        run: cd ion-java-benchmark-cli && java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar run-suite --test-ion-data /home/runner/work/ion-java/ion-java/testData --benchmark-options-combinations tst/com/amazon/ion/benchmark/optionsCombinations.ion /home/runner/work/ion-java/ion-java/benchmarkResults
+      # Get the benchmark results from the base commit
+      - name: Build ion-java from the base commit
+        working-directory: baseline
+        run: |
+          ./gradlew clean publishToMavenLocal
 
-      - name: Upload new benchmark results directory to artifacts
-        uses: actions/upload-artifact@82c141cc518b40d92cc801eee768e7aafc9c2fa2 # v2.3.1
-        with:
-          name: Benchmark result
-          path: benchmarkResults
+      - name: Benchmark ion-java from the base commit
+        working-directory: ion-java-benchmark-cli
+        run: |
+          mvn clean install
+          mkdir /home/runner/work/ion-java/ion-java/benchmarkresults
+          java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar ${{matrix.usage-patterns}} -r ion -o /home/runner/work/ion-java/ion-java/benchmarkresults/before.ion /home/runner/work/ion-java/ion-java/testData/${{matrix.test-data}}.10n 
 
-      - name: Detect performance regression
-        id: regression_result
+      - name: Build ion-java from the new commit
+        working-directory: new
         run: |
-          result=true
-          cd benchmarkResults && for FILE in *; do message=$(java -jar /home/runner/work/ion-java/ion-java/ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar compare --benchmark-result-previous $FILE/previous.ion --benchmark-result-new $FILE/new.ion $FILE/report.ion | tee /dev/stderr) && if [ "$message" != "no regression detected" ]; then result=false; fi; done
-          echo "::set-output name=regression-result::$result"
-          if [ "$result" = "true" ]; then echo "No regression detected!" >> $GITHUB_STEP_SUMMARY; fi
+          ./gradlew clean publishToMavenLocal
 
-      - name: Upload comparison reports to the benchmark results directory
-        uses: actions/upload-artifact@82c141cc518b40d92cc801eee768e7aafc9c2fa2 # v2.3.1
-        with:
-          name: Benchmark result
-          path: benchmarkResults
+      - name: Benchmark ion-java from the new commit
+        working-directory: ion-java-benchmark-cli
+        run: |
+          mvn clean install
+          java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar ${{matrix.usage-patterns}} -r ion -o /home/runner/work/ion-java/ion-java/benchmarkresults/after.ion /home/runner/work/ion-java/ion-java/testData/${{matrix.test-data}}.10n
 
-      - name: Fail the workflow if regression happened
-        env:
-          regression_detect: ${{steps.regression_result.outputs.regression-result}}
-        if: ${{ env.regression_detect == 'false' }}
+      #Detect regression
+      - name: Detect regression
+        working-directory: ion-java-benchmark-cli
         run: |
-          cd benchmarkResults && echo "| Benchmark command | GC Allocation Rate | Heap Usage | Speed |" >> $GITHUB_STEP_SUMMARY && echo "| ----------- | ----------- |----------- | ----------- |" >> $GITHUB_STEP_SUMMARY && for FILE in *; do regressionDetection=$(java -jar /home/runner/work/ion-java/ion-java/ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar compare --benchmark-result-previous $FILE/previous.ion --benchmark-result-new $FILE/new.ion $FILE/report.ion) && if [ "$regressionDetection" != "no regression detected" ]; then command=$(echo $FILE | sed "s/_/ /g") && read gc heapUsage speed <<< $( echo ${regressionDetection} | awk -F", " '{print $1" "$2" "$3}' ) && echo "|$command|$gc|$heapUsage|$speed|" >> $GITHUB_STEP_SUMMARY; fi; done
-          exit 1
+          a=$(java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar compare --benchmark-result-previous /home/runner/work/ion-java/ion-java/benchmarkresults/before.ion --benchmark-result-new /home/runner/work/ion-java/ion-java/benchmarkresults/after.ion)
+          if [ ! -z "$a" ]; then echo "${a}" >> $GITHUB_STEP_SUMMARY; exit 1; fi
diff --git a/.github/workflows/ion-test-driver.yml b/.github/workflows/ion-test-driver.yml
@@ -3,7 +3,7 @@ name: ion-test-driver
 on:
   pull_request:
     paths:
-      - 'src/*'
+      - 'src/**'
 
 permissions:
   contents: read
@@ -13,14 +13,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout ion-java
-        uses: actions/checkout@61b9e3751b92087fd0b06925ba6dd6314e06f089 # master
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # master
         with:
           repository: amazon-ion/ion-java
           ref: master
           path: ion-java
 
       - name: Checkout ion-test-driver
-        uses: actions/checkout@61b9e3751b92087fd0b06925ba6dd6314e06f089 # master
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # master
         with:
           repository: amazon-ion/ion-test-driver
           ref: master
@@ -45,7 +45,7 @@ jobs:
           --replace ion-java,https://github.com/amazon-ion/ion-java.git,$main
 
       - name: Upload result
-        uses: actions/upload-artifact@82c141cc518b40d92cc801eee768e7aafc9c2fa2 # v2.3.1
+        uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
         with:
           name: ion-test-driver-result.ion
           path: output/results/ion-test-driver-results.ion
@@ -60,7 +60,7 @@ jobs:
           ion-java,$main ion-java,$cur output/results/ion-test-driver-results.ion
 
       - name: Upload analysis report
-        uses: actions/upload-artifact@82c141cc518b40d92cc801eee768e7aafc9c2fa2 # v2.3.1
+        uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
         with:
           name: analysis-report.ion
           path: result.ion
@@ -80,7 +80,7 @@ jobs:
     needs: ion-test-driver
     if: ${{ failure() }}
     steps:
-      - uses: actions/checkout@61b9e3751b92087fd0b06925ba6dd6314e06f089 # master
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # master
       - name: Open an issue
         uses: JasonEtco/create-an-issue@e27dddc79c92bc6e4562f268fffa5ed752639abd # v2.9.1
         env:

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -1,4 +1,5 @@
 name: build
+permissions: read-all
 
 on:
   push:
@@ -17,16 +18,16 @@ jobs:
             upload_reports: true
           - java: 17
     steps:
-      - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
           submodules: recursive
       - uses: gradle/wrapper-validation-action@56b90f209b02bf6d1deae490e9ef18b21a389cd4 # v1.1.0
       - name: Use java ${{ matrix.java }}
-        uses: actions/setup-java@cd89f46ac9d01407894225f350157564c9c7cee2 # v3.12.0
+        uses: actions/setup-java@0ab4596768b603586c0de567f2430c30f5b0d2b0 # v3.13.0
         with:
           distribution: 'corretto'
           java-version: ${{ matrix.java }}
-      - uses: gradle/gradle-build-action@ef76a971e2fa3f867b617efd72f2fbd72cf6f8bc # v2.8.0
+      - uses: gradle/gradle-build-action@842c587ad8aa4c68eeba24c396e15af4c2e9f30a # v2.9.0
         with:
           arguments: build
       - run: ./ion-test-driver-run version
@@ -46,7 +47,7 @@ jobs:
     # https://github.com/amazon-ion/ion-java/blob/master/.github/actions/inspect-version/action.yml
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3.6.0
       - name: Get Project Version
         run: |
           echo "PROJECT_VERSION=v$(<project.version)" >> $GITHUB_ENV

diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml
@@ -24,6 +24,7 @@
 # See https://docs.github.com/en/actions/creating-actions/creating-a-composite-action
 
 name: Prepare Release
+permissions: read-all
 on:
   # We have a branch protection rule for master, so all commits to master require
   # an approved PR. That means this workflow trigger is effectively equivalent to
@@ -43,7 +44,7 @@ jobs:
     outputs:
       should_create_draft: ${{ steps.inspect.outputs.is_valid_to_release }}
     steps:
-      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3.6.0
       - name: Get project version
         run: |
           echo "PROJECT_VERSION=$(<project.version)" >> $GITHUB_ENV
@@ -64,7 +65,7 @@ jobs:
     permissions:
       contents: write
     steps:
-      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3.6.0
       - name: Create a draft release
         shell: bash
         env:

diff --git a/.github/workflows/publish-release-artifacts.yml b/.github/workflows/publish-release-artifacts.yml
@@ -29,7 +29,7 @@ jobs:
     # First, a sanity check to ensure that the library version matches the release version
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3.6.0
       - name: Validate project version matches tag
         shell: bash
         run: |
@@ -48,10 +48,10 @@ jobs:
     steps:
         # TODO: replace with artifact upload/download -- make sure there's no race condition with other builds also
         # uploading an artifact.
-      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3.6.0
         with:
           submodules: recursive
-      - uses: gradle/gradle-build-action@ef76a971e2fa3f867b617efd72f2fbd72cf6f8bc # v2.8.0
+      - uses: gradle/gradle-build-action@842c587ad8aa4c68eeba24c396e15af4c2e9f30a # v2.9.0
         with:
           arguments: build cyclonedxBom
       - name: Upload Jar to GitHub release