Scan new versions for existing libraries

.github/workflows/check-new-library-versions.yml

@@ -0,0 +1,142 @@
+name: "Check new library versions"
+
+on:
+  schedule:
+    - cron: "0 0 8 * *"
+    - cron: "0 0 22 * *"
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  actions: write
+
+concurrency:
+  group: "workflow = ${{ github.workflow }}, ref = ${{ github.event.ref }}, pr = ${{ github.event.pull_request.id }}"
+  cancel-in-progress: true
+
+jobs:
+  get-all-libraries:
+    if: github.repository == 'oracle/graalvm-reachability-metadata'
+    name: "📋 Get list of all supported libraries with newer versions"
+    permissions: write-all
+    runs-on: "ubuntu-20.04"
+    timeout-minutes: 5
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+      issue: ${{ steps.set-issue.outputs.issue }}
+    steps:
+      - name: "☁️ Checkout repository"
+        uses: actions/checkout@v4
+      - name: "🔧 Prepare environment"
+        uses: graalvm/setup-graalvm@v1
+        with:
+          java-version: '21'
+          distribution: 'graalvm'
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: "🕸️ Populate matrix"
+        id: set-matrix
+        run: |
+          ./gradlew getExistingLibrariesWithNewVersions -PlibraryMatrixLimit=200
+      - name: "🔨 Create branch"
+        run: |
+          git config --local user.email "[email protected]"
+          git config --local user.name "Github Actions"
+          git switch -C libraryVersionsDependabot/$(date '+%Y-%m-%d')
+          git push origin libraryVersionsDependabot/$(date '+%Y-%m-%d')
+      - name: "🔨 Create issue"
+        id: set-issue
+        run: |
+          git config --local user.email "[email protected]"
+          git config --local user.name "Github Actions"
+
+          issue_url=$(gh issue create --title "List unsupported library versions" --body "This issue lists unsupported versions of the existing libraries in the repo")
+          echo "::set-output name=issue::$issue_url"
+
+  test-all-metadata:
+    name: "🧪 ${{ matrix.coordinates }} (GraalVM for JDK ${{ matrix.version }} @ ${{ matrix.os }})"
+    permissions: write-all
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 20
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    needs: get-all-libraries
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.get-all-libraries.outputs.matrix) }}
+    steps:
+      - name: "☁️ Checkout repository"
+        uses: actions/checkout@v4
+      - name: "🔧 Setup java"
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'oracle'
+          java-version: '21'
+      - name: "🔧 Prepare environment"
+        uses: graalvm/setup-graalvm@v1
+        with:
+          set-java-home: 'false'
+          java-version: ${{ matrix.version }}
+          distribution: 'graalvm'
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          native-image-job-reports: 'true'
+      - name: "Extract test path and library version"
+        run: |
+          LIBRARY_PATH=$(echo ${{ matrix.coordinates }} | cut -d ':' -f1-2 | sed 's/:/\//g')
+          LATEST_VERSION=$(find tests/src/$LIBRARY_PATH/* -maxdepth 1 -type d | sort -V | tail -1 | cut -d '/' -f5)
+          TEST_PATH="$LIBRARY_PATH/$LATEST_VERSION"
+          TEST_COORDINATES=$(echo "$TEST_PATH" | tr / :)
+
+          echo "LATEST_VERSION=$LATEST_VERSION" >> ${GITHUB_ENV}
+          echo "TEST_PATH=$TEST_PATH" >> ${GITHUB_ENV}
+          echo "TEST_COORDINATES=$TEST_COORDINATES" >> ${GITHUB_ENV}
+      - name: "Pull allowed docker images"
+        run: |
+          ./gradlew pullAllowedDockerImages --coordinates=${{ env.TEST_COORDINATES }}
+      - name: "Disable docker"
+        run: |
+          sudo apt-get install openbsd-inetd
+          sudo bash -c "cat ./.github/workflows/discard-port.conf >> /etc/inetd.conf"
+          sudo systemctl start inetd
+          sudo mkdir /etc/systemd/system/docker.service.d
+          sudo bash -c "cat ./.github/workflows/dockerd.service > /etc/systemd/system/docker.service.d/http-proxy.conf"
+          sudo systemctl daemon-reload
+          sudo systemctl restart docker
+      - name: "🧪 Run '${{ env.TEST_COORDINATES }}' tests"
+        run: |
+          TESTING_VERSION=$(echo ${{ matrix.coordinates }} | cut -d ":" -f3)
+          export GVM_TCK_LV=$TESTING_VERSION
+
+          ./gradlew test -Pcoordinates=${{ env.TEST_COORDINATES }}
+      - name: "✔️ New library is supported"
+        if: success()
+        run: |
+          sh ./.github/workflows/tryPushVersionsUpdate.sh ${{ matrix.coordinates }} ${{ env.LATEST_VERSION }}
+      - name: "❗ New library is not supported"
+        if: failure()
+        run: |
+          git config --local user.email "[email protected]"
+          git config --local user.name "Github Actions"
+          gh issue comment "${{ needs.get-all-libraries.outputs.issue }}" --body "${{ matrix.coordinates }}"
+
+  process-results:
+    name: "🧪 Process results"
+    runs-on: "ubuntu-20.04"
+    if: ${{ always() }}
+    needs:
+      - get-all-libraries
+      - test-all-metadata
+    permissions: write-all
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - name: "☁️ Checkout repository"
+        uses: actions/checkout@v4
+      - name: "✏️ PR for supported versions"
+        run: |
+          git config --local user.email "[email protected]"
+          git config --local user.name "Github Actions"
+          git fetch origin libraryVersionsDependabot/$(date '+%Y-%m-%d')
+          git checkout libraryVersionsDependabot/$(date '+%Y-%m-%d')
+          gh pr create --title "Update supported library versions" --body "This pull request updates supported versions of the existing libraries in the repo"

.github/workflows/tryPushVersionsUpdate.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+set -x
+
+git config --local user.email "[email protected]"
+git config --local user.name "Github Actions"
+
+BRANCH="libraryVersionsDependabot/$(date '+%Y-%m-%d')"
+git fetch origin "$BRANCH"
+git checkout "$BRANCH"
+
+while [ true ]
+do
+  ./gradlew addTestedVersion --coordinates="$1" --lastSupportedVersion="$2"
+  git add -u
+  git commit -m "$1"
+  if [ "$(git rev-list --count origin/$BRANCH --not $BRANCH)" -eq 0 ]
+  then
+  git push origin "$BRANCH"
+  PUSH_RETVAL=$?
+  if [ "$PUSH_RETVAL" -eq 0 ]
+  then
+    break
+  fi
+  fi
+
+  git reset --hard HEAD~1
+  git fetch origin "$BRANCH"
+  git rebase -X theirs "origin/$BRANCH"
+done
+
+

tests/tck-build-logic/src/main/groovy/org.graalvm.internal.tck-harness.gradle

@@ -14,6 +14,7 @@ import org.graalvm.internal.tck.DockerTask
 import org.graalvm.internal.tck.ConfigFilesChecker
 import org.graalvm.internal.tck.ScaffoldTask
 import org.graalvm.internal.tck.GrypeTask
+import org.graalvm.internal.tck.TestedVersionUpdaterTask
 import org.graalvm.internal.tck.harness.tasks.CheckstyleInvocationTask
 import org.graalvm.internal.tck.harness.tasks.TestInvocationTask
 
@@ -102,6 +103,59 @@ def matrixDefault = [
 
 final String METADATA_GROUP = "Metadata"
 
+List<String> getNewerVersionsFor(String library, String startingVersion) {
+    def baseUrl = "https://repo1.maven.org/maven2"
+    String group = library.split(":")[0].replace(".", "/")
+    String artifact = library.split(":")[1]
+    def data = new URL(baseUrl + "/" + group + "/" + artifact + "/" + "maven-metadata.xml").getText()
+
+    return tck.getNewerVersionsFromLibraryIndex(data, startingVersion, library)
+}
+
+// gradle getExistingLibrariesWithNewVersions
+Provider<Task> getExistingLibrariesWithNewVersions = tasks.register("getExistingLibrariesWithNewVersions", DefaultTask) { task ->
+    task.setDescription("Returns list of all libraries coordinates")
+    task.setGroup(METADATA_GROUP)
+    task.doFirst {
+        if (!project.hasProperty("libraryMatrixLimit")) {
+            throw new IllegalArgumentException("Please provide the maximum number of libraries versions!")
+        }
+
+        // get all existing libraries
+        Set<String> libraries = []
+        matchingCoordinates.forEach {
+            libraries.add(it.substring(0, it.lastIndexOf(":")))
+        }
+
+        // foreach existing library find newer versions than the latest one tested except for infrastructure tests
+        List<String> infrastructureTests = List.of("samples", "org.example")
+        List<String> newerVersions = new ArrayList<>()
+        libraries.forEach {
+            String libraryName = it
+            if (infrastructureTests.stream().noneMatch(testName -> libraryName.startsWith(testName))) {
+                List<String> versions = getNewerVersionsFor(libraryName, tck.getLatestLibraryVersion(libraryName))
+                versions.forEach {
+                    newerVersions.add(libraryName.concat(":").concat(it))
+                }
+            }
+        }
+
+        // number of discovered versions is potentially huge and we want to limit it
+        int limit = Integer.parseInt((String) project.findProperty("libraryMatrixLimit"))
+        if (newerVersions.size() > limit) {
+            newerVersions = newerVersions.subList(0, limit)
+        }
+
+        def matrix = [
+                "coordinates" : newerVersions,
+                "version"     : ["17"],
+                "os"          : ["ubuntu-latest"]
+        ]
+
+        println "::set-output name=matrix::${JsonOutput.toJson(matrix)}"
+    }
+}
+
 // gradle generateMatrixMatchingCoordinates -Pcoordinates=<maven-coordinates>
 Provider<Task> generateMatrixMatchingCoordinates = tasks.register("generateMatrixMatchingCoordinates", DefaultTask) { task ->
     task.setDescription("Returns matrix definition populated with all matching coordinates")
@@ -182,3 +236,8 @@ tasks.register("checkConfigFiles", ConfigFilesChecker.class) { task ->
     task.setDescription("Checks content of config files for a new library.")
     task.setGroup(METADATA_GROUP)
 }
+
+tasks.register("addTestedVersion", TestedVersionUpdaterTask.class) { task ->
+    task.setDescription("Updates list of tested versions.")
+    task.setGroup(METADATA_GROUP)
+}

tests/tck-build-logic/src/main/groovy/org/graalvm/internal/tck/harness/TckExtension.java

@@ -6,31 +6,33 @@
  */
 package org.graalvm.internal.tck.harness;
 
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import org.graalvm.internal.tck.model.MetadataVersionsIndexEntry;
 import org.gradle.api.Project;
 import org.gradle.api.file.Directory;
 import org.gradle.api.file.DirectoryProperty;
 import org.gradle.api.file.FileSystemLocation;
 import org.gradle.api.provider.Property;
 import org.gradle.api.provider.Provider;
 import org.gradle.process.ExecOperations;
+import org.gradle.util.internal.VersionNumber;
 
 import javax.inject.Inject;
 import java.io.ByteArrayOutputStream;
+import java.io.File;
 import java.io.IOException;
 import java.net.URI;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
+import java.util.*;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -351,4 +353,62 @@ List<String> getMetadataFileList(Path directory) throws IOException {
             return foundFiles;
         }
     }
+
+    String getLatestLibraryVersion(String libraryModule) {
+        try {
+            List<String> coordinates = List.of(libraryModule.split(":"));
+            String group = coordinates.get(0);
+            String artifact = coordinates.get(1);
+
+            File coordinatesMetadataIndex = new File("metadata/" + group + "/" + artifact +"/index.json");
+            ObjectMapper objectMapper = new ObjectMapper()
+                    .enable(SerializationFeature.INDENT_OUTPUT)
+                    .setSerializationInclusion(JsonInclude.Include.NON_NULL);
+
+            List<MetadataVersionsIndexEntry> entries = objectMapper.readValue(coordinatesMetadataIndex, new TypeReference<>() {
+            });
+
+            List<String> allTested = new ArrayList<>();
+            for (MetadataVersionsIndexEntry entry : entries) {
+                allTested.addAll(entry.testedVersions());
+            }
+
+            if (allTested.isEmpty()) {
+                throw new IllegalStateException("Cannot find any tested version for: " + libraryModule);
+            }
+
+            allTested.sort(Comparator.comparing(VersionNumber::parse));
+            return allTested.get(allTested.size() - 1);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    List<String> getNewerVersionsFromLibraryIndex(String index, String startingVersion, String libraryName) {
+        Pattern pattern = Pattern.compile("<version>(.*)</version>");
+        Matcher matcher = pattern.matcher(index);
+        List<String> allVersions = new ArrayList<>();
+
+        if (matcher.groupCount() < 1) {
+            throw new RuntimeException("Cannot find versions in the given index file: " + libraryName);
+        }
+
+        while (matcher.find()) {
+            allVersions.add(matcher.group(1));
+        }
+
+        int indexOfStartingVersion = allVersions.indexOf(startingVersion);
+        if (indexOfStartingVersion < 0) {
+            System.out.println("Cannot find starting version in index file: " + libraryName + " for version " + startingVersion);
+            return new ArrayList<>();
+        }
+
+        allVersions = allVersions.subList(indexOfStartingVersion, allVersions.size());
+        if (allVersions.size() <= 1) {
+            System.out.println("Cannot find newer versions for " + libraryName + " after the version " + startingVersion);
+        }
+
+        return allVersions.subList(1, allVersions.size());
+    }
+
 }

tests/tck-build-logic/src/main/groovy/org/graalvm/internal/tck/harness/tasks/AbstractSubprojectTask.groovy

@@ -93,7 +93,10 @@ abstract class AbstractSubprojectTask extends DefaultTask {
         // Environment variables for setting up TCK
         env.put("GVM_TCK_LC", coordinates)
         env.put("GVM_TCK_EXCLUDE", override.toString())
-        env.put("GVM_TCK_LV", version)
+        if (System.getenv("GVM_TCK_LV") == null) {
+            // we only set this env variable if user didn't specify it manually
+            env.put("GVM_TCK_LV", version)
+        }
         env.put("GVM_TCK_MD", metadataDir.toAbsolutePath().toString())
         env.put("GVM_TCK_TCKDIR", tckExtension.getTckRoot().get().getAsFile().toPath().toAbsolutePath().toString())
         spec.environment(env)