feat(gas): Support for Ruby 3.4 builds (#355)

feat(gas): Update all dependencies to the current latest
fix(gas): Updated `gas build` to install dependencies automatically if not present, rather than prompt interactively
fix(gas): Added a cleanup between different platform builds, to deal with a rare case where one platform build interferes with another

.github/workflows/ci.yml

@@ -17,6 +17,8 @@ jobs:
             ruby: "3.1"
           - os: ubuntu-latest
             ruby: "3.3"
+          - os: ubuntu-latest
+            ruby: "3.4"
       fail-fast: false
     runs-on: ${{ matrix.os }}
     steps:

.kokoro/gas/trigger-cloud.cfg

@@ -31,7 +31,7 @@ env_vars: {
 # List of minor Ruby versions for ruby-cloud builds, colon-delimited.
 env_vars: {
   key: "GAS_RUBY_VERSIONS"
-  value: "2.7:3.0:3.1:3.2:3.3"
+  value: "3.0:3.1:3.2:3.3:3.4"
 }
 
 # Path to the RubyGems API key file for the google-cloud account.

.kokoro/gas/trigger-generic.cfg

@@ -31,7 +31,7 @@ env_vars: {
 # List of minor Ruby versions for generic builds, colon-delimited.
 env_vars: {
   key: "GAS_RUBY_VERSIONS"
-  value: "2.7:3.0:3.1:3.2:3.3"
+  value: "3.0:3.1:3.2:3.3:3.4"
 }
 
 # This must be set when invoking the job.

.kokoro/gas/trigger-protobuf.cfg

@@ -31,7 +31,7 @@ env_vars: {
 # List of minor Ruby versions for protobuf builds, colon-delimited.
 env_vars: {
   key: "GAS_RUBY_VERSIONS"
-  value: "3.0:3.1:3.2:3.3"
+  value: "3.0:3.1:3.2:3.3:3.4"
 }
 
 # Path to the RubyGems API key file for the protobuf account.

gas/.toys/gas/.data/rakefile.erb

@@ -20,6 +20,21 @@ require "yaml"
 permitted_classes = [Gem::Specification, Gem::Dependency, Gem::Version, Gem::Requirement, Time, Symbol]
 spec = YAML.load_file "<%= gem_name %>-<%= gem_version %>.gemspec", permitted_classes: permitted_classes
 
+task :generate_ruby_cc_version do
+  mapping = {}
+  ENV["RUBY_CC_VERSION"].split(":").each do |version|
+    match = /^(\d+\.\d+)\.\d+$/.match version
+    mapping[match[1]] = version if match
+  end
+  requested = "<%= ruby_versions.join ':' %>".split ":"
+  actual = requested.map do |version|
+    full = mapping[version]
+    raise "Ruby version #{version} not found" unless full
+    full
+  end
+  File.write "ruby-cc-version.txt", actual.join(":")
+end
+
 <% extensions.each do |extension| %>
 Rake::ExtensionTask.new "<%= extension %>", spec do |ext|
   ext.cross_compile = true

gas/.toys/gas/.preload.rb

@@ -31,44 +31,46 @@ module Gas
     "x86_64-darwin",
     "x86_64-linux"
   ]
-  
+
   ##
   # Ruby versions to build if no versions are specified on the command line
   #
-  DEFAULT_RUBIES = ["2.6", "2.7", "3.0", "3.1", "3.2", "3.3"]
+  DEFAULT_RUBIES = ["3.0", "3.1", "3.2", "3.3", "3.4"]
 
   ##
   # Version of the Gems gem to install
   #
-  GEMS_VERSION = "1.2.0"
+  GEMS_VERSION = "1.3.0"
 
   ##
   # Version of Rake to install
   #
-  RAKE_VERSION = "13.0.6"
+  RAKE_VERSION = "13.2.1"
 
   ##
   # Version of Rake-compiler to install
   #
-  RAKE_COMPILER_VERSION = "1.2.5"
+  RAKE_COMPILER_VERSION = "1.2.9"
 
   ##
   # Version of Rake-compiler-dock to install
   #
-  RAKE_COMPILER_DOCK_VERSION = "1.4.0"
+  RAKE_COMPILER_DOCK_VERSION = "1.8.0"
 
   ##
-  # RCD images with pinned hashes, for 1.4.0-mri, keyed by platform
+  # RCD images with pinned hashes, for 1.8.0-mri, keyed by platform.
+  # These MUST be updated when rake-compiler-dock is updated. Find them at:
+  # https://github.com/rake-compiler/rake-compiler-dock/pkgs/container/rake-compiler-dock-image
   #
   RAKE_COMPILER_DOCK_IMAGE = {
-    "aarch64-linux" => "ghcr.io/rake-compiler/rake-compiler-dock-image@sha256:37a993592c084198923e9574cbbde1673cfa167cd106bd14364997f1f9981cac",
-    "arm-linux" => "ghcr.io/rake-compiler/rake-compiler-dock-image@sha256:afd8feda44731292fd16b816187d04d0def636546756d1e6b3887ce86d5d52b1",
-    "arm64-darwin" => "ghcr.io/rake-compiler/rake-compiler-dock-image@sha256:16bb1a0746215557f5577e4cd289e10cec593d74b5a3033c9bb2ab4bc3c6662f",
-    "x64-mingw-ucrt" => "ghcr.io/rake-compiler/rake-compiler-dock-image@sha256:beeb644d381f71490e34d40f9697ff3c25fca0db4917565e058a2b39a5815bc3",
-    "x64-mingw32" => "ghcr.io/rake-compiler/rake-compiler-dock-image@sha256:9e9454d45a0568f5842fa8933e9905882635951864a07bc986133d5b732b51f0",
-    "x86-linux" => "ghcr.io/rake-compiler/rake-compiler-dock-image@sha256:4167d00276c0ddb0eb4eb2a0fc2a882a9983c8fb5f1979297e77ed28d4b2e924",
-    "x86-mingw32" => "ghcr.io/rake-compiler/rake-compiler-dock-image@sha256:1b2852da28c272a817b733ba1d6576e3b588b90a50f6159fd1b5c85a13ec48ff",
-    "x86_64-darwin" => "ghcr.io/rake-compiler/rake-compiler-dock-image@sha256:ba8e38140f69ae8febe01f8b168782ec1f15cd2e59dd61719fd1176404138062",
-    "x86_64-linux" => "ghcr.io/rake-compiler/rake-compiler-dock-image@sha256:8fc4fe7a195a970e0437033f2147eb191b2ed5eea50f0aeaf8e317fdb1f3ff14"
+    "aarch64-linux" => "ghcr.io/rake-compiler/rake-compiler-dock-image@sha256:434ec9a0ab986c1e8021e56fb5d92169b68f50a8479ab57bb7c78288ab5c1bd6",
+    "arm-linux" => "ghcr.io/rake-compiler/rake-compiler-dock-image@sha256:421ab2dfa84a3d6253116cdacabc9a7692f7d588dd50ff42c0d7fd71f458921e",
+    "arm64-darwin" => "ghcr.io/rake-compiler/rake-compiler-dock-image@sha256:0f816ad0b08b0826fe64a43750e3e56663d8a697f753e98292994aa988fc9ccc",
+    "x64-mingw-ucrt" => "ghcr.io/rake-compiler/rake-compiler-dock-image@sha256:71c07d2a3d946110ccee8a612ad1950c1809161b35d68e337037f82ebb58879a",
+    "x64-mingw32" => "ghcr.io/rake-compiler/rake-compiler-dock-image@sha256:1fabf2a37c13610cc6fa3311c319a352c17683f03e0a25921122d5cb8f6dce03",
+    "x86-linux" => "ghcr.io/rake-compiler/rake-compiler-dock-image@sha256:23d6452730ae98eca32c39cbdcaa08204a877bd86958c192b9824b0bac251160",
+    "x86-mingw32" => "ghcr.io/rake-compiler/rake-compiler-dock-image@sha256:550a7cf8685b8844c024b3395b2642facd1679a0796afc3ec7120512b27c8afd",
+    "x86_64-darwin" => "ghcr.io/rake-compiler/rake-compiler-dock-image@sha256:c5af54f9f41da6d55f1b8f802923469d870cc9d28918086eaa8a79a18ba2337c",
+    "x86_64-linux" => "ghcr.io/rake-compiler/rake-compiler-dock-image@sha256:c2b005647d54a5a04502b584bc036886c876f6f2384fb6e7bf21f5616fbae853"
   }
 end

gas/.toys/gas/.test/test_build.rb

@@ -29,9 +29,9 @@
     version = match[1].to_i * 100 + match[2].to_i
     version < 302
   }
-  let(:gem_version) { use_older_example ? "3.21.12" : "3.25.2" }
+  let(:gem_version) { @gem_version_override || (use_older_example ? "3.21.12" : "3.25.2") }
   let(:gem_and_version) { "google-protobuf-#{gem_version}" }
-  let(:gem_data_dir) { use_older_example ? "data31" : "data32" }
+  let(:gem_data_dir) { @gem_version_override ? "data33" : use_older_example ? "data31" : "data32" }
   let(:source_gem) { File.join __dir__, gem_data_dir, "#{gem_and_version}.gem" }
   let(:workspace_dir) { Dir.mktmpdir }
   let(:linux_platforms) { ["x86_64-linux", "x86-linux", "aarch64-linux"] }
@@ -43,22 +43,28 @@
   let(:excluded_combinations) { [["x64-mingw32", "3.1"], ["x64-mingw-ucrt", "2.7"]] }
   let(:host_platform) { "#{`uname -m`.strip}-#{`uname -s`.strip.downcase}" }
   let(:host_ruby_version) { RUBY_VERSION.sub(/^(\d+\.\d+).*$/, "\\1") }
+  let(:multi_rubies) { ["3.0", "3.1", "3.2", "3.3", "3.4" ]}
+  let(:gem_version_for_multi_rubies) { "4.29.2" }
+  let(:platform_for_multi_rubies) { "x86_64-linux" }
 
   # Invoke the gas build tool within the test
   def quiet_build platforms, rubies
     result = 0
     out, err = capture_subprocess_io do
-      result = toys_run_tool [
+      result = toys_run_tool([
         "gas", "build", "google-protobuf", gem_version,
         "--workspace-dir", workspace_dir,
         "--source-gem", source_gem,
         "--platform", Array(platforms).join(","),
         "--ruby", Array(rubies).join(","),
-        "--yes"
-      ]
+        "--yes",
+        "--verbose"
+      ])
     end
     unless result.zero?
+      puts "\n******** OUT"
       puts out
+      puts "******** ERR"
       puts err
       flunk "Failed to run gas build: result = #{result}"
     end
@@ -70,6 +76,19 @@ def quiet_build platforms, rubies
     FileUtils.rm_rf workspace_dir
   end
 
+  it "generates various Ruby versions for protobuf" do
+    @gem_version_override = gem_version_for_multi_rubies
+    quiet_build platform_for_multi_rubies, multi_rubies
+    Dir.chdir "#{workspace_dir}/#{gem_and_version}/pkg/" do
+      assert File.exist? "#{gem_and_version}-#{platform_for_multi_rubies}.gem"
+      FileUtils.rm_r "#{gem_and_version}-#{platform_for_multi_rubies}"
+      exec_service.exec ["gem", "unpack", "#{gem_and_version}-#{platform_for_multi_rubies}.gem"], out: :null
+      multi_rubies.each do |ruby|
+        assert File.exist? "#{gem_and_version}-#{platform_for_multi_rubies}/lib/google/#{ruby}/protobuf_c.so"
+      end
+    end
+  end
+
   it "generates linux platforms for protobuf" do
     quiet_build linux_platforms, host_ruby_version
     Dir.chdir "#{workspace_dir}/#{gem_and_version}/pkg/" do

gas/.toys/gas/build.rb

@@ -62,7 +62,7 @@
 
 include :exec, e: true
 include :fileutils
-include :gems
+include :gems, on_missing: :install
 include :terminal
 
 # Main entrypoint
@@ -160,6 +160,7 @@ def perform_builds
   require "rake_compiler_dock"
   success = true
   platforms.each do |platform|
+    rm_rf "tmp"
     success &&= build_platform(platform)
   end
   exit 1 unless success
@@ -172,7 +173,9 @@ def build_platform platform
   begin
     logger.info "Building #{platform} ..."
     ENV["RCD_IMAGE"] = Gas::RAKE_COMPILER_DOCK_IMAGE[platform]
-    script = "bundle install --local && bundle exec rake native:#{platform} gem RUBY_CC_VERSION=#{ruby_cc_version}"
+    script = "bundle install --local " \
+      "&& bundle exec rake generate_ruby_cc_version " \
+      "&& bundle exec rake native:#{platform} gem RUBY_CC_VERSION=$(cat ruby-cc-version.txt)"
     RakeCompilerDock.sh script, platform: platform
     artifact_name = "pkg/#{gem_name}-#{gem_version}-#{platform}.gem"
     raise "Expected artifact #{artifact_name} not produced" unless File.file? artifact_name
@@ -195,8 +198,3 @@ def gemspec
     YAML.load_file "#{gem_name}-#{gem_version}.gemspec", permitted_classes: permitted_classes
   end
 end
-
-# Construct the RUBY_CC_VERSION environment variable format
-def ruby_cc_version
-  ruby_versions.map { |ruby| "#{ruby}.0" }.join ":"
-end