Allow configuring WASI SDK version per Ruby version

Author: kateinoigakukun

Steepfile

@@ -19,6 +19,7 @@ target :lib do
   library "logger"
   library "pathname"
   library "forwardable"
+  library "net/http"
 
   configure_code_diagnostics(D::Ruby.default)
 end

lib/ruby_wasm/build.rb

@@ -43,7 +43,7 @@ def initialize(
     @target = target
     @build_dir = build_dir
     @rubies_dir = rubies_dir
-    @toolchain = (toolchain || RubyWasm::Toolchain.get(target, @build_dir))
+    @toolchain = toolchain || raise("toolchain is required")
 
     @libyaml = RubyWasm::LibYAMLProduct.new(@build_dir, @target, @toolchain)
     @zlib = RubyWasm::ZlibProduct.new(@build_dir, @target, @toolchain)

lib/ruby_wasm/build/executor.rb

@@ -147,6 +147,19 @@ def _print_command(args, env)
     end
   end
 
+  class SilentExecutor
+    def system(*args, chdir: nil, env: nil)
+      kwargs = {}
+      kwargs[:chdir] = chdir if chdir
+      kwargs[:exception] = true
+      if env
+        Kernel.system(env, *args, **kwargs)
+      else
+        Kernel.system(*args, **kwargs)
+      end
+    end
+  end
+
   # Human readable status printer for the build.
   class StatusPrinter
     def initialize

lib/ruby_wasm/build/product/crossruby.rb

@@ -204,7 +204,7 @@ def build_exts(executor)
     def build(executor, remake: false, reconfigure: false)
       executor.mkdir_p dest_dir
       executor.mkdir_p build_dir
-      @toolchain.install
+      @toolchain.install(executor)
       [@source, @baseruby, @libyaml, @zlib, @openssl, @wasi_vfs].each do |prod|
         next unless prod
         executor.begin_section prod.class, prod.name, "Building"

lib/ruby_wasm/build/toolchain.rb

@@ -16,10 +16,10 @@ def check_envvar(name)
       raise "missing environment variable: #{name}" if ENV[name].nil?
     end
 
-    def self.get(target, build_dir = nil)
+    def self.get(target, options, build_dir = nil)
       case target
       when /^wasm32-unknown-wasi/
-        return RubyWasm::WASISDK.new(build_dir: build_dir)
+        return RubyWasm::WASISDK.new(build_dir: build_dir, version: options[:wasi_sdk_version])
       when "wasm32-unknown-emscripten"
         return RubyWasm::Emscripten.new
       else
@@ -56,18 +56,19 @@ class WASISDK < Toolchain
     def initialize(
       wasi_sdk_path = ENV["WASI_SDK_PATH"],
       build_dir: nil,
-      version_major: 22,
-      version_minor: 0,
+      version: "23.0",
       binaryen_version: 108
     )
       @need_fetch_wasi_sdk = wasi_sdk_path.nil?
       if @need_fetch_wasi_sdk
         if build_dir.nil?
           raise "build_dir is required when WASI_SDK_PATH is not set"
         end
-        wasi_sdk_path = File.join(build_dir, "toolchain", "wasi-sdk")
-        @version_major = version_major
-        @version_minor = version_minor
+        wasi_sdk_path = File.join(build_dir, "toolchain", "wasi-sdk-#{version}")
+        if version.nil?
+          raise "version is required when WASI_SDK_PATH is not set"
+        end
+        @version = version
       end
 
       @binaryen = Binaryen.new(build_dir: build_dir, binaryen_version: binaryen_version)
@@ -98,36 +99,69 @@ def wasi_sdk_path
       @wasi_sdk_path
     end
 
-    def download_url(version_major, version_minor)
-      version = "#{version_major}.#{version_minor}"
+    def download_url
+      major, _ = @version.split(".").map(&:to_i)
+      # @type var assets: Array[[Regexp, Array[String]]]
       assets = [
-        [/x86_64-linux/, "wasi-sdk-#{version}-linux.tar.gz"],
-        [/(arm64e?|x86_64)-darwin/, "wasi-sdk-#{version}-macos.tar.gz"]
+        [/x86_64-linux/, [
+          "wasi-sdk-#{@version}-x86_64-linux.tar.gz",
+          # For wasi-sdk version < 23.0
+          "wasi-sdk-#{@version}-linux.tar.gz",
+        ]],
+        [/arm64e?-darwin/, [
+          "wasi-sdk-#{@version}-arm64-macos.tar.gz",
+          # For wasi-sdk version < 23.0
+          "wasi-sdk-#{@version}-macos.tar.gz",
+        ]],
+        [/x86_64-darwin/, [
+          "wasi-sdk-#{@version}-x86_64-macos.tar.gz",
+          # For wasi-sdk version < 23.0
+          "wasi-sdk-#{@version}-macos.tar.gz",
+        ]],
       ]
-      asset = assets.find { |os, _| os =~ RUBY_PLATFORM }&.at(1)
+      asset = assets.find do |os, candidates|
+        os =~ RUBY_PLATFORM
+      end
       if asset.nil?
         raise "unsupported platform for fetching WASI SDK: #{RUBY_PLATFORM}"
       end
-      "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-#{version_major}/#{asset}"
+      _, candidates = asset
+      candidates_urls = candidates.map do |candidate|
+        "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-#{major}/#{candidate}"
+      end
+      require "net/http"
+      # Find an asset that exists by checking HEAD response to see if the asset exists
+      candidates_urls.each do |url_str|
+        # @type var url: URI::HTTPS
+        url = URI.parse(url_str)
+        ok = Net::HTTP.start(url.host, url.port, use_ssl: url.scheme == 'https') do |http|
+          response = http.head(url.request_uri)
+          next response.code == "302"
+        end
+        if ok
+          return url_str
+        end
+      end
+      raise "WASI SDK asset not found: #{candidates_urls.join(", ")}"
     end
 
-    def install_wasi_sdk
+    def install_wasi_sdk(executor)
       return unless @need_fetch_wasi_sdk
       wasi_sdk_tarball =
-        File.join(File.dirname(@wasi_sdk_path), "wasi-sdk.tar.gz")
+        File.join(File.dirname(@wasi_sdk_path), "wasi-sdk-#{@version}.tar.gz")
       unless File.exist? wasi_sdk_tarball
         FileUtils.mkdir_p File.dirname(wasi_sdk_tarball)
-        system "curl -L -o #{wasi_sdk_tarball} #{self.download_url(@version_major, @version_minor)}"
+        executor.system "curl", "-fsSL", "-o", wasi_sdk_tarball, self.download_url
       end
       unless File.exist? @wasi_sdk_path
         FileUtils.mkdir_p @wasi_sdk_path
-        system "tar -C #{@wasi_sdk_path} --strip-component 1 -xzf #{wasi_sdk_tarball}"
+        executor.system "tar", "-C", @wasi_sdk_path, "--strip-component", "1", "-xzf", wasi_sdk_tarball
       end
     end
 
-    def install
-      install_wasi_sdk
-      @binaryen.install
+    def install(executor)
+      install_wasi_sdk(executor)
+      @binaryen.install(executor)
     end
   end
 
@@ -179,17 +213,17 @@ def download_url(version)
       "https://github.com/WebAssembly/binaryen/releases/download/version_#{@binaryen_version}/#{asset}"
     end
 
-    def install
+    def install(executor)
       return unless @need_fetch_binaryen
       binaryen_tarball = File.expand_path("../binaryen.tar.gz", @binaryen_path)
       unless File.exist? binaryen_tarball
         FileUtils.mkdir_p File.dirname(binaryen_tarball)
-        system "curl -L -o #{binaryen_tarball} #{self.download_url(@binaryen_version)}"
+        executor.system "curl", "-L", "-o", binaryen_tarball, self.download_url(@binaryen_version)
       end
 
       unless File.exist? @binaryen_path
         FileUtils.mkdir_p @binaryen_path
-        system "tar -C #{@binaryen_path} --strip-component 1 -xzf #{binaryen_tarball}"
+        executor.system "tar", "-C", @binaryen_path, "--strip-component", "1", "-xzf", binaryen_tarball
       end
     end
 
@@ -201,7 +235,7 @@ def initialize
       @name = "emscripten"
     end
 
-    def install
+    def install(executor)
     end
 
     def find_tool(name)

lib/ruby_wasm/cli.rb

@@ -181,12 +181,12 @@ def pack(args)
     private
 
     def build_config(options)
-      build_source, all_default_exts = compute_build_source(options)
       # @type var config: Packager::build_config
-      config = { target: options[:target_triplet], src: build_source }
+      config = compute_build_alias(options)
+      config[:target] = options[:target_triplet]
       case options[:profile]
       when "full"
-        config[:default_exts] = all_default_exts || ""
+        config[:default_exts] = config[:all_default_exts] || ""
         env_additional_exts = ENV["RUBY_WASM_ADDITIONAL_EXTS"] || ""
         unless env_additional_exts.empty?
           config[:default_exts] += "," + env_additional_exts
@@ -201,80 +201,103 @@ def build_config(options)
       config
     end
 
-    def compute_build_source(options)
+    def compute_build_alias(options)
       src_name = options[:ruby_version]
-      aliases = self.class.build_source_aliases(root)
-      source = aliases[src_name]
-      if source.nil?
+      aliases = self.class.build_config_aliases(root)
+      config = aliases[src_name]
+      if config.nil?
         if File.directory?(src_name)
           # Treat as a local source if the given name is a source directory.
           RubyWasm.logger.debug "Using local source: #{src_name}"
           if options[:patches].any?
             RubyWasm.logger.warn "Patches specified through --patch are ignored for local sources"
           end
-          # @type var local_source: RubyWasm::Packager::build_source_local
-          local_source = { type: "local", path: src_name }
-          # @type var local_source: RubyWasm::Packager::build_source
-          local_source = local_source.merge(name: "local", patches: [])
           # FIXME: We should have a way to specify extensions to be included by users.
           # For now, assume all default extensions available in the head revision are available.
-          return [local_source, RubyWasm::Packager::ALL_DEFAULT_EXTS]
+          return {
+            name: "local",
+            src: {
+              type: "local",
+              path: src_name,
+              patches: []
+            },
+            all_default_exts: RubyWasm::Packager::ALL_DEFAULT_EXTS,
+          }
         end
         # Otherwise, it's an unknown source.
         raise(
           "Unknown Ruby source: #{src_name} (available: #{aliases.keys.join(", ")} or a local directory)"
         )
       end
       # Apply user-specified patches in addition to bundled patches.
-      source[:patches].concat(options[:patches])
-      # @type var all_default_exts: String
-      __skip__ = all_default_exts = source[:all_default_exts]
-      [source, all_default_exts]
+      config[:src][:patches].concat(options[:patches])
+      config
     end
 
     # Retrieves the alias definitions for the Ruby sources.
-    def self.build_source_aliases(root)
-      # @type var sources: Hash[string, RubyWasm::Packager::build_source]
-      sources = {
-        "head" => {
-          type: "github",
-          repo: "ruby/ruby",
-          rev: "master",
+    def self.build_config_aliases(root)
+      # @type var aliases: Array[RubyWasm::Packager::build_source]
+      aliases = [
+        {
+          name: "head",
+          src: {
+            type: "github",
+            repo: "ruby/ruby",
+            rev: "master",
+          },
           all_default_exts: RubyWasm::Packager::ALL_DEFAULT_EXTS,
+          wasi_sdk_version: "23.0",
         },
-        "3.4" => {
-          type: "tarball",
-          url: "https://cache.ruby-lang.org/pub/ruby/3.4/ruby-3.4.1.tar.gz",
+        {
+          name: "3.4",
+          src: {
+            type: "tarball",
+            url: "https://cache.ruby-lang.org/pub/ruby/3.4/ruby-3.4.1.tar.gz",
+          },
           all_default_exts: "cgi/escape,continuation,coverage,date,digest/bubblebabble,digest,digest/md5,digest/rmd160,digest/sha1,digest/sha2,etc,fcntl,json,json/generator,json/parser,objspace,pathname,psych,rbconfig/sizeof,ripper,stringio,strscan,monitor,zlib,openssl",
+          wasi_sdk_version: "22.0",
         },
-        "3.3" => {
-          type: "tarball",
-          url: "https://cache.ruby-lang.org/pub/ruby/3.3/ruby-3.3.3.tar.gz",
+        {
+          name: "3.3",
+          src: {
+            type: "tarball",
+            url: "https://cache.ruby-lang.org/pub/ruby/3.3/ruby-3.3.3.tar.gz",
+          },
           all_default_exts: "bigdecimal,cgi/escape,continuation,coverage,date,dbm,digest/bubblebabble,digest,digest/md5,digest/rmd160,digest/sha1,digest/sha2,etc,fcntl,fiber,gdbm,json,json/generator,json/parser,nkf,objspace,pathname,psych,racc/cparse,rbconfig/sizeof,ripper,stringio,strscan,monitor,zlib,openssl",
+          wasi_sdk_version: "22.0",
         },
-        "3.2" => {
-          type: "tarball",
-          url: "https://cache.ruby-lang.org/pub/ruby/3.2/ruby-3.2.4.tar.gz",
+        {
+          name: "3.2",
+          src: {
+            type: "tarball",
+            url: "https://cache.ruby-lang.org/pub/ruby/3.2/ruby-3.2.4.tar.gz",
+          },
           all_default_exts: "bigdecimal,cgi/escape,continuation,coverage,date,dbm,digest/bubblebabble,digest,digest/md5,digest/rmd160,digest/sha1,digest/sha2,etc,fcntl,fiber,gdbm,json,json/generator,json/parser,nkf,objspace,pathname,psych,racc/cparse,rbconfig/sizeof,ripper,stringio,strscan,monitor,zlib,openssl",
+          wasi_sdk_version: "22.0",
         }
-      }
+      ]
+
+      # Set the name in the source config.
+      aliases.each do |config|
+        config[:src][:name] = config[:name]
+      end
 
       # Apply bundled and user-specified `<root>/patches` directories.
-      sources.each do |name, source|
-        source[:name] = name
+      aliases.each do |config|
         patches_dirs = [bundled_patches_path, File.join(root, "patches")]
-        source[:patches] = patches_dirs.flat_map do |patches_dir|
-          Dir[File.join(patches_dir, name, "*.patch")]
+        config[:src][:patches] = patches_dirs.flat_map do |patches_dir|
+          Dir[File.join(patches_dir, config[:name], "*.patch")]
             .map { |p| File.expand_path(p) }
         end.uniq
       end
 
+      # Pin the revisions based on build_manifest.json if available.
       build_manifest = File.join(root, "build_manifest.json")
       if File.exist?(build_manifest)
         begin
           manifest = JSON.parse(File.read(build_manifest))
           manifest["ruby_revisions"].each do |name, rev|
-            source = sources[name]
+            source = aliases[name][:src]
             next unless source[:type] == "github"
             # @type var source: RubyWasm::Packager::build_source_github
             source[:rev] = rev
@@ -283,7 +306,7 @@ def self.build_source_aliases(root)
           RubyWasm.logger.warn "Failed to load build_manifest.json: #{e}"
         end
       end
-      sources
+      aliases.to_h { |config| [config[:name], config] }
     end
 
     # Retrieves the root directory of the Ruby project.

lib/ruby_wasm/packager.rb

@@ -111,7 +111,7 @@ def full_build_options
     options = build_options
     build_dir = File.join(@root, "build")
     rubies_dir = File.join(@root, "rubies")
-    toolchain = RubyWasm::Toolchain.get(options[:target], build_dir)
+    toolchain = RubyWasm::Toolchain.get(options[:target], options, build_dir)
     options.merge(
       toolchain: toolchain,
       build_dir: build_dir,

lib/ruby_wasm/packager/core.rb

@@ -179,7 +179,7 @@ def _link_gem_exts(executor, build, ruby_root, gem_home, module_bytes)
     end
 
     def _build_gem_exts(executor, build, gem_home)
-      build.toolchain.install
+      build.toolchain.install(executor)
       baseruby = build.baseruby
       unless Dir.exist?(baseruby.install_dir)
         baseruby.build(executor)