diff --git a/.github/actions/capiext/action.yml b/.github/actions/capiext/action.yml index e8ea87e61019d7..c17069f97dae20 100644 --- a/.github/actions/capiext/action.yml +++ b/.github/actions/capiext/action.yml @@ -26,25 +26,51 @@ runs: run: | eval $(grep -e '^arch *=' -e '^ruby_version *=' -e '^DLEXT *=' Makefile | sed 's/ *= */=/') - key=capiexts-${arch}-${ruby_version} + case "${ruby_version}" in + *+*) key=capiexts-${arch}-${ruby_version};; + *) key=;; + esac + echo version=$ruby_version >> $GITHUB_OUTPUT echo key=$key >> $GITHUB_OUTPUT echo DLEXT=$DLEXT >> $GITHUB_OUTPUT working-directory: ${{ inputs.builddir }} - - name: CAPI extensions cache - uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 + - name: Restore previous CAPI extensions + uses: actions/cache/restore@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 id: cache with: path: ${{ inputs.builddir }}/spec/ruby/optional/capi/ext/ key: ${{ steps.config.outputs.key }} + if: ${{ steps.config.outputs.key }} - name: Run test-spec with previous CAPI extension binaries + id: check shell: bash run: | touch spec/ruby/optional/capi/ext/*.$DLEXT [ ! -f spec/ruby/optional/capi/ext/\*.$DLEXT ] ${{ inputs.make }} SPECOPTS=optional/capi test-spec + rm -f spec/ruby/optional/capi/ext/*.c env: DLEXT: ${{ steps.config.outputs.DLEXT }} working-directory: ${{ inputs.builddir }} if: ${{ steps.cache.outputs.cache-hit }} + + - name: Save CAPI extensions + uses: actions/cache/save@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 + with: + path: ${{ inputs.builddir }}/spec/ruby/optional/capi/ext/ + key: ${{ steps.config.outputs.key }} + if: >- + ${{true + && steps.cache.outcome == 'success' + && ! steps.cache.outputs.cache-hit + && github.ref_name == 'master' + }} + + - shell: bash + run: | + echo "::error::Change from ${prev} detected; bump up ABI version" + env: + prev: ${{ steps.config.outputs.version }} + if: ${{ always() && steps.check.outcome == 'failure' }} diff --git a/.github/workflows/annocheck.yml b/.github/workflows/annocheck.yml deleted file mode 100644 index 0aa21706160dbc..00000000000000 --- a/.github/workflows/annocheck.yml +++ /dev/null @@ -1,110 +0,0 @@ -name: Annocheck - -on: - push: - paths-ignore: - - 'doc/**' - - '**/man/*' - - '**.md' - - '**.rdoc' - - '**/.document' - - '.*.yml' - pull_request: - paths-ignore: - - 'doc/**' - - '**/man/*' - - '**.md' - - '**.rdoc' - - '**/.document' - - '.*.yml' - merge_group: - -concurrency: - group: ${{ github.workflow }} / ${{ startsWith(github.event_name, 'pull') && github.ref_name || github.sha }} - cancel-in-progress: ${{ startsWith(github.event_name, 'pull') }} - -permissions: - contents: read - -jobs: - compile: - name: test-annocheck - - runs-on: ubuntu-latest - - container: - image: ghcr.io/ruby/ruby-ci-image:gcc-11 - options: --user root - - if: >- - ${{!(false - || contains(github.event.head_commit.message, '[DOC]') - || contains(github.event.pull_request.title, '[DOC]') - || contains(github.event.pull_request.labels.*.name, 'Documentation') - || (github.event_name == 'push' && github.event.pull_request.user.login == 'dependabot[bot]') - )}} - - env: - CONFIGURE_TTY: never - GITPULLOPTIONS: --no-tags origin ${{ github.ref }} - RUBY_DEBUG: ci rgengc - RUBY_TESTOPTS: >- - -q - --color=always - --tty=no - # FIXME: Drop skipping options - # https://bugs.ruby-lang.org/issues/18061 - # https://sourceware.org/annobin/annobin.html/Test-pie.html - TEST_ANNOCHECK_OPTS: '--skip-pie --skip-gaps' - - steps: - - run: id - working-directory: - - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - sparse-checkout-cone-mode: false - sparse-checkout: /.github - - - uses: ./.github/actions/setup/directories - with: - srcdir: src - builddir: build - makeup: true - - - uses: ruby/setup-ruby@a9bfc2ecf3dd40734a9418f89a7e9d484c32b990 # v1.248.0 - with: - ruby-version: '3.1' - bundler: none - - # Minimal flags to pass the check. - # -g0 disables backtraces when SEGV. Do not set that. - - name: Run configure - run: > - ../src/configure -C - --enable-debug-env - --disable-install-doc - --with-ext=-test-/cxxanyargs,+ - --without-valgrind - --without-jemalloc - --without-gmp - --with-gcc="gcc-11 -fcf-protection -Wa,--generate-missing-build-notes=yes" - --enable-shared - debugflags=-ggdb3 - optflags=-O2 - LDFLAGS=-Wl,-z,now - - - run: make showflags - - - run: make - - - run: make test-annocheck - - - uses: ./.github/actions/slack - with: - SLACK_WEBHOOK_URL: ${{ secrets.SIMPLER_ALERTS_URL }} # ruby-lang slack: ruby/simpler-alerts-bot - if: ${{ failure() }} - -defaults: - run: - working-directory: build diff --git a/.github/workflows/auto_request_review.yml b/.github/workflows/auto_request_review.yml deleted file mode 100644 index 207315a084cc59..00000000000000 --- a/.github/workflows/auto_request_review.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Auto Request Review -on: - pull_request_target: - types: [opened, ready_for_review, reopened] - branches: [master] - -permissions: - contents: read - -jobs: - auto-request-review: - name: Auto Request Review - runs-on: ubuntu-latest - if: ${{ github.repository == 'ruby/ruby' && github.base_ref == 'master' }} - steps: - - name: Request review based on files changes and/or groups the author belongs to - uses: necojackarc/auto-request-review@e89da1a8cd7c8c16d9de9c6e763290b6b0e3d424 # v0.13.0 - with: - # scope: public_repo - token: ${{ secrets.MATZBOT_AUTO_REQUEST_REVIEW_TOKEN }} diff --git a/.github/workflows/baseruby.yml b/.github/workflows/baseruby.yml deleted file mode 100644 index 4537157b830885..00000000000000 --- a/.github/workflows/baseruby.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: BASERUBY Check - -on: - push: - paths-ignore: - - 'doc/**' - - '**/man/*' - - '**.md' - - '**.rdoc' - - '**/.document' - - '.*.yml' - pull_request: - paths-ignore: - - 'doc/**' - - '**/man/*' - - '**.md' - - '**.rdoc' - - '**/.document' - - '.*.yml' - merge_group: - -concurrency: - group: ${{ github.workflow }} / ${{ startsWith(github.event_name, 'pull') && github.ref_name || github.sha }} - cancel-in-progress: ${{ startsWith(github.event_name, 'pull') }} - -permissions: - contents: read - -jobs: - baseruby: - name: BASERUBY - - runs-on: ubuntu-22.04 - - if: >- - ${{!(false - || contains(github.event.head_commit.message, '[DOC]') - || contains(github.event.pull_request.title, '[DOC]') - || contains(github.event.pull_request.labels.*.name, 'Documentation') - || (github.event_name == 'push' && github.event.pull_request.user.login == 'dependabot[bot]') - )}} - - strategy: - matrix: - ruby: - - ruby-3.1 - - ruby-3.2 - - ruby-3.3 - - steps: - - uses: ruby/setup-ruby@a9bfc2ecf3dd40734a9418f89a7e9d484c32b990 # v1.248.0 - with: - ruby-version: ${{ matrix.ruby }} - bundler: none - - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - uses: ./.github/actions/setup/ubuntu - - - uses: ./.github/actions/setup/directories - with: - makeup: true - - - run: ./configure --disable-install-doc - - - run: make all - - - run: make test - - - uses: ./.github/actions/slack - with: - label: ${{ matrix.ruby }} - SLACK_WEBHOOK_URL: ${{ secrets.SIMPLER_ALERTS_URL }} # ruby-lang slack: ruby/simpler-alerts-bot - if: ${{ failure() }} diff --git a/.github/workflows/bundled_gems.yml b/.github/workflows/bundled_gems.yml deleted file mode 100644 index cb13df330bfb93..00000000000000 --- a/.github/workflows/bundled_gems.yml +++ /dev/null @@ -1,184 +0,0 @@ -name: bundled_gems - -on: - push: - branches: ['master'] - paths: - - '.github/workflows/bundled_gems.yml' - - 'gems/bundled_gems' - pull_request: - branches: ['master'] - paths: - - '.github/workflows/bundled_gems.yml' - - 'gems/bundled_gems' - merge_group: - schedule: - - cron: '45 6 * * *' - workflow_dispatch: - -permissions: # added using https://github.com/step-security/secure-workflows - contents: read - -jobs: - update: - permissions: - contents: write # for Git to git push - - if: ${{ github.event_name != 'schedule' || github.repository == 'ruby/ruby' }} - - name: update ${{ github.workflow }} - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - token: ${{ (github.repository == 'ruby/ruby' && !startsWith(github.event_name, 'pull')) && secrets.MATZBOT_AUTO_UPDATE_TOKEN || secrets.GITHUB_TOKEN }} - - - uses: ./.github/actions/setup/directories - with: - # Skip overwriting MATZBOT_AUTO_UPDATE_TOKEN - checkout: '' # false (ref: https://github.com/actions/runner/issues/2238) - - - name: Set ENV - run: | - echo "TODAY=$(date +%F)" >> $GITHUB_ENV - - - name: Download previous gems list - run: | - mkdir -p .downloaded-cache - for data in bundled_gems.json default_gems.json; do - ln -s .downloaded-cache/$data . - curl -O -R -z ./$data https://stdgems.org/$data - done - - - name: Update bundled gems list - id: bundled_gems - run: | - ruby -i~ tool/update-bundled_gems.rb gems/bundled_gems >> $GITHUB_OUTPUT - - - name: Update spec/bundler/support/builders.rb - run: | - #!ruby - rake_version = File.read("gems/bundled_gems")[/^rake\s+(\S+)/, 1] - print ARGF.read.sub(/^ *def rake_version\s*\K".*?"/) {rake_version.dump} - shell: ruby -i~ {0} spec/bundler/support/builders.rb - - - name: Maintain updated gems list in NEWS - run: | - ruby tool/update-NEWS-gemlist.rb bundled - - - name: Check diffs - id: diff - run: | - news= gems= - git diff --color --no-ext-diff --ignore-submodules --exit-code -- NEWS.md || - news=true - git diff --color --no-ext-diff --ignore-submodules --exit-code -- gems/bundled_gems || - gems=true - git add -- NEWS.md gems/bundled_gems - git add -- spec/bundler/support/builders.rb - echo news=$news >> $GITHUB_OUTPUT - echo gems=$gems >> $GITHUB_OUTPUT - echo update=${news:-$gems} >> $GITHUB_OUTPUT - - - name: Commit - id: commit - run: | - git pull --ff-only origin ${GITHUB_REF#refs/heads/} - message="Update bundled gems list" - if [ -z "${gems}" ]; then - git commit --message="[DOC] ${message} at ${GITHUB_SHA:0:30}" - else - git commit --message="${message} as of ${TODAY}" - fi - env: - TODAY: ${{ steps.bundled_gems.outputs.latest_date || env.TODAY }} - EMAIL: svn-admin@ruby-lang.org - GIT_AUTHOR_NAME: git - GIT_COMMITTER_NAME: git - gems: ${{ steps.diff.outputs.gems }} - if: ${{ steps.diff.outputs.update }} - - - name: Development revision of bundled gems - run: | - #!ruby - file = "gems/bundled_gems" - - SECONDS_IN_DAY = 86400 - today = Time.new("#{ENV['TODAY']}Z") - if !(december = today.month == 12) - days = 30 - elsif (days = 26 - today.day).positive? - days += 4 - else - puts "::info:: just after released" - exit - end - - since = "#{today.year-1}-12-26" - ref = ENV['GITHUB_REF'] - puts "::group::\e[94mfetching \e[1m#{file}\e[22m since \e[1m#{since}\e[22m from \e[1m#{ref}\e[m" - system(*%W[git fetch --shallow-since=#{since} --no-tags origin #{ref}], exception: true) - puts "::endgroup::" - - puts "\e[94mchecking development version bundled gems older than \e[1m#{days}\e[22m days\e[m" - limit = today.to_i - days * SECONDS_IN_DAY - old = 0 - IO.popen(%W"git blame --line-porcelain -- #{file}") do |blame| - while head = blame.gets("\n\t") and s = blame.gets - next unless (gem = s.split(/\s+|#.*/)).size > 3 - time = head[/^committer-time \K\d+/].to_i - next if (d = limit - time) <= 0 - d /= SECONDS_IN_DAY - line = head[/\A\h+ \d+ \K\d+/].to_i - level = if d < days; 'warning'; else old += 1; 'error'; end - d += days - puts "::#{level} file=#{file},line=#{line},title=Older than #{d} days::#{gem[0]} #{gem[3]}" - end - end - abort "::error title=Too long-standing gems::The release comes soon." if december and old.nonzero? - shell: ruby {0} - env: - file: ${{ steps.logs.outputs.file }} - days: ${{ steps.logs.outputs.days }} - - - name: Install libraries - uses: ./.github/actions/setup/ubuntu - if: ${{ steps.diff.outputs.gems }} - - - name: Build - run: | - ./autogen.sh - ./configure -C --disable-install-doc - make - if: ${{ steps.diff.outputs.gems }} - - - name: Prepare bundled gems - run: | - make -s prepare-gems - if: ${{ steps.diff.outputs.gems }} - - - name: Test bundled gems - run: | - make -s test-bundled-gems - timeout-minutes: 30 - env: - RUBY_TESTOPTS: '-q --tty=no' - TEST_BUNDLED_GEMS_ALLOW_FAILURES: '' - if: ${{ steps.diff.outputs.gems }} - - - name: Push - run: | - git push origin ${GITHUB_REF#refs/heads/} - if: >- - ${{ - github.repository == 'ruby/ruby' && - !startsWith(github.event_name, 'pull') && - steps.commit.outcome == 'success' - }} - - - uses: ./.github/actions/slack - with: - SLACK_WEBHOOK_URL: ${{ secrets.SIMPLER_ALERTS_URL }} # ruby-lang slack: ruby/simpler-alerts-bot - if: ${{ failure() }} diff --git a/.github/workflows/check_dependencies.yml b/.github/workflows/check_dependencies.yml deleted file mode 100644 index e8b37d616143b8..00000000000000 --- a/.github/workflows/check_dependencies.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Check Dependencies -on: - push: - paths-ignore: - - 'doc/**' - - '**/man/*' - - '**.md' - - '**.rdoc' - - '**/.document' - - '.*.yml' - pull_request: - merge_group: - -concurrency: - group: ${{ github.workflow }} / ${{ startsWith(github.event_name, 'pull') && github.ref_name || github.sha }} - cancel-in-progress: ${{ startsWith(github.event_name, 'pull') }} - -permissions: - contents: read - -jobs: - update-deps: - name: Dependency checks - - strategy: - matrix: - os: [ubuntu-latest] - fail-fast: true - - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - uses: ./.github/actions/setup/ubuntu - if: ${{ contains(matrix.os, 'ubuntu') }} - - - uses: ./.github/actions/setup/macos - if: ${{ contains(matrix.os, 'macos') }} - - - uses: ./.github/actions/setup/directories - - - uses: ruby/setup-ruby@a9bfc2ecf3dd40734a9418f89a7e9d484c32b990 # v1.248.0 - with: - ruby-version: '3.1' - bundler: none - - - name: Run configure - run: ./configure -C --disable-install-doc --disable-rubygems --with-gcc 'optflags=-O0' 'debugflags=-save-temps=obj -g' - - - run: make fix-depends - - - run: git diff --no-ext-diff --ignore-submodules --exit-code - - - uses: ./.github/actions/slack - with: - label: ${{ matrix.os }} / Dependencies need to update - SLACK_WEBHOOK_URL: ${{ secrets.SIMPLER_ALERTS_URL }} # ruby-lang slack: ruby/simpler-alerts-bot - if: ${{ failure() }} diff --git a/.github/workflows/check_misc.yml b/.github/workflows/check_misc.yml deleted file mode 100644 index d4a2c6f3fef6eb..00000000000000 --- a/.github/workflows/check_misc.yml +++ /dev/null @@ -1,118 +0,0 @@ -name: Misc -on: [push, pull_request, merge_group] - -concurrency: - group: ${{ github.workflow }} / ${{ startsWith(github.event_name, 'pull') && github.ref_name || github.sha }} - cancel-in-progress: ${{ startsWith(github.event_name, 'pull') }} - -permissions: - contents: read - -jobs: - checks: - name: Miscellaneous checks - - permissions: - contents: write # for Git to git push - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - token: ${{ (github.repository == 'ruby/ruby' && !startsWith(github.event_name, 'pull')) && secrets.MATZBOT_AUTO_UPDATE_TOKEN || secrets.GITHUB_TOKEN }} - - - uses: ./.github/actions/setup/directories - with: - makeup: true - # Skip overwriting MATZBOT_AUTO_UPDATE_TOKEN - checkout: '' # false (ref: https://github.com/actions/runner/issues/2238) - - # Run this step first to make sure auto-style commits are pushed - - name: ${{ github.ref == 'refs/heads/master' && 'Auto-correct' || 'Check for' }} code styles - run: | - set -x - ruby tool/auto-style.rb "$GITHUB_OLD_SHA" "$GITHUB_NEW_SHA" "$PUSH_REF" - env: - EMAIL: svn-admin@ruby-lang.org - GIT_AUTHOR_NAME: git - GIT_COMMITTER_NAME: git - GITHUB_OLD_SHA: ${{ github.event.pull_request.base.sha }} - GITHUB_NEW_SHA: ${{ github.event.pull_request.merge_commit_sha }} - PUSH_REF: ${{ github.ref == 'refs/heads/master' && github.ref || '' }} - if: ${{ github.repository == 'ruby/ruby' && startsWith(github.event_name, 'pull') }} - - - name: Check if C-sources are US-ASCII - run: | - grep -r -n --include='*.[chyS]' --include='*.asm' $'[^\t-~]' -- . && exit 1 || : - - - name: Check for bash specific substitution in configure.ac - run: | - git grep -n '\${[A-Za-z_0-9]*/' -- configure.ac && exit 1 || : - - - name: Check for header macros - run: | - fail= - for header in ruby/*.h; do - git grep -l -F -e $header -e HAVE_`echo $header | tr a-z./ A-Z__` -- . > /dev/null && continue - fail=1 - echo $header - done - exit $fail - working-directory: include - - - name: Check if to generate documents - id: rdoc - run: | - ref=$(sed 's/#.*//;/^rdoc /!d' gems/bundled_gems | awk '{print $4}') - echo ref=$ref >> $GITHUB_OUTPUT - # Generate only when document commit/PR - if: >- - ${{false - || contains(github.event.head_commit.message, '[ruby/rdoc]') - || contains(github.event.head_commit.message, '[DOC]') - || contains(github.event.pull_request.title, '[DOC]') - || contains(github.event.pull_request.labels.*.name, 'Documentation') - }} - - - name: Checkout rdoc - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - repository: ruby/rdoc - ref: ${{ steps.rdoc.outputs.ref }} - path: .bundle/gems/rdoc-0 - if: ${{ steps.rdoc.outputs.ref != '' }} - - - name: Generate rdoc - run: | - set -x - gempath=$(ruby -e 'print Gem.user_dir, "/bin"') - PATH=$gempath:$PATH - gem install --user bundler - bundle config --local path vendor/bundle - bundle install --jobs 4 - bundle exec rake generate - working-directory: .bundle/gems/rdoc-0 - if: ${{ steps.rdoc.outputs.ref != '' }} - - - name: Generate docs - id: docs - run: | - $RDOC -C -x ^ext -x ^lib . - $RDOC --op html . - echo htmlout=ruby-html-${GITHUB_SHA:0:10} >> $GITHUB_OUTPUT - env: - RDOC: ruby -W0 --disable-gems tool/rdoc-srcdir -q - if: ${{ steps.rdoc.outcome == 'success' }} - - - name: Upload docs - uses: actions/upload-artifact@v4 - with: - path: html - name: ${{ steps.docs.outputs.htmlout }} - if: ${{ steps.docs.outcome == 'success' }} - - - uses: ./.github/actions/slack - with: - SLACK_WEBHOOK_URL: ${{ secrets.SIMPLER_ALERTS_URL }} # ruby-lang slack: ruby/simpler-alerts-bot - if: ${{ failure() }} diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index 0b4103913bcae5..00000000000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,121 +0,0 @@ -name: 'CodeQL' - -on: - push: - branches: ['master'] - paths-ignore: - - 'doc/**' - - '**/man/*' - - '**.md' - - '**.rdoc' - - '**/.document' - - '.*.yml' - pull_request: - paths-ignore: - - 'doc/**' - - '**/man/*' - - '**.md' - - '**.rdoc' - - '**/.document' - - '.*.yml' - schedule: - - cron: '0 12 * * *' - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }} / ${{ startsWith(github.event_name, 'pull') && github.ref_name || github.sha }} - cancel-in-progress: ${{ startsWith(github.event_name, 'pull') }} - -permissions: # added using https://github.com/step-security/secure-workflows - contents: read - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - permissions: - actions: read # for github/codeql-action/init to get workflow details - contents: read # for actions/checkout to fetch code - security-events: write # for github/codeql-action/autobuild to send a status report - # CodeQL fails to run pull requests from dependabot due to missing write access to upload results. - if: >- - ${{!(false - || contains(github.event.head_commit.message, '[DOC]') - || contains(github.event.pull_request.title, '[DOC]') - || contains(github.event.pull_request.labels.*.name, 'Documentation') - || (github.event_name == 'push' && github.event.pull_request.user.login == 'dependabot[bot]') - )}} - - env: - enable_install_doc: no - - strategy: - fail-fast: false - matrix: - include: - - language: cpp - - language: ruby - - steps: - - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - name: Install libraries - if: ${{ contains(matrix.os, 'macos') }} - uses: ./.github/actions/setup/macos - - - name: Install libraries - if : ${{ matrix.os == 'ubuntu-latest' }} - uses: ./.github/actions/setup/ubuntu - - - uses: ./.github/actions/setup/directories - - - name: Remove an obsolete rubygems vendored file - if: ${{ matrix.os == 'ubuntu-latest' }} - run: sudo rm /usr/lib/ruby/vendor_ruby/rubygems/defaults/operating_system.rb - - - name: Initialize CodeQL - uses: github/codeql-action/init@df409f7d9260372bd5f19e5b04e83cb3c43714ae # v3.27.9 - with: - languages: ${{ matrix.language }} - trap-caching: false - debug: true - - - name: Autobuild - uses: github/codeql-action/autobuild@df409f7d9260372bd5f19e5b04e83cb3c43714ae # v3.27.9 - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@df409f7d9260372bd5f19e5b04e83cb3c43714ae # v3.27.9 - with: - category: '/language:${{ matrix.language }}' - upload: False - output: sarif-results - - - name: filter-sarif - uses: advanced-security/filter-sarif@f3b8118a9349d88f7b1c0c488476411145b6270d # v1.0.1 - with: - patterns: | - +**/*.rb - -lib/uri/mailto.rb:rb/overly-large-range - -lib/uri/rfc3986_parser.rb:rb/overly-large-range - -lib/bundler/vendor/uri/lib/uri/mailto.rb:rb/overly-large-range - -lib/bundler/vendor/uri/lib/uri/rfc3986_parser.rb:rb/overly-large-range - -test/ruby/test_io.rb:rb/non-constant-kernel-open - -test/open-uri/test_open-uri.rb:rb/non-constant-kernel-open - -test/open-uri/test_ssl.rb:rb/non-constant-kernel-open - -spec/ruby/core/io/binread_spec.rb:rb/non-constant-kernel-open - -spec/ruby/core/io/readlines_spec.rb:rb/non-constant-kernel-open - -spec/ruby/core/io/foreach_spec.rb:rb/non-constant-kernel-open - -spec/ruby/core/io/write_spec.rb:rb/non-constant-kernel-open - -spec/ruby/core/io/read_spec.rb:rb/non-constant-kernel-open - -spec/ruby/core/kernel/open_spec.rb:rb/non-constant-kernel-open - input: sarif-results/${{ matrix.language }}.sarif - output: sarif-results/${{ matrix.language }}.sarif - if: ${{ matrix.language == 'ruby' }} - continue-on-error: true - - - name: Upload SARIF - uses: github/codeql-action/upload-sarif@df409f7d9260372bd5f19e5b04e83cb3c43714ae # v3.27.9 - with: - sarif_file: sarif-results/${{ matrix.language }}.sarif - continue-on-error: true diff --git a/.github/workflows/compilers.yml b/.github/workflows/compilers.yml deleted file mode 100644 index cf456bc5bb88e6..00000000000000 --- a/.github/workflows/compilers.yml +++ /dev/null @@ -1,337 +0,0 @@ -# Some tests depending on this name 'Compilations' via $GITHUB_WORKFLOW. Make sure to update such tests when renaming this workflow. -name: Compilations - -on: - push: - paths-ignore: - - 'doc/**' - - '**/man/*' - - '**.md' - - '**.rdoc' - - '**/.document' - - '.*.yml' - pull_request: - paths-ignore: - - 'doc/**' - - '**/man/*' - - '**.md' - - '**.rdoc' - - '**/.document' - - '.*.yml' - merge_group: - -concurrency: - group: ${{ github.workflow }} / ${{ startsWith(github.event_name, 'pull') && github.ref_name || github.sha }} - cancel-in-progress: ${{ startsWith(github.event_name, 'pull') }} - -permissions: - contents: read - -# Each job is split so that they roughly take 30min to run through. -jobs: - compile-if: - name: 'omnibus compilations, trigger' - runs-on: ubuntu-latest - if: >- - ${{!(false - || contains(github.event.head_commit.message, '[DOC]') - || contains(github.event.pull_request.title, '[DOC]') - || contains(github.event.pull_request.labels.*.name, 'Documentation') - || (github.event_name == 'push' && github.event.pull_request.user.login == 'dependabot[bot]') - )}} - steps: - - run: true - working-directory: - - compile1: - name: 'omnibus compilations, #1' - runs-on: ubuntu-latest - needs: compile-if - if: ${{ needs.compile-if.result == 'success' }} - timeout-minutes: 60 - services: { docuum: { image: 'stephanmisc/docuum', options: '--init', volumes: [ '/root', '/var/run/docker.sock:/var/run/docker.sock' ] } } - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: { sparse-checkout-cone-mode: false, sparse-checkout: /.github } - # Set fetch-depth: 10 so that Launchable can receive commits information. - - { uses: './.github/actions/setup/directories', with: { srcdir: 'src', builddir: 'build', makeup: true, fetch-depth: 10 } } - - name: 'clang 18 LTO' - uses: './.github/actions/compilers' - with: - tag: clang-18 - with_gcc: 'clang-18 -flto=auto' - optflags: '-O2' - enable_shared: false - - { uses: './.github/actions/compilers', name: '-O0', with: { optflags: '-O0 -march=x86-64 -mtune=generic' } } - # - { uses: './.github/actions/compilers', name: '-O3', with: { optflags: '-O3 -march=x86-64 -mtune=generic', check: true } } - - compile2: - name: 'omnibus compilations, #2' - runs-on: ubuntu-latest - needs: compile-if - if: ${{ needs.compile-if.result == 'success' }} - timeout-minutes: 60 - services: { docuum: { image: 'stephanmisc/docuum', options: '--init', volumes: [ '/root', '/var/run/docker.sock:/var/run/docker.sock' ] } } - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: { sparse-checkout-cone-mode: false, sparse-checkout: /.github } - - { uses: './.github/actions/setup/directories', with: { srcdir: 'src', builddir: 'build', makeup: true, fetch-depth: 10 } } - - name: 'GCC 15 LTO' - uses: './.github/actions/compilers' - with: - tag: gcc-15 - with_gcc: 'gcc-15 -flto=auto -ffat-lto-objects -Werror=lto-type-mismatch' - optflags: '-O2' - enable_shared: false - - { uses: './.github/actions/compilers', name: 'ext/Setup', with: { static_exts: 'etc json/* */escape' } } - - { uses: './.github/actions/compilers', name: 'GCC 15', with: { tag: 'gcc-15' } } - - { uses: './.github/actions/compilers', name: 'GCC 14', with: { tag: 'gcc-14' } } - - { uses: './.github/actions/compilers', name: 'GCC 13', with: { tag: 'gcc-13' } } - - { uses: './.github/actions/compilers', name: 'GCC 12', with: { tag: 'gcc-12' } } - - { uses: './.github/actions/compilers', name: 'GCC 11', with: { tag: 'gcc-11' } } - - { uses: './.github/actions/compilers', name: 'GCC 10', with: { tag: 'gcc-10' } } - - { uses: './.github/actions/compilers', name: 'GCC 9', with: { tag: 'gcc-9' } } - - { uses: './.github/actions/compilers', name: 'GCC 8', with: { tag: 'gcc-8' } } - - { uses: './.github/actions/compilers', name: 'GCC 7', with: { tag: 'gcc-7' } } - - compile3: - name: 'omnibus compilations, #3' - runs-on: ubuntu-latest - needs: compile-if - if: ${{ needs.compile-if.result == 'success' }} - timeout-minutes: 60 - services: { docuum: { image: 'stephanmisc/docuum', options: '--init', volumes: [ '/root', '/var/run/docker.sock:/var/run/docker.sock' ] } } - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: { sparse-checkout-cone-mode: false, sparse-checkout: /.github } - - { uses: './.github/actions/setup/directories', with: { srcdir: 'src', builddir: 'build', makeup: true, fetch-depth: 10 } } - - { uses: './.github/actions/compilers', name: 'clang 22', with: { tag: 'clang-22' } } - - { uses: './.github/actions/compilers', name: 'clang 21', with: { tag: 'clang-21' } } - - { uses: './.github/actions/compilers', name: 'clang 20', with: { tag: 'clang-20' } } - - { uses: './.github/actions/compilers', name: 'clang 19', with: { tag: 'clang-19' } } - - { uses: './.github/actions/compilers', name: 'clang 18', with: { tag: 'clang-18' } } - - { uses: './.github/actions/compilers', name: 'clang 17', with: { tag: 'clang-17' } } - - { uses: './.github/actions/compilers', name: 'clang 16', with: { tag: 'clang-16' } } - - { uses: './.github/actions/compilers', name: 'clang 15', with: { tag: 'clang-15' } } - - { uses: './.github/actions/compilers', name: 'clang 14', with: { tag: 'clang-14' } } - - compile4: - name: 'omnibus compilations, #4' - runs-on: ubuntu-latest - needs: compile-if - if: ${{ needs.compile-if.result == 'success' }} - timeout-minutes: 60 - services: { docuum: { image: 'stephanmisc/docuum', options: '--init', volumes: [ '/root', '/var/run/docker.sock:/var/run/docker.sock' ] } } - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: { sparse-checkout-cone-mode: false, sparse-checkout: /.github } - - { uses: './.github/actions/setup/directories', with: { srcdir: 'src', builddir: 'build', makeup: true, fetch-depth: 10 } } - - { uses: './.github/actions/compilers', name: 'clang 13', with: { tag: 'clang-13' } } - - { uses: './.github/actions/compilers', name: 'clang 12', with: { tag: 'clang-12' } } - - { uses: './.github/actions/compilers', name: 'clang 11', with: { tag: 'clang-11' } } - - { uses: './.github/actions/compilers', name: 'clang 10', with: { tag: 'clang-10' } } - # llvm-objcopy<=9 doesn't have --wildcard. It compiles, but leaves Rust symbols in libyjit.o. - - { uses: './.github/actions/compilers', name: 'clang 9', with: { tag: 'clang-9', append_configure: '--disable-yjit' } } - - { uses: './.github/actions/compilers', name: 'clang 8', with: { tag: 'clang-8', append_configure: '--disable-yjit' } } - - { uses: './.github/actions/compilers', name: 'clang 7', with: { tag: 'clang-7', append_configure: '--disable-yjit' } } - - { uses: './.github/actions/compilers', name: 'clang 6', with: { tag: 'clang-6.0', append_configure: '--disable-yjit' } } - - compile5: - name: 'omnibus compilations, #5' - runs-on: ubuntu-latest - needs: compile-if - if: ${{ needs.compile-if.result == 'success' }} - timeout-minutes: 60 - services: { docuum: { image: 'stephanmisc/docuum', options: '--init', volumes: [ '/root', '/var/run/docker.sock:/var/run/docker.sock' ] } } - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: { sparse-checkout-cone-mode: false, sparse-checkout: /.github } - - { uses: './.github/actions/setup/directories', with: { srcdir: 'src', builddir: 'build', makeup: true, fetch-depth: 10 } } - # -Wno-strict-prototypes is necessary with current clang-15 since - # older autoconf generate functions without prototype and -pedantic - # now implies strict-prototypes. Disabling the error but leaving the - # warning generates a lot of noise from use of ANYARGS in - # rb_define_method() and friends. - # See: https://github.com/llvm/llvm-project/commit/11da1b53d8cd3507959022cd790d5a7ad4573d94 - - { uses: './.github/actions/compilers', name: 'C99', with: { CFLAGS: '-std=c99 -Werror=pedantic -pedantic-errors -Wno-strict-prototypes' } } - - { uses: './.github/actions/compilers', name: 'C11', with: { CFLAGS: '-std=c11 -Werror=pedantic -pedantic-errors -Wno-strict-prototypes' } } - - { uses: './.github/actions/compilers', name: 'C17', with: { CFLAGS: '-std=c17 -Werror=pedantic -pedantic-errors -Wno-strict-prototypes' } } - - { uses: './.github/actions/compilers', name: 'C23', with: { CFLAGS: '-std=c2x -Werror=pedantic -pedantic-errors -Wno-strict-prototypes' } } - - { uses: './.github/actions/compilers', name: 'C++98', with: { CXXFLAGS: '-std=c++98 -Werror=pedantic -pedantic-errors -Wno-c++11-long-long' } } - - { uses: './.github/actions/compilers', name: 'C++11', with: { CXXFLAGS: '-std=c++11 -Werror=pedantic -pedantic-errors -Wno-c++11-long-long' } } - - { uses: './.github/actions/compilers', name: 'C++14', with: { CXXFLAGS: '-std=c++14 -Werror=pedantic -pedantic-errors -Wno-c++11-long-long' } } - - { uses: './.github/actions/compilers', name: 'C++17', with: { CXXFLAGS: '-std=c++17 -Werror=pedantic -pedantic-errors -Wno-c++11-long-long' } } - - compile6: - name: 'omnibus compilations, #6' - runs-on: ubuntu-latest - needs: compile-if - if: ${{ needs.compile-if.result == 'success' }} - timeout-minutes: 60 - services: { docuum: { image: 'stephanmisc/docuum', options: '--init', volumes: [ '/root', '/var/run/docker.sock:/var/run/docker.sock' ] } } - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: { sparse-checkout-cone-mode: false, sparse-checkout: /.github } - - { uses: './.github/actions/setup/directories', with: { srcdir: 'src', builddir: 'build', makeup: true, fetch-depth: 10 } } - - { uses: './.github/actions/compilers', name: 'C++20', with: { CXXFLAGS: '-std=c++20 -Werror=pedantic -pedantic-errors -Wno-c++11-long-long' } } - - { uses: './.github/actions/compilers', name: 'C++23', with: { CXXFLAGS: '-std=c++23 -Werror=pedantic -pedantic-errors -Wno-c++11-long-long' } } - - { uses: './.github/actions/compilers', name: 'C++26', with: { CXXFLAGS: '-std=c++26 -Werror=pedantic -pedantic-errors -Wno-c++11-long-long' } } - - { uses: './.github/actions/compilers', name: 'gmp', with: { append_configure: '--with-gmp', check: 'ruby/test_bignum.rb', mspecopt: "/github/workspace/src/spec/ruby/core/integer" } } - - { uses: './.github/actions/compilers', name: 'jemalloc', with: { append_configure: '--with-jemalloc' } } - - { uses: './.github/actions/compilers', name: 'valgrind', with: { append_configure: '--with-valgrind' } } - - { uses: './.github/actions/compilers', name: 'coroutine=ucontext', with: { append_configure: '--with-coroutine=ucontext' } } - - { uses: './.github/actions/compilers', name: 'coroutine=pthread', with: { append_configure: '--with-coroutine=pthread' } } - - compile7: - name: 'omnibus compilations, #7' - runs-on: ubuntu-latest - needs: compile-if - if: ${{ needs.compile-if.result == 'success' }} - timeout-minutes: 60 - services: { docuum: { image: 'stephanmisc/docuum', options: '--init', volumes: [ '/root', '/var/run/docker.sock:/var/run/docker.sock' ] } } - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: { sparse-checkout-cone-mode: false, sparse-checkout: /.github } - - { uses: './.github/actions/setup/directories', with: { srcdir: 'src', builddir: 'build', makeup: true, fetch-depth: 10 } } - - { uses: './.github/actions/compilers', name: 'disable-jit', with: { append_configure: '--disable-yjit --disable-zjit' } } - - { uses: './.github/actions/compilers', name: 'disable-yjit', with: { append_configure: '--disable-yjit' } } - - { uses: './.github/actions/compilers', name: 'disable-zjit', with: { append_configure: '--disable-zjit' } } - - { uses: './.github/actions/compilers', name: 'disable-dln', with: { append_configure: '--disable-dln' } } - - { uses: './.github/actions/compilers', name: 'enable-mkmf-verbose', with: { append_configure: '--enable-mkmf-verbose' } } - - { uses: './.github/actions/compilers', name: 'disable-rubygems', with: { append_configure: '--disable-rubygems' } } - - { uses: './.github/actions/compilers', name: 'RUBY_DEVEL', with: { append_configure: '--enable-devel' } } - - { uses: './.github/actions/compilers', name: 'OPT_THREADED_CODE=0', with: { cppflags: '-DOPT_THREADED_CODE=0' } } - - { uses: './.github/actions/compilers', name: 'OPT_THREADED_CODE=1', with: { cppflags: '-DOPT_THREADED_CODE=1' } } - - { uses: './.github/actions/compilers', name: 'OPT_THREADED_CODE=2', with: { cppflags: '-DOPT_THREADED_CODE=2' } } - - compile8: - name: 'omnibus compilations, #8' - runs-on: ubuntu-latest - needs: compile-if - if: ${{ needs.compile-if.result == 'success' }} - timeout-minutes: 60 - services: { docuum: { image: 'stephanmisc/docuum', options: '--init', volumes: [ '/root', '/var/run/docker.sock:/var/run/docker.sock' ] } } - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: { sparse-checkout-cone-mode: false, sparse-checkout: /.github } - - { uses: './.github/actions/setup/directories', with: { srcdir: 'src', builddir: 'build', makeup: true, fetch-depth: 10 } } - - { uses: './.github/actions/compilers', name: 'NDEBUG', with: { cppflags: '-DNDEBUG' } } - - { uses: './.github/actions/compilers', name: 'RUBY_DEBUG', with: { cppflags: '-DRUBY_DEBUG' } } - - { uses: './.github/actions/compilers', name: 'ARRAY_DEBUG', with: { cppflags: '-DARRAY_DEBUG' } } - - { uses: './.github/actions/compilers', name: 'BIGNUM_DEBUG', with: { cppflags: '-DBIGNUM_DEBUG' } } - - { uses: './.github/actions/compilers', name: 'CCAN_LIST_DEBUG', with: { cppflags: '-DCCAN_LIST_DEBUG' } } - - { uses: './.github/actions/compilers', name: 'CPDEBUG=-1', with: { cppflags: '-DCPDEBUG=-1' } } - - { uses: './.github/actions/compilers', name: 'ENC_DEBUG', with: { cppflags: '-DENC_DEBUG' } } - - { uses: './.github/actions/compilers', name: 'GC_DEBUG', with: { cppflags: '-DGC_DEBUG' } } - - compile9: - name: 'omnibus compilations, #9' - runs-on: ubuntu-latest - needs: compile-if - if: ${{ needs.compile-if.result == 'success' }} - timeout-minutes: 60 - services: { docuum: { image: 'stephanmisc/docuum', options: '--init', volumes: [ '/root', '/var/run/docker.sock:/var/run/docker.sock' ] } } - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: { sparse-checkout-cone-mode: false, sparse-checkout: /.github } - - { uses: './.github/actions/setup/directories', with: { srcdir: 'src', builddir: 'build', makeup: true, fetch-depth: 10 } } - - { uses: './.github/actions/compilers', name: 'HASH_DEBUG', with: { cppflags: '-DHASH_DEBUG' } } - - { uses: './.github/actions/compilers', name: 'ID_TABLE_DEBUG', with: { cppflags: '-DID_TABLE_DEBUG' } } - - { uses: './.github/actions/compilers', name: 'RGENGC_DEBUG=-1', with: { cppflags: '-DRGENGC_DEBUG=-1' } } - - { uses: './.github/actions/compilers', name: 'SYMBOL_DEBUG', with: { cppflags: '-DSYMBOL_DEBUG' } } - - { uses: './.github/actions/compilers', name: 'RGENGC_CHECK_MODE', with: { cppflags: '-DRGENGC_CHECK_MODE' } } - - { uses: './.github/actions/compilers', name: 'VM_CHECK_MODE', with: { cppflags: '-DVM_CHECK_MODE' } } - - { uses: './.github/actions/compilers', name: 'USE_EMBED_CI=0', with: { cppflags: '-DUSE_EMBED_CI=0' } } - - { uses: './.github/actions/compilers', name: 'USE_FLONUM=0', with: { cppflags: '-DUSE_FLONUM=0', append_configure: '--disable-yjit' } } - - compileX: - name: 'omnibus compilations, #10' - runs-on: ubuntu-latest - needs: compile-if - if: ${{ needs.compile-if.result == 'success' }} - timeout-minutes: 60 - services: { docuum: { image: 'stephanmisc/docuum', options: '--init', volumes: [ '/root', '/var/run/docker.sock:/var/run/docker.sock' ] } } - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: { sparse-checkout-cone-mode: false, sparse-checkout: /.github } - - { uses: './.github/actions/setup/directories', with: { srcdir: 'src', builddir: 'build', makeup: true, fetch-depth: 10 } } - - { uses: './.github/actions/compilers', name: 'USE_LAZY_LOAD', with: { cppflags: '-DUSE_LAZY_LOAD' } } - - { uses: './.github/actions/compilers', name: 'USE_SYMBOL_GC=0', with: { cppflags: '-DUSE_SYMBOL_GC=0' } } - - { uses: './.github/actions/compilers', name: 'USE_THREAD_CACHE=0', with: { cppflags: '-DUSE_THREAD_CACHE=0' } } - - { uses: './.github/actions/compilers', name: 'USE_RUBY_DEBUG_LOG=1', with: { cppflags: '-DUSE_RUBY_DEBUG_LOG=1' } } - - { uses: './.github/actions/compilers', name: 'USE_DEBUG_COUNTER', with: { cppflags: '-DUSE_DEBUG_COUNTER=1' } } - - { uses: './.github/actions/compilers', name: 'SHARABLE_MIDDLE_SUBSTRING', with: { cppflags: '-DSHARABLE_MIDDLE_SUBSTRING=1' } } - - { uses: './.github/actions/compilers', name: 'DEBUG_FIND_TIME_NUMGUESS', with: { cppflags: '-DDEBUG_FIND_TIME_NUMGUESS' } } - - { uses: './.github/actions/compilers', name: 'DEBUG_INTEGER_PACK', with: { cppflags: '-DDEBUG_INTEGER_PACK' } } - - compileB: - name: 'omnibus compilations, #11' - runs-on: ubuntu-latest - needs: compile-if - if: ${{ needs.compile-if.result == 'success' }} - timeout-minutes: 60 - services: { docuum: { image: 'stephanmisc/docuum', options: '--init', volumes: [ '/root', '/var/run/docker.sock:/var/run/docker.sock' ] } } - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: { sparse-checkout-cone-mode: false, sparse-checkout: /.github } - - { uses: './.github/actions/setup/directories', with: { srcdir: 'src', builddir: 'build', makeup: true, fetch-depth: 10 } } - - { uses: './.github/actions/compilers', name: 'GC_DEBUG_STRESS_TO_CLASS', with: { cppflags: '-DGC_DEBUG_STRESS_TO_CLASS' } } - - { uses: './.github/actions/compilers', name: 'GC_ENABLE_LAZY_SWEEP=0', with: { cppflags: '-DGC_ENABLE_LAZY_SWEEP=0' } } - - { uses: './.github/actions/compilers', name: 'GC_PROFILE_DETAIL_MEMORY', with: { cppflags: '-DGC_PROFILE_DETAIL_MEMORY' } } - - { uses: './.github/actions/compilers', name: 'GC_PROFILE_MORE_DETAIL', with: { cppflags: '-DGC_PROFILE_MORE_DETAIL' } } - - { uses: './.github/actions/compilers', name: 'MALLOC_ALLOCATED_SIZE_CHECK', with: { cppflags: '-DMALLOC_ALLOCATED_SIZE_CHECK' } } - - { uses: './.github/actions/compilers', name: 'RGENGC_ESTIMATE_OLDMALLOC', with: { cppflags: '-DRGENGC_ESTIMATE_OLDMALLOC' } } - - { uses: './.github/actions/compilers', name: 'RGENGC_OBJ_INFO', with: { cppflags: '-DRGENGC_OBJ_INFO' } } - - { uses: './.github/actions/compilers', name: 'RGENGC_PROFILE', with: { cppflags: '-DRGENGC_PROFILE' } } - - compileC: - name: 'omnibus compilations, #12' - runs-on: ubuntu-latest - needs: compile-if - if: ${{ needs.compile-if.result == 'success' }} - timeout-minutes: 60 - services: { docuum: { image: 'stephanmisc/docuum', options: '--init', volumes: [ '/root', '/var/run/docker.sock:/var/run/docker.sock' ] } } - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: { sparse-checkout-cone-mode: false, sparse-checkout: /.github } - - { uses: './.github/actions/setup/directories', with: { srcdir: 'src', builddir: 'build', makeup: true, fetch-depth: 10 } } - - { uses: './.github/actions/compilers', name: 'VM_DEBUG_BP_CHECK', with: { cppflags: '-DVM_DEBUG_BP_CHECK' } } - - { uses: './.github/actions/compilers', name: 'VM_DEBUG_VERIFY_METHOD_CACHE', with: { cppflags: '-DVM_DEBUG_VERIFY_METHOD_CACHE' } } - - { uses: './.github/actions/compilers', name: 'enable-yjit', with: { append_configure: '--enable-yjit' } } - - { uses: './.github/actions/compilers', name: 'enable-{y,z}jit', with: { append_configure: '--enable-yjit --enable-zjit' } } - - { uses: './.github/actions/compilers', name: 'enable-{y,z}jit=dev', with: { append_configure: '--enable-yjit=dev --enable-zjit' } } - - { uses: './.github/actions/compilers', name: 'YJIT_FORCE_ENABLE', with: { cppflags: '-DYJIT_FORCE_ENABLE' } } - - { uses: './.github/actions/compilers', name: 'UNIVERSAL_PARSER', with: { cppflags: '-DUNIVERSAL_PARSER' } } - - compilemax: - name: 'omnibus compilations, result' - runs-on: ubuntu-latest - if: ${{ always() }} - needs: - - 'compile1' - - 'compile2' - - 'compile3' - - 'compile4' - - 'compile5' - - 'compile6' - - 'compile7' - - 'compile8' - - 'compile9' - - 'compileX' - - 'compileB' - - 'compileC' - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: { sparse-checkout-cone-mode: false, sparse-checkout: /.github } - - uses: ./.github/actions/slack - with: - label: 'omnibus' - SLACK_WEBHOOK_URL: ${{ secrets.SIMPLER_ALERTS_URL }} # ruby-lang slack: ruby/simpler-alerts-bot - if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }} - - run: false - working-directory: - if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }} - -defaults: - run: - working-directory: build diff --git a/.github/workflows/cygwin.yml b/.github/workflows/cygwin.yml deleted file mode 100644 index 39a98cd30ec9d1..00000000000000 --- a/.github/workflows/cygwin.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: Cygwin -on: - push: - paths-ignore: - - 'doc/**' - - '**/man/*' - - '**.md' - - '**.rdoc' - - '**/.document' - - '.*.yml' - pull_request: - paths-ignore: - - 'doc/**' - - '**/man/*' - - '**.md' - - '**.rdoc' - - '**/.document' - - '.*.yml' - merge_group: - -concurrency: - group: ${{ github.workflow }} / ${{ startsWith(github.event_name, 'pull') && github.ref_name || github.sha }} - cancel-in-progress: ${{ startsWith(github.event_name, 'pull') }} - -permissions: - contents: read - -jobs: - make: - runs-on: windows-2022 - - if: >- - ${{!(false - || contains(github.event.head_commit.message, '[DOC]') - || contains(github.event.pull_request.title, '[DOC]') - || contains(github.event.pull_request.labels.*.name, 'Documentation') - || (github.event_name == 'push' && github.event.pull_request.user.login == 'dependabot[bot]') - )}} - - steps: - - run: git config --global core.autocrlf input - - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - name: Setup Cygwin - uses: cygwin/cygwin-install-action@master - with: - packages: ruby gcc-core make autoconf libtool libssl-devel libyaml-devel libffi-devel zlib-devel - - - name: configure - run: | - ./autogen.sh - ./configure --disable-install-doc - shell: C:\cygwin\bin\bash.EXE --noprofile --norc -e -o igncr -o pipefail {0} - - - name: Extract bundled gems - run: | - make ruby -j5 - make extract-gems - shell: C:\cygwin\bin\bash.EXE --noprofile --norc -e -o igncr -o pipefail {0} - - - name: make all - timeout-minutes: 30 - run: make -j4 V=1 - shell: C:\cygwin\bin\bash.EXE --noprofile --norc -e -o igncr -o pipefail {0} diff --git a/.github/workflows/default_gems.yml b/.github/workflows/default_gems.yml deleted file mode 100644 index 7935afc44a333c..00000000000000 --- a/.github/workflows/default_gems.yml +++ /dev/null @@ -1,96 +0,0 @@ -name: Update default gems list -on: [push, pull_request, merge_group] - -concurrency: - group: ${{ github.workflow }} / ${{ startsWith(github.event_name, 'pull') && github.ref_name || github.sha }} - cancel-in-progress: ${{ startsWith(github.event_name, 'pull') }} - -permissions: - contents: read - -jobs: - update_default_gems: - name: Update default gems list - - permissions: - contents: write # for Git to git push - - runs-on: ubuntu-latest - - if: ${{ github.repository == 'ruby/ruby' }} - - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - token: ${{ (github.repository == 'ruby/ruby' && !startsWith(github.event_name, 'pull')) && secrets.MATZBOT_AUTO_UPDATE_TOKEN || secrets.GITHUB_TOKEN }} - - - id: gems - run: true - if: ${{ github.ref == 'refs/heads/master' }} - - - uses: ./.github/actions/setup/directories - with: - makeup: true - # Skip overwriting MATZBOT_AUTO_UPDATE_TOKEN - checkout: '' # false (ref: https://github.com/actions/runner/issues/2238) - if: ${{ steps.gems.outcome == 'success' }} - - - name: Download previous gems list - run: | - data=default_gems.json - mkdir -p .downloaded-cache - ln -s .downloaded-cache/$data . - curl -O -R -z ./$data https://stdgems.org/$data - if: ${{ steps.gems.outcome == 'success' }} - - - name: Make default gems list - run: | - #!ruby - require 'rubygems' - $:.unshift "lib" - rgver = File.foreach("lib/rubygems.rb") do |line| - break $1 if /^\s*VERSION\s*=\s*"([^"]+)"/ =~ line - end - gems = Dir.glob("{ext,lib}/**/*.gemspec").map do |f| - spec = Gem::Specification.load(f) - "#{spec.name} #{spec.version}" - end.sort - File.open("gems/default_gems", "w") do |f| - f.puts "RubyGems #{rgver}" - f.puts gems - end - shell: ruby --disable=gems {0} - if: ${{ steps.gems.outcome == 'success' }} - - - name: Maintain updated gems list in NEWS - run: | - ruby tool/update-NEWS-gemlist.rb default - if: ${{ steps.gems.outcome == 'success' }} - - - name: Check diffs - id: diff - run: | - git diff --color --no-ext-diff --ignore-submodules --exit-code NEWS.md || - echo update=true >> $GITHUB_OUTPUT - if: ${{ steps.gems.outcome == 'success' }} - - - name: Commit - run: | - git pull --ff-only origin ${GITHUB_REF#refs/heads/} - git commit --message="Update default gems list at ${GITHUB_SHA:0:30} [ci skip]" NEWS.md - git push origin ${GITHUB_REF#refs/heads/} - env: - EMAIL: svn-admin@ruby-lang.org - GIT_AUTHOR_NAME: git - GIT_COMMITTER_NAME: git - if: >- - ${{ - github.repository == 'ruby/ruby' && - !startsWith(github.event_name, 'pull') && - steps.diff.outputs.update - }} - - - uses: ./.github/actions/slack - with: - SLACK_WEBHOOK_URL: ${{ secrets.SIMPLER_ALERTS_URL }} # ruby-lang slack: ruby/simpler-alerts-bot - if: ${{ failure() }} diff --git a/.github/workflows/dependabot_automerge.yml b/.github/workflows/dependabot_automerge.yml deleted file mode 100644 index 3a2ba704aec7ec..00000000000000 --- a/.github/workflows/dependabot_automerge.yml +++ /dev/null @@ -1,32 +0,0 @@ -# from https://github.com/gofiber/swagger/blob/main/.github/workflows/dependabot_automerge.yml -name: Dependabot auto-merge -on: - pull_request: - -permissions: - contents: write - pull-requests: write - -jobs: - automerge: - runs-on: ubuntu-latest - if: github.event.pull_request.user.login == 'dependabot[bot]' && github.repository == 'ruby/ruby' - steps: - - name: Dependabot metadata - uses: dependabot/fetch-metadata@08eff52bf64351f401fb50d4972fa95b9f2c2d1b # v2.4.0 - id: metadata - - - name: Wait for status checks - uses: lewagon/wait-on-check-action@0dceb95e7c4cad8cc7422aee3885998f5cab9c79 # v1.4.0 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - ref: ${{ github.event.pull_request.head.sha || github.sha }} - check-regexp: 'make \(check, .*\)' - wait-interval: 30 - - - name: Auto-merge for Dependabot PRs - if: ${{ steps.metadata.outputs.update-type == 'version-update:semver-minor' || steps.metadata.outputs.update-type == 'version-update:semver-patch' }} - run: gh pr merge --auto --rebase "$PR_URL" - env: - PR_URL: ${{ github.event.pull_request.html_url }} - GITHUB_TOKEN: ${{ secrets.MATZBOT_DEPENDABOT_MERGE_TOKEN }} diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml deleted file mode 100644 index e57cd86e2b3c7f..00000000000000 --- a/.github/workflows/labeler.yml +++ /dev/null @@ -1,12 +0,0 @@ -name: "Pull Request Labeler" -on: -- pull_request_target - -jobs: - labeler: - permissions: - contents: read - pull-requests: write - runs-on: ubuntu-latest - steps: - - uses: actions/labeler@v5 diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml deleted file mode 100644 index d22ada675545bb..00000000000000 --- a/.github/workflows/macos.yml +++ /dev/null @@ -1,205 +0,0 @@ -name: macOS -on: - push: - paths-ignore: - - 'doc/**' - - '**/man/*' - - '**.md' - - '**.rdoc' - - '**/.document' - - '.*.yml' - pull_request: - # Do not use paths-ignore for required status checks - # https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/collaborating-on-repositories-with-code-quality-features/troubleshooting-required-status-checks#handling-skipped-but-required-checks - merge_group: - -concurrency: - group: ${{ github.workflow }} / ${{ startsWith(github.event_name, 'pull') && github.ref_name || github.sha }} - cancel-in-progress: ${{ startsWith(github.event_name, 'pull') }} - -permissions: - contents: read - -jobs: - make: - strategy: - matrix: - include: - - test_task: check - os: macos-14 - - test_task: check - os: macos-14 - configure_args: '--with-gcc=gcc-14' - - test_task: check - os: macos-14 - configure_args: '--with-jemalloc --with-opt-dir=$(brew --prefix jemalloc)' - - test_task: check - os: macos-14 - configure_args: '--with-gmp' - - test_task: test-all - test_opts: --repeat-count=2 - os: macos-14 - - test_task: test-bundler-parallel - os: macos-14 - - test_task: test-bundled-gems - os: macos-14 - - test_task: check - os: macos-15 - capi_check: capi - - test_task: check - os: macos-13 - fail-fast: false - - env: - GITPULLOPTIONS: --no-tags origin ${{ github.ref }} - - runs-on: ${{ matrix.os }} - - if: >- - ${{!(false - || contains(github.event.head_commit.message, '[DOC]') - || contains(github.event.pull_request.title, '[DOC]') - || contains(github.event.pull_request.labels.*.name, 'Documentation') - || (github.event_name == 'push' && github.event.pull_request.user.login == 'dependabot[bot]') - )}} - - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - sparse-checkout-cone-mode: false - sparse-checkout: /.github - - - name: Install libraries - uses: ./.github/actions/setup/macos - - - uses: ./.github/actions/setup/directories - with: - srcdir: src - builddir: build - makeup: true - clean: true - dummy-files: ${{ matrix.test_task == 'check' }} - # Set fetch-depth: 0 so that Launchable can receive commits information. - fetch-depth: 10 - - - name: make sure that kern.coredump=1 - run: | - sysctl -n kern.coredump - sudo sysctl -w kern.coredump=1 - sudo chmod -R +rwx /cores/ - - - name: Delete unused SDKs - # To free up disk space to not run out during the run - run: | - sudo rm -rf ~/.dotnet - sudo rm -rf /Library/Android - sudo rm -rf /Library/Developer/CoreSimulator - continue-on-error: true - - - name: Run configure - run: ../src/configure -C --disable-install-doc ${ruby_configure_args} ${{ matrix.configure_args }} - - - run: make prepare-gems - if: ${{ matrix.test_task == 'test-bundled-gems' }} - - - run: make - - - run: make hello - - - name: runirb - run: | - echo IRB::VERSION | make runirb RUNOPT="-- -f" - - - name: Set test options for skipped tests - run: | - set -x - TESTS="$(echo "${{ matrix.skipped_tests }}" | sed 's| |$$/ -n!/|g;s|^|-n!/|;s|$|$$/|')" - echo "TESTS=${TESTS}" >> $GITHUB_ENV - if: ${{ matrix.test_task == 'check' && matrix.skipped_tests }} - - - name: Set up Launchable - id: launchable - uses: ./.github/actions/launchable/setup - with: - os: ${{ matrix.os }} - test-opts: ${{ matrix.test_opts }} - launchable-token: ${{ secrets.LAUNCHABLE_TOKEN }} - builddir: build - srcdir: src - continue-on-error: true - timeout-minutes: 3 - - - name: Set extra test options - run: | - echo "TESTS=$TESTS ${{ matrix.test_opts }}" >> $GITHUB_ENV - echo "RUBY_TEST_TIMEOUT_SCALE=10" >> $GITHUB_ENV # With --repeat-count=2, flaky test by timeout occurs frequently for some reason - if: matrix.test_opts - - - name: make ${{ matrix.test_task }} - run: | - test -n "${LAUNCHABLE_STDOUT}" && exec 1> >(tee "${LAUNCHABLE_STDOUT}") - test -n "${LAUNCHABLE_STDERR}" && exec 2> >(tee "${LAUNCHABLE_STDERR}") - - ulimit -c unlimited - make -s ${{ matrix.test_task }} ${TESTS:+TESTS="$TESTS"} - timeout-minutes: 90 - env: - RUBY_TESTOPTS: '-q --tty=no' - TEST_BUNDLED_GEMS_ALLOW_FAILURES: '' - PRECHECK_BUNDLED_GEMS: 'no' - LAUNCHABLE_STDOUT: ${{ steps.launchable.outputs.stdout_report_path }} - LAUNCHABLE_STDERR: ${{ steps.launchable.outputs.stderr_report_path }} - - - name: make skipped tests - run: | - make -s test-all TESTS="${TESTS//-n!\//-n/}" - env: - GNUMAKEFLAGS: '' - RUBY_TESTOPTS: '-v --tty=no' - PRECHECK_BUNDLED_GEMS: 'no' - if: ${{ matrix.test_task == 'check' && matrix.skipped_tests }} - continue-on-error: ${{ matrix.continue-on-skipped_tests || false }} - - - name: CAPI extensions - uses: ./.github/actions/capiext - with: - builddir: build - env: - RUBY_TESTOPTS: '-v --tty=no' - if: ${{ matrix.capi_check }} - - - uses: ./.github/actions/slack - with: - label: ${{ matrix.os }} / ${{ matrix.test_task }} - SLACK_WEBHOOK_URL: ${{ secrets.SIMPLER_ALERTS_URL }} # ruby-lang slack: ruby/simpler-alerts-bot - if: ${{ failure() }} - - - name: Resolve job ID - id: job_id - uses: actions/github-script@main - env: - matrix: ${{ toJson(matrix) }} - with: - script: | - const { data: workflow_run } = await github.rest.actions.listJobsForWorkflowRun({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.runId - }); - const matrix = JSON.parse(process.env.matrix); - const job_name = `${context.job}${matrix ? ` (${Object.values(matrix).join(", ")})` : ""}`; - return workflow_run.jobs.find((job) => job.name === job_name).id; - - result: - if: ${{ always() }} - name: ${{ github.workflow }} result - runs-on: macos-latest - needs: [make] - steps: - - run: exit 1 - working-directory: - if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }} - -defaults: - run: - working-directory: build diff --git a/.github/workflows/mingw.yml b/.github/workflows/mingw.yml deleted file mode 100644 index 4a71f1753011a6..00000000000000 --- a/.github/workflows/mingw.yml +++ /dev/null @@ -1,221 +0,0 @@ -name: MinGW -on: - push: - paths-ignore: - - 'doc/**' - - '**/man/*' - - '**.md' - - '**.rdoc' - - '**/.document' - - '.*.yml' - pull_request: - paths-ignore: - - 'doc/**' - - '**/man/*' - - '**.md' - - '**.rdoc' - - '**/.document' - - '.*.yml' - merge_group: - -concurrency: - group: ${{ github.workflow }} / ${{ startsWith(github.event_name, 'pull') && github.ref_name || github.sha }} - cancel-in-progress: ${{ startsWith(github.event_name, 'pull') }} - -permissions: - contents: read - -# Notes: -# Actions console encoding causes issues, see test-all & test-spec steps -# -jobs: - make: - runs-on: windows-2022 - - name: ${{ github.workflow }} (${{ matrix.msystem }}) - - env: - MSYSTEM: ${{ matrix.msystem }} - MSYS2_ARCH: x86_64 - CHOST: 'x86_64-w64-mingw32' - CFLAGS: '-march=x86-64 -mtune=generic -O3 -pipe' - CXXFLAGS: '-march=x86-64 -mtune=generic -O3 -pipe' - CPPFLAGS: '-D_FORTIFY_SOURCE=2 -D__USE_MINGW_ANSI_STDIO=1 -DFD_SETSIZE=2048' - LDFLAGS: '-pipe' - GITPULLOPTIONS: --no-tags origin ${{ github.ref }} - - strategy: - matrix: - include: - # To mitigate flakiness of MinGW CI, we test only one runtime that newer MSYS2 uses. - # Ruby 3.2 is the first Windows Ruby to use OpenSSL 3.x - - msystem: 'UCRT64' - test_task: 'check' - test-all-opts: '--name=!/TestObjSpace#test_reachable_objects_during_iteration/' - fail-fast: false - - if: >- - ${{!(false - || contains(github.event.head_commit.message, '[DOC]') - || contains(github.event.pull_request.title, '[DOC]') - || contains(github.event.pull_request.labels.*.name, 'Documentation') - || (github.event_name == 'push' && github.event.pull_request.user.login == 'dependabot[bot]') - )}} - - steps: - - uses: msys2/setup-msys2@fb197b72ce45fb24f17bf3f807a388985654d1f2 # v2.29.0 - id: msys2 - with: - msystem: UCRT64 - update: true - install: >- - git - make - ruby - autoconf - mingw-w64-ucrt-x86_64-gcc - mingw-w64-ucrt-x86_64-ragel - mingw-w64-ucrt-x86_64-openssl - mingw-w64-ucrt-x86_64-libyaml - mingw-w64-ucrt-x86_64-libffi - - - name: Set up env - id: setup-env - working-directory: - run: | - $msys2 = ${env:MSYS2_LOCATION} - echo $msys2\usr\bin $msys2\ucrt64\bin | - Tee-Object ${env:GITHUB_PATH} -Append -Encoding utf-8 - - # Use the fast device for the temporary directory. - # %TEMP% is inconsistent with %TMP% and test-all expects they are consistent. - # https://github.com/actions/virtual-environments/issues/712#issuecomment-613004302 - $tmp = ${env:RUNNER_TEMP} - echo HOME=$home TMP=$tmp TEMP=$tmp TMPDIR=$tmp | - Tee-Object ${env:GITHUB_ENV} -Append -Encoding utf-8 - shell: pwsh # cmd.exe does not strip spaces before `|`. - env: - MSYS2_LOCATION: ${{ steps.msys2.outputs.msys2-location }} - - - name: Remove Strawberry Perl pkg-config - working-directory: - # `pkg-config.bat` included in Strawberry Perl is written in - # Perl and doesn't work when another msys2 `perl` precede its - # own `perl`. - # - # ``` - # Can't find C:\Strawberry\perl\bin\pkg-config.bat on PATH, '.' not in PATH. - # ``` - run: | - Get-Command pkg-config.bat | % { ren $_.path ($_.path + "~") } - shell: pwsh - - - name: Misc system & package info - working-directory: - run: | - group() { echo ::group::$'\e[94;1m'"$*"$'\e[m'; } - endgroup() { echo ::endgroup::; } - - group Path - cygpath -wa / . $(type -p cygpath bash sh) - endgroup - - I() { - group $1 - run Where type -pa $1 && { [ $# -eq 1 ] || run Version "$@"; } || - failed+=($1) - endgroup - } - run() { local w m=$1; shift; w="$("$@")" && show "$m" && indent "$w"; } - indent() { [ -z "$1" ] || echo "$1" | /bin/sed '/^$/!s/^/ /'; } - show() { echo $'\e[96m'"$*"$'\e[m'; } - - failed=() - - I gcc.exe --version - I ragel.exe --version - I make.exe --version - I openssl.exe version - I libcrypto-3-x64.dll - I libssl-3-x64.dll - - group Packages - pacman -Qs mingw-w64-ucrt-x86_64-* | /bin/sed -n "s,local/mingw-w64-ucrt-x86_64-,,p" - endgroup - - [ ${#failed[@]} -eq 0 ] - shell: sh - - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - sparse-checkout-cone-mode: false - sparse-checkout: /.github - - - uses: ./.github/actions/setup/directories - with: - srcdir: src - builddir: build - makeup: true - # Set fetch-depth: 10 so that Launchable can receive commits information. - fetch-depth: 10 - - - name: configure - run: > - ../src/configure --disable-install-doc --prefix=/. - --build=$CHOST --host=$CHOST --target=$CHOST - shell: sh - - - name: make all - timeout-minutes: 30 - run: make -j4 - - - name: make install - run: make DESTDIR=../install install-nodoc - - - name: Set up Launchable - uses: ./.github/actions/launchable/setup - with: - os: windows-2022 - launchable-token: ${{ secrets.LAUNCHABLE_TOKEN }} - builddir: build - srcdir: src - test-tasks: '["test", "test-all", "test-spec"]' - continue-on-error: true - timeout-minutes: 3 - - - name: test - timeout-minutes: 30 - run: make test - env: - GNUMAKEFLAGS: '' - RUBY_TESTOPTS: '-v --tty=no' - if: ${{ matrix.test_task == 'check' || matrix.test_task == 'test' }} - - - name: test-all - timeout-minutes: 45 - run: | - make ${{ StartsWith(matrix.test_task, 'test/') && matrix.test_task || 'test-all' }} - env: - RUBY_TESTOPTS: >- - --retry --job-status=normal --show-skip --timeout-scale=1.5 -j4 - ${{ matrix.test-all-opts }} - ${{ env.TESTS }} - BUNDLER_VERSION: - if: ${{ matrix.test_task == 'check' || matrix.test_task == 'test-all' || StartsWith(matrix.test_task, 'test/') }} - - - name: test-spec - timeout-minutes: 10 - run: | - make ${{ StartsWith(matrix.test_task, 'spec/') && matrix.test_task || 'test-spec' }} - if: ${{ matrix.test_task == 'check' || matrix.test_task == 'test-spec' || StartsWith(matrix.test_task, 'spec/') }} - - - uses: ./src/.github/actions/slack - with: - label: ${{ matrix.msystem }} / ${{ matrix.test_task }} - SLACK_WEBHOOK_URL: ${{ secrets.SIMPLER_ALERTS_URL }} # ruby-lang slack: ruby/simpler-alerts-bot - if: ${{ failure() }} - -defaults: - run: - working-directory: build - shell: cmd diff --git a/.github/workflows/modgc.yml b/.github/workflows/modgc.yml deleted file mode 100644 index 96b1afa05fc48b..00000000000000 --- a/.github/workflows/modgc.yml +++ /dev/null @@ -1,176 +0,0 @@ -name: ModGC -on: - push: - paths-ignore: - - 'doc/**' - - '**/man/*' - - '**.md' - - '**.rdoc' - - '**/.document' - - '.*.yml' - pull_request: - # Do not use paths-ignore for required status checks - # https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/collaborating-on-repositories-with-code-quality-features/troubleshooting-required-status-checks#handling-skipped-but-required-checks - merge_group: - -concurrency: - group: ${{ github.workflow }} / ${{ startsWith(github.event_name, 'pull') && github.ref_name || github.sha }} - cancel-in-progress: ${{ startsWith(github.event_name, 'pull') }} - -permissions: - contents: read - -jobs: - check: - strategy: - matrix: - gc: - - name: default - - name: mmtk - mmtk_build: release - os: [macos-latest, ubuntu-latest] - include: - - test_task: check - fail-fast: false - - env: - GITPULLOPTIONS: --no-tags origin ${{ github.ref }} - RUBY_DEBUG: ci - - runs-on: ${{ matrix.os }} - - if: >- - ${{!(false - || contains(github.event.head_commit.message, '[DOC]') - || contains(github.event.pull_request.title, '[DOC]') - || contains(github.event.pull_request.labels.*.name, 'Documentation') - || (github.event_name == 'push' && github.event.pull_request.user.login == 'dependabot[bot]') - )}} - - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - sparse-checkout-cone-mode: false - sparse-checkout: /.github - - - name: Install libraries (macOS) - uses: ./.github/actions/setup/macos - if: ${{ contains(matrix.os, 'macos') }} - - - name: Install libraries (Ubuntu) - uses: ./.github/actions/setup/ubuntu - if: ${{ contains(matrix.os, 'ubuntu') }} - - - uses: ruby/setup-ruby@a9bfc2ecf3dd40734a9418f89a7e9d484c32b990 # v1.248.0 - with: - ruby-version: '3.1' - bundler: none - if: ${{ contains(matrix.os, 'ubuntu') }} - - - uses: ./.github/actions/setup/directories - with: - srcdir: src - builddir: build - makeup: true - clean: true - dummy-files: false - # Set fetch-depth: 10 so that Launchable can receive commits information. - fetch-depth: 10 - - - name: make sure that kern.coredump=1 - run: | - sysctl -n kern.coredump - sudo sysctl -w kern.coredump=1 - sudo chmod -R +rwx /cores/ - if: ${{ contains(matrix.os, 'macos') }} - - - name: Delete unused SDKs - # To free up disk space to not run out during the run - run: | - sudo rm -rf ~/.dotnet - sudo rm -rf /Library/Android - sudo rm -rf /Library/Developer/CoreSimulator - continue-on-error: true - if: ${{ contains(matrix.os, 'macos') }} - - - name: Setup Ruby GC Directory - run: | - echo "MODULAR_GC_DIR=$HOME/ruby_gc" >> $GITHUB_ENV - - - name: Run configure - env: - arch: ${{ matrix.arch }} - run: >- - $SETARCH ../src/configure -C --disable-install-doc --with-modular-gc=${{ env.MODULAR_GC_DIR }} - ${arch:+--target=$arch-$OSTYPE --host=$arch-$OSTYPE} - - - uses: actions-rust-lang/setup-rust-toolchain@v1 - - name: Set MMTk environment variables - run: | - echo 'EXCLUDES=../src/test/.excludes-mmtk' >> $GITHUB_ENV - echo 'MSPECOPT=-B../src/spec/mmtk.mspec' >> $GITHUB_ENV - if: ${{ matrix.gc.name == 'mmtk' }} - - - run: $SETARCH make - - - name: Build Modular GC - run: | - echo "RUBY_GC_LIBRARY=${{ matrix.gc.name }}" >> $GITHUB_ENV - make install-modular-gc MODULAR_GC=${{ matrix.gc.name }} MMTK_BUILD=${{ matrix.gc.mmtk_build }} - make distclean-modular-gc MODULAR_GC=${{ matrix.gc.name }} - - - run: $SETARCH make hello - - - name: Set test options for skipped tests - run: | - set -x - TESTS="$(echo "${{ matrix.skipped_tests }}" | sed 's| |$$/ -n!/|g;s|^|-n!/|;s|$|$$/|')" - echo "TESTS=${TESTS}" >> $GITHUB_ENV - if: ${{ matrix.test_task == 'check' && matrix.skipped_tests }} - - - name: Set up Launchable - id: launchable - uses: ./.github/actions/launchable/setup - with: - os: ${{ matrix.os || 'ubuntu-22.04' }} - test-opts: ${{ matrix.configure }} - launchable-token: ${{ secrets.LAUNCHABLE_TOKEN }} - builddir: build - srcdir: src - continue-on-error: true - timeout-minutes: 3 - - - name: make ${{ matrix.test_task }} - run: | - test -n "${LAUNCHABLE_STDOUT}" && exec 1> >(tee "${LAUNCHABLE_STDOUT}") - test -n "${LAUNCHABLE_STDERR}" && exec 2> >(tee "${LAUNCHABLE_STDERR}") - - $SETARCH make -s ${{ matrix.test_task }} \ - ${TESTS:+TESTS="$TESTS"} \ - ${{ !contains(matrix.test_task, 'bundle') && 'RUBYOPT=-w' || '' }} - timeout-minutes: ${{ matrix.gc.timeout || 40 }} - env: - RUBY_TESTOPTS: '-q --tty=no' - TEST_BUNDLED_GEMS_ALLOW_FAILURES: '' - PRECHECK_BUNDLED_GEMS: 'no' - LAUNCHABLE_STDOUT: ${{ steps.launchable.outputs.stdout_report_path }} - LAUNCHABLE_STDERR: ${{ steps.launchable.outputs.stderr_report_path }} - - - name: make skipped tests - run: | - $SETARCH make -s test-all TESTS="${TESTS//-n!\//-n/}" - env: - GNUMAKEFLAGS: '' - RUBY_TESTOPTS: '-v --tty=no' - if: ${{ matrix.test_task == 'check' && matrix.skipped_tests }} - continue-on-error: ${{ matrix.continue-on-skipped_tests || false }} - - - uses: ./.github/actions/slack - with: - label: ${{ matrix.test_task }} ${{ matrix.configure }}${{ matrix.arch }} - SLACK_WEBHOOK_URL: ${{ secrets.SIMPLER_ALERTS_URL }} # ruby-lang slack: ruby/simpler-alerts-bot - if: ${{ failure() }} - -defaults: - run: - working-directory: build diff --git a/.github/workflows/parse_y.yml b/.github/workflows/parse_y.yml deleted file mode 100644 index 04be9f309d1c33..00000000000000 --- a/.github/workflows/parse_y.yml +++ /dev/null @@ -1,100 +0,0 @@ -name: parse.y -on: - push: - paths-ignore: - - 'doc/**' - - '**/man/*' - - '**.md' - - '**.rdoc' - - '**/.document' - - '.*.yml' - pull_request: - paths-ignore: - - 'doc/**' - - '**/man/*' - - '**.md' - - '**.rdoc' - - '**/.document' - - '.*.yml' - merge_group: - -concurrency: - group: ${{ github.workflow }} / ${{ startsWith(github.event_name, 'pull') && github.ref_name || github.sha }} - cancel-in-progress: ${{ startsWith(github.event_name, 'pull') }} - -permissions: - contents: read - -jobs: - make: - strategy: - matrix: - include: - - test_task: check - - test_task: test-bundler-parallel - - test_task: test-bundled-gems - fail-fast: false - - env: - GITPULLOPTIONS: --no-tags origin ${{ github.ref }} - RUBY_DEBUG: ci - SETARCH: ${{ matrix.arch && format('setarch {0}', matrix.arch) }} - - runs-on: ubuntu-22.04 - - if: >- - ${{!(false - || contains(github.event.head_commit.message, '[DOC]') - || contains(github.event.pull_request.title, '[DOC]') - || contains(github.event.pull_request.labels.*.name, 'Documentation') - || (github.event_name == 'push' && github.event.pull_request.user.login == 'dependabot[bot]') - )}} - - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - sparse-checkout-cone-mode: false - sparse-checkout: /.github - - - uses: ./.github/actions/setup/ubuntu - - - uses: ruby/setup-ruby@a9bfc2ecf3dd40734a9418f89a7e9d484c32b990 # v1.248.0 - with: - ruby-version: '3.1' - bundler: none - - - uses: ./.github/actions/setup/directories - with: - srcdir: src - builddir: build - makeup: true - clean: true - dummy-files: ${{ matrix.test_task == 'check' }} - - - name: Run configure - run: ../src/configure -C --disable-install-doc cppflags=-DRUBY_DEBUG --with-parser=parse.y - - - run: make - - - run: make TESTRUN_SCRIPT='-renvutil -v -e "exit EnvUtil.current_parser == %[parse.y]"' run - env: - RUNOPT0: -I$(tooldir)/lib - - - name: make ${{ matrix.test_task }} - run: make -s ${{ matrix.test_task }} RUN_OPTS="$RUN_OPTS" SPECOPTS="$SPECOPTS" - env: - RUBY_TESTOPTS: ${{ matrix.testopts }} - EXCLUDES: '../src/test/.excludes-parsey' - RUN_OPTS: ${{ matrix.run_opts || '--parser=parse.y' }} - SPECOPTS: ${{ matrix.specopts || '-T --parser=parse.y' }} - TEST_BUNDLED_GEMS_ALLOW_FAILURES: '' - - - uses: ./.github/actions/slack - with: - label: ${{ matrix.run_opts }} - SLACK_WEBHOOK_URL: ${{ secrets.SIMPLER_ALERTS_URL }} # ruby-lang slack: ruby/simpler-alerts-bot - if: ${{ failure() }} - -defaults: - run: - working-directory: build diff --git a/.github/workflows/pr-playground.yml b/.github/workflows/pr-playground.yml deleted file mode 100644 index f3c05564294778..00000000000000 --- a/.github/workflows/pr-playground.yml +++ /dev/null @@ -1,127 +0,0 @@ -name: Post Playground link to PR -on: - pull_request_target: - types: [labeled] - workflow_run: - workflows: ["WebAssembly"] - types: [completed] - -jobs: - post-summary: - name: Post Playground link - runs-on: ubuntu-latest - permissions: - pull-requests: write - # Post a comment only if the PR status check is passed and the PR is labeled with `Playground`. - # Triggered twice: when the PR is labeled and when PR build is passed. - if: >- - ${{ false - || (true - && github.event_name == 'pull_request_target' - && contains(github.event.pull_request.labels.*.name, 'Playground')) - || (true - && github.event_name == 'workflow_run' - && github.event.workflow_run.conclusion == 'success' - && github.event.workflow_run.event == 'pull_request') - }} - steps: - - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const fs = require('fs/promises'); - - const buildWorkflowPath = '.github/workflows/wasm.yml'; - const findSuccessfuBuildRun = async (pr) => { - const opts = github.rest.actions.listWorkflowRunsForRepo.endpoint.merge({ - owner: context.repo.owner, - repo: context.repo.repo, - status: 'success', - branch: pr.head.ref, - }); - const runs = await github.paginate(opts); - const buildRun = runs.find(run => run.path == buildWorkflowPath); - return buildRun; - } - - const postComment = async (body, pr) => { - const { data: comments } = await github.rest.issues.listComments({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: pr.number, - }); - - const commentOpts = { owner: context.repo.owner, repo: context.repo.repo, body: comment }; - - const existingComment = comments.find(comment => comment.body.startsWith(magicComment)); - if (existingComment) { - core.info(`Updating existing comment: ${existingComment.html_url}`); - await github.rest.issues.updateComment({ - ...commentOpts, comment_id: existingComment.id - }); - } else { - await github.rest.issues.createComment({ - ...commentOpts, issue_number: pr.number - }); - } - } - - const derivePRNumber = async () => { - if (context.payload.pull_request) { - return context.payload.pull_request.number; - } - // Workaround for https://github.com/orgs/community/discussions/25220 - - const { data: { artifacts } } = await github.rest.actions.listWorkflowRunArtifacts({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.payload.workflow_run.id, - }); - const artifact = artifacts.find(artifact => artifact.name == 'github-pr-info'); - if (!artifact) { - throw new Error('Cannot find github-pr-info.txt artifact'); - } - - const { data } = await github.rest.actions.downloadArtifact({ - owner: context.repo.owner, - repo: context.repo.repo, - artifact_id: artifact.id, - archive_format: 'zip', - }); - - await fs.writeFile('pr-info.zip', Buffer.from(data)); - await exec.exec('unzip', ['pr-info.zip']); - return await fs.readFile('github-pr-info.txt', 'utf8'); - } - - const prNumber = await derivePRNumber(); - - const { data: pr } = await github.rest.pulls.get({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: prNumber, - }); - - core.info(`Checking if the PR ${prNumber} is labeled with Playground...`); - if (!pr.labels.some(label => label.name == 'Playground')) { - core.info(`The PR is not labeled with Playground.`); - return; - } - - core.info(`Checking if the build is successful for ${pr.head.ref} in ${pr.head.repo.owner.login}/${pr.head.repo.name}...`); - const buildRun = await findSuccessfuBuildRun(pr); - if (!buildRun) { - core.info(`No successful build run found for ${buildWorkflowPath} on ${pr.head.ref} yet.`); - return; - } - core.info(`Found a successful build run: ${buildRun.html_url}`); - - const runLink = `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`; - const magicComment = ``; - const comment = `${magicComment} - **Try on Playground**: https://ruby.github.io/play-ruby?run=${buildRun.id} - This is an automated comment by [\`pr-playground.yml\`](${runLink}) workflow. - `; - core.info(`Comment: ${comment}`); - await postComment(comment, pr); - diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml deleted file mode 100644 index 5d4474d978f51a..00000000000000 --- a/.github/workflows/publish.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Start release workflow -on: - push: - tags: - - '*' - -jobs: - notify: - runs-on: ubuntu-latest - steps: - - name: Build release package - run: | - curl -L -X POST \ - -H "Authorization: Bearer ${{ secrets.MATZBOT_GITHUB_WORKFLOW_TOKEN }}" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/ruby/actions/dispatches \ - -d '{"event_type": "${{ github.ref }}"}' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index aeb116bf453032..00000000000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,107 +0,0 @@ -name: Publish Ruby packages - -on: - repository_dispatch: - types: - - release - workflow_dispatch: - inputs: - version: - description: 'Version of the Ruby package to release' - required: true - default: '3.3.4' - -jobs: - release: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v5 - - - uses: ruby/setup-ruby@v1 - with: - ruby-version: 3.3.4 - - - name: Store Ruby version - run: | - echo "RUBY_VERSION=${{ github.event.client_payload.version || github.event.inputs.version }}" >> $GITHUB_ENV - - - name: Store ABI version - run: echo "ABI_VERSION=$(echo ${{ env.RUBY_VERSION }} | cut -d '.' -f 1-2)" >> $GITHUB_ENV - - - name: Copy draft package `/tmp` to `/pub` directory - run: tool/release.sh ${{ env.RUBY_VERSION }} - env: - AWS_ACCESS_KEY_ID: ${{ secrets.FTP_R_L_O_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.FTP_R_L_O_AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: us-west-2 - - - name: Purge URLs of release package - run: | - curl -X POST \ - -H "Fastly-Key: ${{ secrets.FASTLY_PURGE_TOKEN }}" \ - https://api.fastly.com/purge/cache.ruby-lang.org/pub/ruby/${{ env.ABI_VERSION }}/ruby-${{ env.RUBY_VERSION }}.tar.gz - curl -X POST \ - -H "Fastly-Key: ${{ secrets.FASTLY_PURGE_TOKEN }}" \ - https://api.fastly.com/purge/cache.ruby-lang.org/pub/ruby/${{ env.ABI_VERSION }}/ruby-${{ env.RUBY_VERSION }}.tar.xz - curl -X POST \ - -H "Fastly-Key: ${{ secrets.FASTLY_PURGE_TOKEN }}" \ - https://api.fastly.com/purge/cache.ruby-lang.org/pub/ruby/${{ env.ABI_VERSION }}/ruby-${{ env.RUBY_VERSION }}.zip - - - name: Create a release on GitHub - run: | - RELEASE_TAG=$(echo v${{ env.RUBY_VERSION }} | sed 's/\./_/g') - echo $RELEASE_TAG - PREVIOUS_RELEASE_TAG=$(echo $RELEASE_TAG | awk 'BEGIN {FS="_"; OFS="_"}{ $NF=$NF-1; print }') - echo $PREVIOUS_RELEASE_TAG - tool/gen-github-release.rb $PREVIOUS_RELEASE_TAG $RELEASE_TAG --no-dry-run - env: - GITHUB_TOKEN: ${{ secrets.MATZBOT_AUTO_UPDATE_TOKEN }} - - - name: Update versions index - run: | - curl -L -X POST \ - -H "Authorization: Bearer ${{ secrets.MATZBOT_GITHUB_WORKFLOW_TOKEN }}" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/ruby/actions/dispatches \ - -d '{"event_type": "update_index"}' - - - name: Build and push Docker images - run: | - curl -L -X POST \ - -H "Authorization: Bearer ${{ secrets.MATZBOT_GITHUB_WORKFLOW_TOKEN }}" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/ruby/docker-images/dispatches \ - -d '{"event_type": "build", "client_payload": {"ruby_version": "${{ env.RUBY_VERSION }}"}}' - - - name: Build snapcraft packages - run: | - curl -L -X POST \ - -H "Authorization: Bearer ${{ secrets.MATZBOT_GITHUB_WORKFLOW_TOKEN }}" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/ruby/snap.ruby/dispatches \ - -d '{"event_type": "build", "client_payload": {"ruby_version": "${{ env.RUBY_VERSION }}"}}' - - - name: Store the latest LTS version of OpenSSL - run: | - echo "OPENSSL_VERSION=`curl -s https://api.github.com/repos/openssl/openssl/releases | jq -r '.[].tag_name | select(startswith("openssl-3.0"))' | sort -Vr | head -n1 | cut -d'-' -f2`" >> $GITHUB_ENV - - - name: Update ruby-build definition - run: | - curl -L -X POST \ - -H "Authorization: Bearer ${{ secrets.RUBY_BUILD_WORKFLOW_TOKEN }}" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/rbenv/ruby-build/dispatches \ - -d '{"event_type": "update-ruby", "client_payload": {"ruby_version": "${{ env.RUBY_VERSION }}", "openssl_version": "${{ env.OPENSSL_VERSION }}"}}' - - - name: Update all-ruby definition - run: | - curl -L -X POST \ - -H "Authorization: Bearer ${{ secrets.MATZBOT_GITHUB_WORKFLOW_TOKEN }}" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/ruby/all-ruby/dispatches \ - -d '{"event_type": "update"}' diff --git a/.github/workflows/rust-warnings.yml b/.github/workflows/rust-warnings.yml deleted file mode 100644 index 98ddfead5c8522..00000000000000 --- a/.github/workflows/rust-warnings.yml +++ /dev/null @@ -1,51 +0,0 @@ -# Surface Rust warnings on PRs that touch any Rust code. -# Not a required check so we never block people over new warnings -# that might come from a new Rust version being released. -name: Rust warnings -on: - pull_request: - types: - - opened - - synchronize - - reopened - paths: - - '**.rs' - - '!**.inc.rs' - merge_group: - -concurrency: - group: ${{ github.workflow }} / ${{ startsWith(github.event_name, 'pull') && github.ref_name || github.sha }} - cancel-in-progress: ${{ startsWith(github.event_name, 'pull') }} - -permissions: - contents: read - -jobs: - make: - env: - GITPULLOPTIONS: --no-tags origin ${{ github.ref }} - - runs-on: ubuntu-24.04 - - if: >- - ${{!(false - || contains(github.event.head_commit.message, '[DOC]') - || contains(github.event.pull_request.title, '[DOC]') - || contains(github.event.pull_request.labels.*.name, 'Documentation') - || (github.event_name == 'push' && github.event.pull_request.user.login == 'dependabot[bot]') - )}} - - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - name: Install Rust - run: rustup default beta - - - name: Rust warnings - shell: bash - run: | - set -eu - cargo check --quiet --all-features --message-format=json \ - | jq -r 'select(.message.level == "warning" or .message.level == "error") | .message.rendered' \ - | tee messages.txt - (exit "${PIPESTATUS[0]}") && ! grep --quiet '[^[:space:]]' messages.txt diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml deleted file mode 100644 index c997a6a453be76..00000000000000 --- a/.github/workflows/scorecards.yml +++ /dev/null @@ -1,78 +0,0 @@ -# This workflow uses actions that are not certified by GitHub. They are provided -# by a third-party and are governed by separate terms of service, privacy -# policy, and support documentation. - -name: Scorecard supply-chain security -on: - # For Branch-Protection check. Only the default branch is supported. See - # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection - branch_protection_rule: - # To guarantee Maintained check is occasionally updated. See - # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained - schedule: - - cron: '39 3 * * 5' - # push: - # branches: [ "master" ] - -# Declare default permissions as read only. -permissions: read-all - -jobs: - analysis: - name: Scorecard analysis - runs-on: ubuntu-latest - # `publish_results: true` only works when run from the default branch. conditional can be removed if disabled. - if: github.event.repository.default_branch == github.ref_name || github.event_name == 'pull_request' - permissions: - # Needed to upload the results to code-scanning dashboard. - security-events: write - # Needed to publish results and get a badge (see publish_results below). - id-token: write - # Uncomment the permissions below if installing in a private repository. - # contents: read - # actions: read - - steps: - - name: "Checkout code" - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - persist-credentials: false - - - name: "Run analysis" - uses: ossf/scorecard-action@05b42c624433fc40578a4040d5cf5e36ddca8cde # v2.4.2 - with: - results_file: results.sarif - results_format: sarif - # (Optional) "write" PAT token. Uncomment the `repo_token` line below if: - # - you want to enable the Branch-Protection check on a *public* repository, or - # - you are installing Scorecard on a *private* repository - # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action?tab=readme-ov-file#authentication-with-fine-grained-pat-optional. - # repo_token: ${{ secrets.SCORECARD_TOKEN }} - - # Public repositories: - # - Publish results to OpenSSF REST API for easy access by consumers - # - Allows the repository to include the Scorecard badge. - # - See https://github.com/ossf/scorecard-action#publishing-results. - # For private repositories: - # - `publish_results` will always be set to `false`, regardless - # of the value entered here. - publish_results: true - - # (Optional) Uncomment file_mode if you have a .gitattributes with files marked export-ignore - # file_mode: git - - # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF - # format to the repository Actions tab. - - name: "Upload artifact" - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: SARIF file - path: results.sarif - retention-days: 5 - - # Upload the results to GitHub's code scanning dashboard (optional). - # Commenting out will disable upload of results to your repo's Code Scanning dashboard - - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: results.sarif diff --git a/.github/workflows/spec_guards.yml b/.github/workflows/spec_guards.yml deleted file mode 100644 index 81b242bbca52ae..00000000000000 --- a/.github/workflows/spec_guards.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: Rubyspec Version Guards Check - -on: - push: - paths: - - '.github/workflows/spec_guards.yml' - - 'spec/**' - - '!spec/*.md' - pull_request: - paths: - - '.github/workflows/spec_guards.yml' - - 'spec/**' - - '!spec/*.md' - merge_group: - -concurrency: - group: ${{ github.workflow }} / ${{ startsWith(github.event_name, 'pull') && github.ref_name || github.sha }} - cancel-in-progress: ${{ startsWith(github.event_name, 'pull') }} - -permissions: - contents: read - -jobs: - rubyspec: - name: Rubyspec - - runs-on: ubuntu-22.04 - - if: >- - ${{!(false - || contains(github.event.head_commit.message, '[DOC]') - || contains(github.event.pull_request.title, '[DOC]') - || contains(github.event.pull_request.labels.*.name, 'Documentation') - || (github.event_name == 'push' && github.event.pull_request.user.login == 'dependabot[bot]') - )}} - - strategy: - matrix: - # Specs from ruby/spec should still run on all supported Ruby versions. - # This also ensures the needed ruby_version_is guards are there, see spec/README.md. - ruby: - - ruby-3.2 - - ruby-3.3 - - ruby-3.4 - - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - uses: ruby/setup-ruby@a9bfc2ecf3dd40734a9418f89a7e9d484c32b990 # v1.248.0 - with: - ruby-version: ${{ matrix.ruby }} - bundler: none - - - run: gem install webrick - - - run: ruby ../mspec/bin/mspec - working-directory: spec/ruby - env: - CHECK_LEAKS: true - - - uses: ./.github/actions/slack - with: - label: ${{ matrix.ruby }} - SLACK_WEBHOOK_URL: ${{ secrets.SIMPLER_ALERTS_URL }} # ruby-lang slack: ruby/simpler-alerts-bot - if: ${{ failure() }} diff --git a/.github/workflows/ubuntu-ibm.yml b/.github/workflows/ubuntu-ibm.yml deleted file mode 100644 index 8bcce6e4a1867e..00000000000000 --- a/.github/workflows/ubuntu-ibm.yml +++ /dev/null @@ -1,183 +0,0 @@ -name: Ubuntu IBM -on: - push: - paths-ignore: - - 'doc/**' - - '**/man/*' - - '**.md' - - '**.rdoc' - - '**/.document' - - '.*.yml' - pull_request: - # Do not use paths-ignore for required status checks - # https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/collaborating-on-repositories-with-code-quality-features/troubleshooting-required-status-checks#handling-skipped-but-required-checks - merge_group: - -concurrency: - group: ${{ github.workflow }} / ${{ startsWith(github.event_name, 'pull') && github.ref_name || github.sha }} - cancel-in-progress: ${{ startsWith(github.event_name, 'pull') }} - -permissions: - contents: read - -jobs: - make: - strategy: - matrix: - test_task: [check] - configure: [''] - os: - - ubuntu-24.04-ppc64le - - ubuntu-24.04-s390x - # Add a x86_64 case to make this CI pass on fork repositories. - - ubuntu-24.04 - # The ppc64le/s390x runners work only in the registered repositories. - # They don't work in forked repositories. - # https://github.com/IBM/actionspz/blob/main/docs/FAQ.md#what-about-forked-repos - upstream: - - ${{ github.repository == 'ruby/ruby' }} - exclude: - - os: ubuntu-24.04-ppc64le - upstream: false - - os: ubuntu-24.04-s390x - upstream: false - - os: ubuntu-24.04 - upstream: true - fail-fast: false - - env: - GITPULLOPTIONS: --no-tags origin ${{ github.ref }} - RUBY_DEBUG: ci - - runs-on: ${{ matrix.os }} - - if: >- - ${{!(false - || contains(github.event.head_commit.message, '[DOC]') - || contains(github.event.pull_request.title, '[DOC]') - || contains(github.event.pull_request.labels.*.name, 'Documentation') - || (github.event_name == 'push' && github.event.pull_request.user.login == 'dependabot[bot]') - )}} - - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - sparse-checkout-cone-mode: false - sparse-checkout: /.github - - - uses: ./.github/actions/setup/ubuntu - - - uses: ruby/setup-ruby@a9bfc2ecf3dd40734a9418f89a7e9d484c32b990 # v1.248.0 - with: - ruby-version: '3.1' - bundler: none - if: ${{ !endsWith(matrix.os, 'ppc64le') && !endsWith(matrix.os, 's390x') }} - - # Avoid possible test failures with the zlib applying the following patch - # on s390x CPU architecture. - # https://github.com/madler/zlib/pull/410 - - name: Disable DFLTCC - run: echo "DFLTCC=0" >> $GITHUB_ENV - working-directory: - if: ${{ endsWith(matrix.os, 's390x') }} - - # A temporary workaround: Set HOME env to pass the step - # ./.github/actions/setup/directories. - # https://github.com/IBM/actionspz/issues/30 - - name: Set HOME env - run: | - echo "HOME: ${HOME}" - echo "HOME=$(ls -d ~)" >> $GITHUB_ENV - working-directory: - if: ${{ endsWith(matrix.os, 'ppc64le') || endsWith(matrix.os, 's390x') }} - - - uses: ./.github/actions/setup/directories - with: - srcdir: src - builddir: build - makeup: true - clean: true - dummy-files: ${{ matrix.test_task == 'check' }} - # Set fetch-depth: 10 so that Launchable can receive commits information. - fetch-depth: 10 - - - name: Run configure - env: - configure: ${{ matrix.configure }} - run: >- - ../src/configure -C --disable-install-doc ${configure:-cppflags=-DRUBY_DEBUG} - - - run: make - - - run: make hello - - - name: runirb - run: | - echo IRB::VERSION | make runirb RUNOPT="-- -f" - - - name: Set test options for skipped tests - run: | - set -x - TESTS="$(echo "${{ matrix.skipped_tests }}" | sed 's| |$$/ -n!/|g;s|^|-n!/|;s|$|$$/|')" - echo "TESTS=${TESTS}" >> $GITHUB_ENV - if: ${{ matrix.test_task == 'check' && matrix.skipped_tests }} - - - name: Set up Launchable - id: launchable - uses: ./.github/actions/launchable/setup - with: - os: ${{ matrix.os }} - test-opts: ${{ matrix.configure }} - launchable-token: ${{ secrets.LAUNCHABLE_TOKEN }} - builddir: build - srcdir: src - continue-on-error: true - timeout-minutes: 3 - - # A temporary workaround: Set the user's primary group to avoid a mismatch - # between the group IDs of "id -g" and C function getpwuid(uid_t uid) - # pw_gid. - # https://github.com/IBM/actionspz/issues/31 - - name: Set user's group id - run: sudo usermod -g "$(id -g)" runner - if: ${{ endsWith(matrix.os, 'ppc64le') || endsWith(matrix.os, 's390x') }} - - - name: make ${{ matrix.test_task }} - run: | - test -n "${LAUNCHABLE_STDOUT}" && exec 1> >(tee "${LAUNCHABLE_STDOUT}") - test -n "${LAUNCHABLE_STDERR}" && exec 2> >(tee "${LAUNCHABLE_STDERR}") - - make -s ${{ matrix.test_task }} \ - ${TESTS:+TESTS="$TESTS"} \ - ${{ !contains(matrix.test_task, 'bundle') && 'RUBYOPT=-w' || '' }} - timeout-minutes: ${{ matrix.timeout || 40 }} - env: - RUBY_TESTOPTS: '-q --tty=no' - TEST_BUNDLED_GEMS_ALLOW_FAILURES: '' - PRECHECK_BUNDLED_GEMS: 'no' - LAUNCHABLE_STDOUT: ${{ steps.launchable.outputs.stdout_report_path }} - LAUNCHABLE_STDERR: ${{ steps.launchable.outputs.stderr_report_path }} - - - name: make skipped tests - run: | - make -s test-all TESTS="${TESTS//-n!\//-n/}" - env: - GNUMAKEFLAGS: '' - RUBY_TESTOPTS: '-v --tty=no' - if: ${{ matrix.test_task == 'check' && matrix.skipped_tests }} - continue-on-error: ${{ matrix.continue-on-skipped_tests || false }} - - - name: test-pc - run: | - DESTDIR=${RUNNER_TEMP-${TMPDIR-/tmp}}/installed - make test-pc "DESTDIR=$DESTDIR" - - - uses: ./.github/actions/slack - with: - label: ${{ matrix.test_task }} ${{ matrix.configure }} - SLACK_WEBHOOK_URL: ${{ secrets.SIMPLER_ALERTS_URL }} # ruby-lang slack: ruby/simpler-alerts-bot - if: ${{ failure() }} - -defaults: - run: - working-directory: build diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml deleted file mode 100644 index 5c8a072a16742c..00000000000000 --- a/.github/workflows/ubuntu.yml +++ /dev/null @@ -1,183 +0,0 @@ -name: Ubuntu -on: - push: - paths-ignore: - - 'doc/**' - - '**/man/*' - - '**.md' - - '**.rdoc' - - '**/.document' - - '.*.yml' - pull_request: - # Do not use paths-ignore for required status checks - # https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/collaborating-on-repositories-with-code-quality-features/troubleshooting-required-status-checks#handling-skipped-but-required-checks - merge_group: - -concurrency: - group: ${{ github.workflow }} / ${{ startsWith(github.event_name, 'pull') && github.ref_name || github.sha }} - cancel-in-progress: ${{ startsWith(github.event_name, 'pull') }} - -permissions: - contents: read - -jobs: - make: - strategy: - matrix: - # We enumerate every job in matrix.include to save build time - include: - - test_task: check - configure: 'cppflags=-DVM_CHECK_MODE' - - test_task: check - arch: i686 - - test_task: check - configure: '--disable-yjit' - - test_task: check - configure: '--enable-shared --enable-load-relative' - - test_task: test-bundler-parallel - timeout: 50 - - test_task: test-bundled-gems - - test_task: check - os: ubuntu-24.04 - capi_check: capi - # ubuntu-24.04-arm jobs don't start on ruby/ruby as of 2025-09-04 - #- test_task: check - # os: ubuntu-24.04-arm - fail-fast: false - - env: - GITPULLOPTIONS: --no-tags origin ${{ github.ref }} - RUBY_DEBUG: ci - - runs-on: ${{ matrix.os || 'ubuntu-22.04' }} - - if: >- - ${{!(false - || contains(github.event.head_commit.message, '[DOC]') - || contains(github.event.pull_request.title, '[DOC]') - || contains(github.event.pull_request.labels.*.name, 'Documentation') - || (github.event_name == 'push' && github.event.pull_request.user.login == 'dependabot[bot]') - )}} - - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - sparse-checkout-cone-mode: false - sparse-checkout: /.github - - - uses: ./.github/actions/setup/ubuntu - with: - arch: ${{ matrix.arch }} - - - uses: ruby/setup-ruby@a9bfc2ecf3dd40734a9418f89a7e9d484c32b990 # v1.248.0 - with: - ruby-version: '3.1' - bundler: none - if: ${{ !endsWith(matrix.os, 'arm') }} - - - uses: ./.github/actions/setup/directories - with: - srcdir: src - builddir: build - makeup: true - clean: true - dummy-files: ${{ matrix.test_task == 'check' }} - # Set fetch-depth: 10 so that Launchable can receive commits information. - fetch-depth: 10 - - - name: Run configure - env: - arch: ${{ matrix.arch }} - configure: ${{ matrix.configure }} - run: >- - $SETARCH ../src/configure -C --disable-install-doc ${configure:-cppflags=-DRUBY_DEBUG} - ${arch:+--target=$arch-$OSTYPE --host=$arch-$OSTYPE} - - - run: $SETARCH make prepare-gems - if: ${{ matrix.test_task == 'test-bundled-gems' }} - - - run: $SETARCH make - - - run: $SETARCH make hello - - - name: runirb - run: | - echo IRB::VERSION | $SETARCH make runirb RUNOPT="-- -f" - - - name: Set test options for skipped tests - run: | - set -x - TESTS="$(echo "${{ matrix.skipped_tests }}" | sed 's| |$$/ -n!/|g;s|^|-n!/|;s|$|$$/|')" - echo "TESTS=${TESTS}" >> $GITHUB_ENV - if: ${{ matrix.test_task == 'check' && matrix.skipped_tests }} - - - name: Set up Launchable - id: launchable - uses: ./.github/actions/launchable/setup - with: - os: ${{ matrix.os || 'ubuntu-22.04' }} - test-opts: ${{ matrix.configure }} - launchable-token: ${{ secrets.LAUNCHABLE_TOKEN }} - builddir: build - srcdir: src - continue-on-error: true - timeout-minutes: 3 - - - name: make ${{ matrix.test_task }} - run: | - test -n "${LAUNCHABLE_STDOUT}" && exec 1> >(tee "${LAUNCHABLE_STDOUT}") - test -n "${LAUNCHABLE_STDERR}" && exec 2> >(tee "${LAUNCHABLE_STDERR}") - - $SETARCH make -s ${{ matrix.test_task }} \ - ${TESTS:+TESTS="$TESTS"} \ - ${{ !contains(matrix.test_task, 'bundle') && 'RUBYOPT=-w' || '' }} - timeout-minutes: ${{ matrix.timeout || 40 }} - env: - RUBY_TESTOPTS: '-q --tty=no' - TEST_BUNDLED_GEMS_ALLOW_FAILURES: '' - PRECHECK_BUNDLED_GEMS: 'no' - LAUNCHABLE_STDOUT: ${{ steps.launchable.outputs.stdout_report_path }} - LAUNCHABLE_STDERR: ${{ steps.launchable.outputs.stderr_report_path }} - - - name: make skipped tests - run: | - $SETARCH make -s test-all TESTS="${TESTS//-n!\//-n/}" - env: - GNUMAKEFLAGS: '' - RUBY_TESTOPTS: '-v --tty=no' - if: ${{ matrix.test_task == 'check' && matrix.skipped_tests }} - continue-on-error: ${{ matrix.continue-on-skipped_tests || false }} - - - name: test-pc - run: | - DESTDIR=${RUNNER_TEMP-${TMPDIR-/tmp}}/installed - $SETARCH make test-pc "DESTDIR=$DESTDIR" - - - name: CAPI extensions - uses: ./.github/actions/capiext - with: - builddir: build - make: '$SETARCH make' - env: - RUBY_TESTOPTS: '-v --tty=no' - if: ${{ matrix.capi_check }} - - - uses: ./.github/actions/slack - with: - label: ${{ matrix.test_task }} ${{ matrix.configure }}${{ matrix.arch }} - SLACK_WEBHOOK_URL: ${{ secrets.SIMPLER_ALERTS_URL }} # ruby-lang slack: ruby/simpler-alerts-bot - if: ${{ failure() }} - - result: - if: ${{ always() }} - name: ${{ github.workflow }} result - runs-on: ubuntu-latest - needs: [make] - steps: - - run: exit 1 - working-directory: - if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }} - -defaults: - run: - working-directory: build diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml deleted file mode 100644 index 011351161aea07..00000000000000 --- a/.github/workflows/wasm.yml +++ /dev/null @@ -1,179 +0,0 @@ -name: WebAssembly -on: - push: - paths-ignore: - - 'doc/**' - - '**/man/*' - - '**.md' - - '**.rdoc' - - '**/.document' - - '.*.yml' - pull_request: - paths-ignore: - - 'doc/**' - - '**/man/*' - - '**.md' - - '**.rdoc' - - '**/.document' - - '.*.yml' - merge_group: - -concurrency: - group: ${{ github.workflow }} / ${{ startsWith(github.event_name, 'pull') && github.ref_name || github.sha }} - cancel-in-progress: ${{ startsWith(github.event_name, 'pull') }} - -permissions: # added using https://github.com/step-security/secure-workflows - contents: read - -jobs: - make: - strategy: - matrix: - entry: -# # wasmtime can't compile non-optimized Asyncified binary due to locals explosion -# - { name: O0-debuginfo, optflags: '-O0', debugflags: '-g', wasmoptflags: '-O1' } -# - { name: O1, optflags: '-O1', debugflags: '' , wasmoptflags: '-O1' } - - { name: O2, optflags: '-O2', debugflags: '', wasmoptflags: '-O2' } -# - { name: O3, optflags: '-O3', debugflags: '' , wasmoptflags: '-O3' } -# # -O4 is equivalent to -O3 in clang, but it's different in wasm-opt -# - { name: O4, optflags: '-O3', debugflags: '' , wasmoptflags: '-O4' } -# - { name: Oz, optflags: '-Oz', debugflags: '' , wasmoptflags: '-Oz' } - fail-fast: false - - env: - RUBY_TESTOPTS: '-q --tty=no' - GITPULLOPTIONS: --no-tags origin ${{ github.ref }} - WASI_SDK_VERSION_MAJOR: 25 - WASI_SDK_VERSION_MINOR: 0 - BINARYEN_VERSION: 113 - WASMTIME_VERSION: v15.0.0 - - runs-on: ubuntu-22.04 - - if: >- - ${{!(false - || contains(github.event.head_commit.message, '[DOC]') - || contains(github.event.pull_request.title, '[DOC]') - || contains(github.event.pull_request.labels.*.name, 'Documentation') - || (github.event_name == 'push' && github.event.pull_request.user.login == 'dependabot[bot]') - )}} - - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - sparse-checkout-cone-mode: false - sparse-checkout: /.github - - - uses: ./.github/actions/setup/directories - with: - srcdir: src - builddir: build - makeup: true - - - name: Install libraries - run: | - set -ex - sudo apt-get update -q || : - sudo apt-get install --no-install-recommends -q -y ruby make autoconf git wget - - wasi_sdk_deb="wasi-sdk-${WASI_SDK_VERSION_MAJOR}.${WASI_SDK_VERSION_MINOR}-x86_64-linux.deb" - wget "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION_MAJOR}/${wasi_sdk_deb}" - sudo dpkg -i "$wasi_sdk_deb" - rm -f "$wasi_sdk_deb" - - mkdir build-sdk - pushd build-sdk - - wasmtime_url="https://github.com/bytecodealliance/wasmtime/releases/download/${WASMTIME_VERSION}/wasmtime-${WASMTIME_VERSION}-x86_64-linux.tar.xz" - wget -O - "$wasmtime_url" | tar xJf - - sudo ln -fs "$PWD/wasmtime-${WASMTIME_VERSION}-x86_64-linux/wasmtime" /usr/local/bin/wasmtime - - binaryen_tarball="binaryen-version_${BINARYEN_VERSION}-x86_64-linux.tar.gz" - binaryen_url="https://github.com/WebAssembly/binaryen/releases/download/version_${BINARYEN_VERSION}/${binaryen_tarball}" - wget -O - "$binaryen_url" | tar xfz - - sudo ln -fs "$PWD/binaryen-version_${BINARYEN_VERSION}/bin/wasm-opt" /usr/local/bin/wasm-opt - working-directory: src - - - name: Set ENV - run: | - echo "WASI_SDK_PATH=/opt/wasi-sdk" >> $GITHUB_ENV - - - uses: ruby/setup-ruby@a9bfc2ecf3dd40734a9418f89a7e9d484c32b990 # v1.248.0 - with: - ruby-version: '3.1' - bundler: none - - - name: Build baseruby - run: | - set -ex - mkdir ../baseruby - pushd ../baseruby - ../src/configure --prefix=$PWD/install - make - make install - - - name: Download config.guess with wasi version - run: | - rm tool/config.guess tool/config.sub - ruby tool/downloader.rb -d tool -e gnu config.guess config.sub - working-directory: src - - - name: Run configure - run: | - ../src/configure \ - --host wasm32-unknown-wasi \ - --with-baseruby=$PWD/../baseruby/install/bin/ruby \ - --with-static-linked-ext \ - --with-ext=cgi/escape,continuation,coverage,date,digest/bubblebabble,digest,digest/md5,digest/rmd160,digest/sha1,digest/sha2,etc,fcntl,json,json/generator,json/parser,objspace,pathname,rbconfig/sizeof,ripper,stringio,strscan,monitor \ - LDFLAGS=" \ - -Xlinker --stack-first \ - -Xlinker -z -Xlinker stack-size=16777216 \ - " \ - optflags="${{ matrix.entry.optflags }}" \ - debugflags="${{ matrix.entry.debugflags }}" \ - wasmoptflags="${{ matrix.entry.wasmoptflags }} ${{ matrix.entry.debugflags }}" - - # miniruby may not be built when cross-compling - - run: make mini ruby - - - run: make install DESTDIR=$PWD/../install - - run: tar cfz ../install.tar.gz -C ../install . - - - name: Upload artifacts - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: ruby-wasm-install - path: ${{ github.workspace }}/install.tar.gz - - name: Show Playground URL to try the build - run: | - echo "Try on Playground: https://ruby.github.io/play-ruby?run=$GITHUB_RUN_ID" >> $GITHUB_STEP_SUMMARY - - - name: Run basictest - run: wasmtime run ./../build/miniruby --mapdir /::./ -- basictest/test.rb - working-directory: src - - - name: Run bootstraptest (no thread) - run: | - NO_THREAD_TESTS="$(grep -L Thread -R ./bootstraptest | awk -F/ '{ print $NF }' | uniq | sed -n 's/test_\(.*\).rb/\1/p' | paste -s -d, -)" - ruby ./bootstraptest/runner.rb --ruby="$(which wasmtime) run $PWD/../build/ruby --mapdir /::./ -- " --verbose "--sets=$NO_THREAD_TESTS" - working-directory: src - - - uses: ./.github/actions/slack - with: - label: ${{ matrix.entry.name }} - SLACK_WEBHOOK_URL: ${{ secrets.SIMPLER_ALERTS_URL }} # ruby-lang slack: ruby/simpler-alerts-bot - if: ${{ failure() }} - - # Workaround for https://github.com/orgs/community/discussions/25220 - - name: Save Pull Request number - if: ${{ github.event_name == 'pull_request' }} - run: echo "${{ github.event.pull_request.number }}" >> ${{ github.workspace }}/github-pr-info.txt - - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - if: ${{ github.event_name == 'pull_request' }} - with: - name: github-pr-info - path: ${{ github.workspace }}/github-pr-info.txt - -defaults: - run: - working-directory: build diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml deleted file mode 100644 index 1083de63b33567..00000000000000 --- a/.github/workflows/windows.yml +++ /dev/null @@ -1,220 +0,0 @@ -name: Windows -on: - push: - paths-ignore: - - 'doc/**' - - '**/man/*' - - '**.md' - - '**.rdoc' - - '**/.document' - - '.*.yml' - pull_request: - # Do not use paths-ignore for required status checks - # https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/collaborating-on-repositories-with-code-quality-features/troubleshooting-required-status-checks#handling-skipped-but-required-checks - merge_group: - -concurrency: - group: ${{ github.workflow }} / ${{ startsWith(github.event_name, 'pull') && github.ref_name || github.sha }} - cancel-in-progress: ${{ startsWith(github.event_name, 'pull') }} - -permissions: - contents: read - -jobs: - make: - strategy: - matrix: - include: - - os: 2022 - vc: 2022 - test_task: check - - os: 2025 - vc: 2022 - test_task: check - - os: 11-arm - test_task: 'btest test-basic test-tool' # check and test-spec are broken yet. - target: arm64 - - os: 2025 - vc: 2022 - test_task: test-bundled-gems - fail-fast: false - - runs-on: windows-${{ matrix.os }} - - if: >- - ${{!(false - || contains(github.event.head_commit.message, '[DOC]') - || contains(github.event.pull_request.title, '[DOC]') - || contains(github.event.pull_request.labels.*.name, 'Documentation') - || (github.event_name == 'push' && github.event.pull_request.user.login == 'dependabot[bot]') - )}} - - name: Windows ${{ matrix.os }}/Visual C++ ${{ matrix.vc }} (${{ matrix.test_task }}) - - env: - GITPULLOPTIONS: --no-tags origin ${{ github.ref }} - VCPKG_DEFAULT_TRIPLET: ${{ matrix.target || 'x64' }}-windows - - steps: - - run: md build - working-directory: - - - uses: ruby/setup-ruby@a9bfc2ecf3dd40734a9418f89a7e9d484c32b990 # v1.248.0 - with: - # windows-11-arm has only 3.4.1, 3.4.2, 3.4.3, head - ruby-version: ${{ !endsWith(matrix.os, 'arm') && '3.1' || '3.4' }} - bundler: none - windows-toolchain: none - - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - sparse-checkout-cone-mode: false - sparse-checkout: /.github - - - uses: ./.github/actions/setup/directories - with: - srcdir: src - builddir: build - - - name: Install tools with scoop - run: | - Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser - iwr -useb get.scoop.sh | iex - Join-Path (Resolve-Path ~).Path "scoop\shims" >> $Env:GITHUB_PATH - scoop install vcpkg uutils-coreutils cmake@3.31.6 - shell: pwsh - - - name: Restore vcpkg artifact - id: restore-vcpkg - uses: actions/cache/restore@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 - with: - path: src\vcpkg_installed - key: windows-${{ matrix.os }}-vcpkg-${{ hashFiles('src/vcpkg.json') }} - - - name: Install libraries with vcpkg - run: | - vcpkg install --vcpkg-root=%USERPROFILE%\scoop\apps\vcpkg\current - working-directory: src - if: ${{ ! steps.restore-vcpkg.outputs.cache-hit }} - - - name: Save vcpkg artifact - uses: actions/cache/save@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 - with: - path: src\vcpkg_installed - key: windows-${{ matrix.os }}-vcpkg-${{ hashFiles('src/vcpkg.json') }} - if: ${{ ! steps.restore-vcpkg.outputs.cache-hit && (github.ref_name == 'master' || startsWith(github.ref_name, 'ruby_')) }} - - - name: setup env - # Available Ruby versions: https://github.com/actions/runner-images/blob/main/images/windows/Windows2019-Readme.md#ruby - # %TEMP% is inconsistent with %TMP% and test-all expects they are consistent. - # https://github.com/actions/virtual-environments/issues/712#issuecomment-613004302 - run: | - ::- Set up VC ${{ matrix.vc }} - set | uutils sort > old.env - call ..\src\win32\vssetup.cmd ^ - -arch=${{ matrix.target || 'amd64' }} ^ - ${{ matrix.vcvars && '-vcvars_ver=' || '' }}${{ matrix.vcvars }} - nmake -f nul - set TMP=%RUNNER_TEMP% - set TEMP=%RUNNER_TEMP% - set MAKEFLAGS=l - set /a TEST_JOBS=(15 * %NUMBER_OF_PROCESSORS% / 10) > nul - set RUBY_OPT_DIR=%GITHUB_WORKSPACE:\=/%/src/vcpkg_installed/%VCPKG_DEFAULT_TRIPLET% - set | uutils sort > new.env - uutils comm -13 old.env new.env >> %GITHUB_ENV% - del *.env - - - name: baseruby version - run: ruby -v - - - name: compiler version - run: cl - - - name: volume info - run: Get-Volume - shell: pwsh - - # TODO: We should use `../src` instead of `D:/a/ruby/ruby/src` - - name: Configure - run: >- - ../src/win32/configure.bat --disable-install-doc - --with-opt-dir=%RUBY_OPT_DIR% - --with-gmp - - - run: nmake prepare-vcpkg - - - run: nmake incs - - - run: nmake extract-extlibs - - # On all other platforms, test-spec depending on extract-gems (in common.mk) is enough. - # But not for this Visual Studio workflow. So here we extract gems before building. - - run: nmake extract-gems - - # windows-11-arm runner cannot run `ruby tool/file2lastrev.rb --revision.h --output=revision.h` - - name: make revision.h - run: | - if not exist revision.h ( - for /f "tokens=1-3" %%I in ('git log -1 "--date=format-local:%%F %%T" "--format=%%H %%cd" @') do ( - set rev=%%I - set dt=%%J - set tm=%%K - ) - call set yy=%%dt:~0,4%% - call set /a mm=100%%dt:~5,2%% %%%% 100 - call set /a dd=100%%dt:~8,2%% %%%% 100 - call set branch=%%GITHUB_REF:refs/heads/=%% - ( - call echo #define RUBY_REVISION "%%rev:~,10%%" - call echo #define RUBY_FULL_REVISION "%%rev%%" - call echo #define RUBY_BRANCH_NAME "%%branch%%" - call echo #define RUBY_RELEASE_DATETIME "%%dt%%T%%tm%%" - call echo #define RUBY_RELEASE_YEAR %%yy%% - call echo #define RUBY_RELEASE_MONTH %%mm%% - call echo #define RUBY_RELEASE_DAY %%dd%% - ) > revision.h - copy /y NUL .revision.time - ) - type revision.h - env: - TZ: UTC - - - run: nmake - - - name: Set up Launchable - uses: ./.github/actions/launchable/setup - with: - os: windows-${{ matrix.os }} - launchable-token: ${{ secrets.LAUNCHABLE_TOKEN }} - builddir: build - srcdir: src - test-task: ${{ matrix.test_task || 'check' }} - continue-on-error: true - if: ${{ matrix.test_task != 'test-bundled-gems' }} - timeout-minutes: 3 - - - run: nmake ${{ matrix.test_task || 'check' }} - env: - RUBY_TESTOPTS: -j${{ env.TEST_JOBS || 4 }} - timeout-minutes: 70 - - - uses: ./.github/actions/slack - with: - label: Windows ${{ matrix.os }} / VC ${{ matrix.vc }} / ${{ matrix.test_task || 'check' }} - SLACK_WEBHOOK_URL: ${{ secrets.SIMPLER_ALERTS_URL }} # ruby-lang slack: ruby/simpler-alerts-bot - if: ${{ failure() }} - - result: - if: ${{ always() }} - name: ${{ github.workflow }} result - runs-on: windows-latest - needs: [make] - steps: - - run: exit 1 - working-directory: - if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }} - -defaults: - run: - working-directory: build - shell: cmd diff --git a/.github/workflows/wsl.yml b/.github/workflows/wsl.yml deleted file mode 100644 index 640f18ce42e876..00000000000000 --- a/.github/workflows/wsl.yml +++ /dev/null @@ -1,70 +0,0 @@ -name: Ubuntu on WSL - -on: - push: - paths-ignore: - - 'doc/**' - - '**/man/*' - - '**.md' - - '**.rdoc' - - '**/.document' - - '.*.yml' - pull_request: - # Do not use paths-ignore for required status checks - # https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/collaborating-on-repositories-with-code-quality-features/troubleshooting-required-status-checks#handling-skipped-but-required-checks - merge_group: - -jobs: - wsl: - runs-on: windows-2025 - - if: >- - ${{!(false - || contains(github.event.head_commit.message, '[DOC]') - || contains(github.event.pull_request.title, '[DOC]') - || contains(github.event.pull_request.labels.*.name, 'Documentation') - || (github.event_name == 'push' && github.event.pull_request.user.login == 'dependabot[bot]') - )}} - - steps: - - name: Install or update WSL - uses: Ubuntu/WSL/.github/actions/wsl-install@main - with: - distro: Ubuntu-24.04 - - - name: Install dependencies - uses: Ubuntu/WSL/.github/actions/wsl-bash@main - with: - distro: Ubuntu-24.04 - working-dir: /tmp/github/ - exec: | - DEBIAN_FRONTEND=noninteractive sudo apt update - DEBIAN_FRONTEND=noninteractive sudo apt install -y ruby build-essential autoconf libssl-dev libyaml-dev zlib1g-dev libgmp-dev libffi-dev - - - name: Check out the repository - uses: Ubuntu/WSL/.github/actions/wsl-checkout@main - with: - distro: Ubuntu-24.04 - working-dir: /tmp/github/ - submodules: true - - - name: Build - uses: Ubuntu/WSL/.github/actions/wsl-bash@main - with: - distro: Ubuntu-24.04 - working-dir: /tmp/github/ - exec: | - ./autogen.sh - ./configure --disable-install-doc - make ruby -j4 - make extract-gems - make -j4 - - - name: Test - uses: Ubuntu/WSL/.github/actions/wsl-bash@main - with: - distro: Ubuntu-24.04 - working-dir: /tmp/github/ - exec: | - ./ruby -v - # make check TESTS="-j4" MSPECOPT="-j" diff --git a/.github/workflows/yjit-macos.yml b/.github/workflows/yjit-macos.yml index 53ffcbe217069e..b44187b0fb1cd8 100644 --- a/.github/workflows/yjit-macos.yml +++ b/.github/workflows/yjit-macos.yml @@ -115,8 +115,10 @@ jobs: ruby -ne 'raise "Disassembly seems broken in dev build (output has too few lines)" unless $_.to_i > 10' if: ${{ contains(matrix.configure, 'jit=dev') }} - - name: Enable YJIT through ENV - run: echo "RUBY_YJIT_ENABLE=1" >> $GITHUB_ENV + - name: Set ENV for YJIT + run: | + echo "RUBY_YJIT_ENABLE=1" >> $GITHUB_ENV + echo "RUBY_CRASH_REPORT=$(pwd)/rb_crash_%p.txt" >> $GITHUB_ENV - name: Set test options for skipped tests run: | @@ -166,6 +168,10 @@ jobs: if: ${{ matrix.test_task == 'check' && matrix.skipped_tests }} continue-on-error: ${{ matrix.continue-on-skipped_tests || false }} + - if: ${{ failure() }} + continue-on-error: true + run: tail --verbose --lines=+1 rb_crash_*.txt + - uses: ./.github/actions/slack with: label: ${{ matrix.test_task }} ${{ matrix.configure }} ${{ matrix.yjit_opts }} diff --git a/.github/workflows/yjit-ubuntu.yml b/.github/workflows/yjit-ubuntu.yml index 1961d1262ab3a0..adf7c9404cc119 100644 --- a/.github/workflows/yjit-ubuntu.yml +++ b/.github/workflows/yjit-ubuntu.yml @@ -145,6 +145,12 @@ jobs: if: ${{ matrix.rust_version }} run: rustup install ${{ matrix.rust_version }} --profile minimal + - name: Remove cargo + # Since this tests a `rustc` build for release, remove `cargo` to ensure + # that only `rustc` is used. + if: ${{ contains(matrix.configure, 'rustc') }} + run: sudo rm $(which -a cargo | uniq) + - name: Run configure run: ../src/configure -C --disable-install-doc --prefix=$(pwd)/install ${{ matrix.configure }} @@ -163,7 +169,9 @@ jobs: if: ${{ contains(matrix.configure, 'jit=dev') }} - name: Enable YJIT through ENV - run: echo "RUBY_YJIT_ENABLE=1" >> $GITHUB_ENV + run: | + echo "RUBY_YJIT_ENABLE=1" >> $GITHUB_ENV + echo "RUBY_CRASH_REPORT=$(pwd)/rb_crash_%p.txt" >> $GITHUB_ENV # Check that the binary was built with YJIT - name: Check YJIT enabled @@ -202,6 +210,11 @@ jobs: LAUNCHABLE_STDERR: ${{ steps.launchable.outputs.stderr_report_path }} continue-on-error: ${{ matrix.continue-on-test_task || false }} + - name: Dump crash logs + if: ${{ failure() }} + continue-on-error: true + run: tail --verbose --lines=+1 rb_crash_*.txt + - uses: ./.github/actions/slack with: label: ${{ matrix.test_task }} ${{ matrix.configure }} diff --git a/Cargo.lock b/Cargo.lock index 6312cb46a9b2ca..9a4b2ebbbaf2ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -62,11 +62,7 @@ dependencies = [ [[package]] name = "jit" -version = "0.0.0" -dependencies = [ - "yjit", - "zjit", -] +version = "0.1.0" [[package]] name = "lazy_static" @@ -86,6 +82,14 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "ruby" +version = "0.0.0" +dependencies = [ + "yjit", + "zjit", +] + [[package]] name = "shlex" version = "1.3.0" @@ -176,6 +180,7 @@ name = "yjit" version = "0.1.0" dependencies = [ "capstone", + "jit", ] [[package]] @@ -184,4 +189,5 @@ version = "0.0.1" dependencies = [ "capstone", "insta", + "jit", ] diff --git a/Cargo.toml b/Cargo.toml index 3f373fdace9cbf..ec2ce880ca4c48 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,10 +3,10 @@ # TODO(alan) notes about rust version requirements. Undecided yet. [workspace] -members = ["zjit", "yjit"] +members = ["zjit", "yjit", "jit"] [package] -name = "jit" +name = "ruby" version = "0.0.0" edition = "2024" rust-version = "1.85.0" @@ -18,7 +18,7 @@ zjit = { path = "zjit", optional = true } [lib] crate-type = ["staticlib"] -path = "jit.rs" +path = "ruby.rs" [features] disasm = ["yjit?/disasm", "zjit?/disasm"] diff --git a/NEWS.md b/NEWS.md index a4f3c723df79fd..7534539a21d2c9 100644 --- a/NEWS.md +++ b/NEWS.md @@ -10,6 +10,25 @@ Note that each entry is kept to a minimum, see links for details. * `*nil` no longer calls `nil.to_a`, similar to how `**nil` does not call `nil.to_hash`. [[Feature #21047]] +* Logical binary operators (`||`, `&&`, `and` and `or`) at the + beginning of a line continue the previous line, like fluent dot. + The following two code are equal: + + ```ruby + if condition1 + && condition2 + ... + end + ``` + + ```ruby + if condition1 && condition2 + ... + end + ``` + + [[Feature #20925]] + ## Core classes updates Note: We're only listing outstanding class updates. @@ -44,7 +63,7 @@ Note: We're only listing outstanding class updates. * IO - * `IO.select` accepts +Float::INFINITY+ as a timeout argument. + * `IO.select` accepts `Float::INFINITY` as a timeout argument. [[Feature #20610]] * Math @@ -98,9 +117,9 @@ Note: We're only listing outstanding class updates. * `Ractor#close_incoming` and `Ractor#close_outgoing` were removed. -* Set +* `Set` - * Set is now a core class, instead of an autoloaded stdlib class. + * `Set` is now a core class, instead of an autoloaded stdlib class. [[Feature #21216]] * String @@ -167,7 +186,7 @@ The following default gems are updated. * io-wait 0.3.2 * json 2.13.2 * optparse 0.7.0.dev.2 -* prism 1.4.0 +* prism 1.5.1 * psych 5.2.6 * resolv 0.6.2 * stringio 3.1.8.dev @@ -285,6 +304,7 @@ A lot of work has gone into making Ractors more stable, performant, and usable. [Feature #19908]: https://bugs.ruby-lang.org/issues/19908 [Feature #20610]: https://bugs.ruby-lang.org/issues/20610 [Feature #20724]: https://bugs.ruby-lang.org/issues/20724 +[Feature #20925]: https://bugs.ruby-lang.org/issues/20925 [Feature #21047]: https://bugs.ruby-lang.org/issues/21047 [Bug #21049]: https://bugs.ruby-lang.org/issues/21049 [Feature #21166]: https://bugs.ruby-lang.org/issues/21166 diff --git a/addr2line.c b/addr2line.c index 745364cc0f78e8..19a6a425c1e138 100644 --- a/addr2line.c +++ b/addr2line.c @@ -637,12 +637,13 @@ follow_debuglink_build_id(const char *build_id, size_t build_id_size, int num_tr obj_info_t **objp, line_info_t *lines, int offset, FILE *errout) { static const char global_debug_dir[] = "/usr/lib/debug/.build-id/"; + static const char debug_suffix[] = ".debug"; const size_t global_debug_dir_len = sizeof(global_debug_dir) - 1; char *p; obj_info_t *o1 = *objp, *o2; size_t i; - if (PATH_MAX < global_debug_dir_len + 1 + build_id_size * 2 + 6) return; + if (PATH_MAX < global_debug_dir_len + build_id_size * 2 + sizeof(debug_suffix)) return; memcpy(binary_filename, global_debug_dir, global_debug_dir_len); p = binary_filename + global_debug_dir_len; @@ -653,7 +654,7 @@ follow_debuglink_build_id(const char *build_id, size_t build_id_size, int num_tr *p++ = tbl[n % 16]; if (i == 0) *p++ = '/'; } - strcpy(p, ".debug"); + memcpy(p, debug_suffix, sizeof(debug_suffix)); append_obj(objp); o2 = *objp; diff --git a/common.mk b/common.mk index 5cc7886796243f..ef7eb6ab58ca1e 100644 --- a/common.mk +++ b/common.mk @@ -266,6 +266,7 @@ MAKE_LINK = $(MINIRUBY) -rfileutils -e "include FileUtils::Verbose" \ YJIT_RUSTC_ARGS = --crate-name=yjit \ --crate-type=staticlib \ --edition=2021 \ + --cfg 'feature="stats_allocator"' \ -g \ -C lto=thin \ -C opt-level=3 \ @@ -276,6 +277,7 @@ YJIT_RUSTC_ARGS = --crate-name=yjit \ ZJIT_RUSTC_ARGS = --crate-name=zjit \ --crate-type=staticlib \ --edition=2024 \ + --cfg 'feature="stats_allocator"' \ -g \ -C lto=thin \ -C opt-level=3 \ diff --git a/configure.ac b/configure.ac index 47f8b79b038fbe..8cb30429dcddca 100644 --- a/configure.ac +++ b/configure.ac @@ -4019,9 +4019,9 @@ AS_IF([test x"$JIT_CARGO_SUPPORT" != "xno" -o \( x"$YJIT_SUPPORT" != "xno" -a x" ]) CARGO_BUILD_ARGS="--profile ${JIT_CARGO_SUPPORT} --features ${rb_cargo_features}" AS_IF([test "${JIT_CARGO_SUPPORT}" = "dev"], [ - RUST_LIB="target/debug/libjit.a" + RUST_LIB="target/debug/libruby.a" ], [ - RUST_LIB="target/${JIT_CARGO_SUPPORT}/libjit.a" + RUST_LIB="target/${JIT_CARGO_SUPPORT}/libruby.a" ]) ]) diff --git a/defs/jit.mk b/defs/jit.mk index 28d8f2da3a87f7..a537d803002856 100644 --- a/defs/jit.mk +++ b/defs/jit.mk @@ -17,7 +17,7 @@ CARGO_VERBOSE = $(CARGO_VERBOSE_$(V)) # ld: warning: object file (target/debug/libjit.a()) was built for # newer macOS version (15.2) than being linked (15.0) # This limits us to an older set of macOS API in the rust code, but we don't use any. -$(RUST_LIB): $(srcdir)/jit.rs +$(RUST_LIB): $(srcdir)/ruby.rs $(Q)if [ '$(ZJIT_SUPPORT)' != no -a '$(YJIT_SUPPORT)' != no ]; then \ echo 'building YJIT and ZJIT ($(JIT_CARGO_SUPPORT:yes=release) mode)'; \ elif [ '$(ZJIT_SUPPORT)' != no ]; then \ diff --git a/dir.c b/dir.c index b934f2795c90e0..25ed59c668fab6 100644 --- a/dir.c +++ b/dir.c @@ -1480,7 +1480,7 @@ rb_dir_getwd_ospath(void) VALUE cwd; VALUE path_guard; - path_guard = rb_imemo_tmpbuf_auto_free_pointer(); + path_guard = rb_imemo_tmpbuf_new(); path = ruby_getcwd(); rb_imemo_tmpbuf_set_ptr(path_guard, path); #ifdef __APPLE__ diff --git a/ext/date/date_core.c b/ext/date/date_core.c index dbee067f6baa8e..457838e41f8d55 100644 --- a/ext/date/date_core.c +++ b/ext/date/date_core.c @@ -3974,7 +3974,7 @@ rt_complete_frags(VALUE klass, VALUE hash) rb_gc_register_mark_object(tab); } - k = Qnil; + k = a = Qnil; { long i, eno = 0; diff --git a/ext/extmk.rb b/ext/extmk.rb index 39cbce1bc9fe68..888eba6c676d41 100755 --- a/ext/extmk.rb +++ b/ext/extmk.rb @@ -43,6 +43,7 @@ def self.target_rbconfig $topdir = "." $top_srcdir = srcdir +$extmk = true inplace = File.identical?($top_srcdir, $topdir) $" << "mkmf.rb" diff --git a/ext/openssl/ossl_pkey.c b/ext/openssl/ossl_pkey.c index e88074ddf2582c..ee6a7a988f59f0 100644 --- a/ext/openssl/ossl_pkey.c +++ b/ext/openssl/ossl_pkey.c @@ -83,33 +83,44 @@ ossl_pkey_wrap(EVP_PKEY *pkey) # include static EVP_PKEY * -ossl_pkey_read(BIO *bio, const char *input_type, int selection, VALUE pass) +ossl_pkey_read(BIO *bio, const char *input_type, int selection, VALUE pass, + int *retryable) { void *ppass = (void *)pass; OSSL_DECODER_CTX *dctx; EVP_PKEY *pkey = NULL; int pos = 0, pos2; + *retryable = 0; dctx = OSSL_DECODER_CTX_new_for_pkey(&pkey, input_type, NULL, NULL, selection, NULL, NULL); if (!dctx) goto out; - if (OSSL_DECODER_CTX_set_pem_password_cb(dctx, ossl_pem_passwd_cb, + if (selection == EVP_PKEY_KEYPAIR && + OSSL_DECODER_CTX_set_pem_password_cb(dctx, ossl_pem_passwd_cb, ppass) != 1) goto out; while (1) { if (OSSL_DECODER_from_bio(dctx, bio) == 1) - goto out; - if (BIO_eof(bio)) break; + // Error queue may not be populated in OpenSSL < 3.0.11 and < 3.1.3 + // https://github.com/openssl/openssl/pull/21603 + unsigned long err = ERR_peek_error(); + if (err && ERR_GET_REASON(err) != ERR_R_UNSUPPORTED) + break; + if (BIO_eof(bio) == 1) { + *retryable = 1; + break; + } pos2 = BIO_tell(bio); - if (pos2 < 0 || pos2 <= pos) + if (pos2 < 0 || pos2 <= pos) { + *retryable = 1; break; + } ossl_clear_error(); pos = pos2; } out: - OSSL_BIO_reset(bio); OSSL_DECODER_CTX_free(dctx); return pkey; } @@ -117,7 +128,6 @@ ossl_pkey_read(BIO *bio, const char *input_type, int selection, VALUE pass) EVP_PKEY * ossl_pkey_read_generic(BIO *bio, VALUE pass) { - EVP_PKEY *pkey = NULL; /* First check DER, then check PEM. */ const char *input_types[] = {"DER", "PEM"}; int input_type_num = (int)(sizeof(input_types) / sizeof(char *)); @@ -166,18 +176,22 @@ ossl_pkey_read_generic(BIO *bio, VALUE pass) EVP_PKEY_PUBLIC_KEY }; int selection_num = (int)(sizeof(selections) / sizeof(int)); - int i, j; - for (i = 0; i < input_type_num; i++) { - for (j = 0; j < selection_num; j++) { - pkey = ossl_pkey_read(bio, input_types[i], selections[j], pass); - if (pkey) { - goto out; + for (int i = 0; i < input_type_num; i++) { + for (int j = 0; j < selection_num; j++) { + if (i || j) { + ossl_clear_error(); + BIO_reset(bio); } + + int retryable; + EVP_PKEY *pkey = ossl_pkey_read(bio, input_types[i], selections[j], + pass, &retryable); + if (pkey || !retryable) + return pkey; } } - out: - return pkey; + return NULL; } #else EVP_PKEY * diff --git a/ext/socket/getaddrinfo.c b/ext/socket/getaddrinfo.c index bf0d90129f7601..5b824996552e5f 100644 --- a/ext/socket/getaddrinfo.c +++ b/ext/socket/getaddrinfo.c @@ -171,9 +171,7 @@ static const char *const ai_errlist[] = { #define GET_CANONNAME(ai, str) \ if (pai->ai_flags & AI_CANONNAME) {\ - if (((ai)->ai_canonname = (char *)malloc(strlen(str) + 1)) != NULL) {\ - strcpy((ai)->ai_canonname, (str));\ - } else {\ + if (((ai)->ai_canonname = strdup(str)) == NULL) {\ error = EAI_MEMORY;\ goto free;\ }\ diff --git a/ext/socket/getnameinfo.c b/ext/socket/getnameinfo.c index ae5284fab6fc59..ffda7f98c73f6d 100644 --- a/ext/socket/getnameinfo.c +++ b/ext/socket/getnameinfo.c @@ -158,16 +158,14 @@ getnameinfo(const struct sockaddr *sa, socklen_t salen, char *host, socklen_t ho /* what we should do? */ } else if (flags & NI_NUMERICSERV) { snprintf(numserv, sizeof(numserv), "%d", ntohs(port)); - if (strlen(numserv) + 1 > servlen) + if (strlcpy(serv, numserv, servlen) >= servlen) return ENI_MEMORY; - strcpy(serv, numserv); } else { #if defined(HAVE_GETSERVBYPORT) struct servent *sp = getservbyport(port, (flags & NI_DGRAM) ? "udp" : "tcp"); if (sp) { - if (strlen(sp->s_name) + 1 > servlen) + if (strlcpy(serv, sp->s_name, servlen) >= servlen) return ENI_MEMORY; - strcpy(serv, sp->s_name); } else return ENI_NOSERVNAME; #else @@ -202,9 +200,8 @@ getnameinfo(const struct sockaddr *sa, socklen_t salen, char *host, socklen_t ho if (inet_ntop(afd->a_af, addr, numaddr, sizeof(numaddr)) == NULL) return ENI_SYSTEM; - if (strlen(numaddr) > hostlen) + if (strlcpy(host, numaddr, hostlen) >= hostlen) return ENI_MEMORY; - strcpy(host, numaddr); } else { #ifdef INET6 hp = getipnodebyaddr(addr, afd->a_addrlen, afd->a_af, &h_error); @@ -218,13 +215,12 @@ getnameinfo(const struct sockaddr *sa, socklen_t salen, char *host, socklen_t ho p = strchr(hp->h_name, '.'); if (p) *p = '\0'; } - if (strlen(hp->h_name) + 1 > hostlen) { + if (strlcpy(host, hp->h_name, hostlen) >= hostlen) { #ifdef INET6 freehostent(hp); #endif return ENI_MEMORY; } - strcpy(host, hp->h_name); #ifdef INET6 freehostent(hp); #endif @@ -234,9 +230,8 @@ getnameinfo(const struct sockaddr *sa, socklen_t salen, char *host, socklen_t ho if (inet_ntop(afd->a_af, addr, numaddr, sizeof(numaddr)) == NULL) return ENI_NOHOSTNAME; - if (strlen(numaddr) > hostlen) + if (strlcpy(host, numaddr, hostlen) >= hostlen) return ENI_MEMORY; - strcpy(host, numaddr); } } return SUCCESS; diff --git a/ext/socket/ipsocket.c b/ext/socket/ipsocket.c index edd270c3adafcd..fa41d89936fa2f 100644 --- a/ext/socket/ipsocket.c +++ b/ext/socket/ipsocket.c @@ -208,8 +208,8 @@ rsock_init_inetsock( VALUE self, VALUE remote_host, VALUE remote_serv, VALUE local_host, VALUE local_serv, int type, VALUE resolv_timeout, VALUE connect_timeout, VALUE open_timeout, - VALUE _fast_fallback, VALUE _test_mode_settings -) { + VALUE _fast_fallback, VALUE _test_mode_settings) +{ if (!NIL_P(open_timeout) && (!NIL_P(resolv_timeout) || !NIL_P(connect_timeout))) { rb_raise(rb_eArgError, "Cannot specify open_timeout along with connect_timeout or resolv_timeout"); } @@ -423,8 +423,8 @@ select_expires_at( struct timeval *connection_attempt_delay, struct timeval *user_specified_resolv_timeout_at, struct timeval *user_specified_connect_timeout_at, - struct timeval *user_specified_open_timeout_at -) { + struct timeval *user_specified_open_timeout_at) +{ if (any_addrinfos(resolution_store)) { struct timeval *delay; delay = resolution_delay ? resolution_delay : connection_attempt_delay; @@ -526,7 +526,8 @@ in_progress_fds(int fds_size) } static void -remove_connection_attempt_fd(int *fds, int *fds_size, int removing_fd) { +remove_connection_attempt_fd(int *fds, int *fds_size, int removing_fd) +{ int i, j; for (i = 0; i < *fds_size; i++) { @@ -1283,8 +1284,8 @@ rsock_init_inetsock( VALUE self, VALUE remote_host, VALUE remote_serv, VALUE local_host, VALUE local_serv, int type, VALUE resolv_timeout, VALUE connect_timeout, VALUE open_timeout, - VALUE fast_fallback, VALUE test_mode_settings -) { + VALUE fast_fallback, VALUE test_mode_settings) +{ if (!NIL_P(open_timeout) && (!NIL_P(resolv_timeout) || !NIL_P(connect_timeout))) { rb_raise(rb_eArgError, "Cannot specify open_timeout along with connect_timeout or resolv_timeout"); } @@ -1315,7 +1316,7 @@ rsock_init_inetsock( ); struct addrinfo *tmp_p = local_res->ai; - for (tmp_p; tmp_p != NULL; tmp_p = tmp_p->ai_next) { + for (; tmp_p != NULL; tmp_p = tmp_p->ai_next) { if (target_families[0] == 0 && tmp_p->ai_family == AF_INET6) { target_families[0] = AF_INET6; resolving_family_size++; diff --git a/ext/socket/raddrinfo.c b/ext/socket/raddrinfo.c index 6bacc1c22162a2..22ab34a073a0f1 100644 --- a/ext/socket/raddrinfo.c +++ b/ext/socket/raddrinfo.c @@ -387,7 +387,7 @@ allocate_getaddrinfo_arg(const char *hostp, const char *portp, const struct addr if (hostp) { arg->node = buf + hostp_offset; - strcpy(arg->node, hostp); + memcpy(arg->node, hostp, portp_offset - hostp_offset); } else { arg->node = NULL; @@ -395,7 +395,7 @@ allocate_getaddrinfo_arg(const char *hostp, const char *portp, const struct addr if (portp) { arg->service = buf + portp_offset; - strcpy(arg->service, portp); + memcpy(arg->service, portp, bufsize - portp_offset); } else { arg->service = NULL; diff --git a/file.c b/file.c index ff755b4ac312d7..4fc2fec75f59a6 100644 --- a/file.c +++ b/file.c @@ -5060,8 +5060,11 @@ rb_file_dirname_n(VALUE fname, int n) break; } } - if (p == name) - return rb_usascii_str_new2("."); + if (p == name) { + dirname = rb_str_new(".", 1); + rb_enc_copy(dirname, fname); + return dirname; + } #ifdef DOSISH_DRIVE_LETTER if (has_drive_letter(name) && isdirsep(*(name + 2))) { const char *top = skiproot(name + 2, end, enc); diff --git a/gc.c b/gc.c index e8709dcf281a64..0433da2e67c282 100644 --- a/gc.c +++ b/gc.c @@ -1010,6 +1010,18 @@ newobj_of(rb_ractor_t *cr, VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v RB_VM_LOCK_LEAVE_CR_LEV(cr, &lev); } +#if RGENGC_CHECK_MODE +# ifndef GC_DEBUG_SLOT_FILL_SPECIAL_VALUE +# define GC_DEBUG_SLOT_FILL_SPECIAL_VALUE 255 +# endif + + memset( + (void *)(obj + RVALUE_SIZE), + GC_DEBUG_SLOT_FILL_SPECIAL_VALUE, + rb_gc_obj_slot_size(obj) - RVALUE_SIZE + ); +#endif + return obj; } @@ -4953,12 +4965,6 @@ rb_raw_obj_info(char *const buff, const size_t buff_size, VALUE obj) #undef APPEND_F #undef BUFF_ARGS -#if RGENGC_OBJ_INFO -#define OBJ_INFO_BUFFERS_NUM 10 -#define OBJ_INFO_BUFFERS_SIZE 0x100 -static rb_atomic_t obj_info_buffers_index = 0; -static char obj_info_buffers[OBJ_INFO_BUFFERS_NUM][OBJ_INFO_BUFFERS_SIZE]; - /* Increments *var atomically and resets *var to 0 when maxval is * reached. Returns the wraparound old *var value (0...maxval). */ static rb_atomic_t @@ -4976,17 +4982,18 @@ atomic_inc_wraparound(rb_atomic_t *var, const rb_atomic_t maxval) static const char * obj_info(VALUE obj) { - rb_atomic_t index = atomic_inc_wraparound(&obj_info_buffers_index, OBJ_INFO_BUFFERS_NUM); - char *const buff = obj_info_buffers[index]; - return rb_raw_obj_info(buff, OBJ_INFO_BUFFERS_SIZE, obj); -} -#else -static const char * -obj_info(VALUE obj) -{ + if (RGENGC_OBJ_INFO) { + static struct { + rb_atomic_t index; + char buffers[10][0x100]; + } info = {0}; + + rb_atomic_t index = atomic_inc_wraparound(&info.index, numberof(info.buffers)); + char *const buff = info.buffers[index]; + return rb_raw_obj_info(buff, sizeof(info.buffers[0]), obj); + } return obj_type_name(obj); } -#endif /* ------------------------ Extended allocator ------------------------ diff --git a/gc/default/default.c b/gc/default/default.c index d5b18d20b96c8b..7264fb799627eb 100644 --- a/gc/default/default.c +++ b/gc/default/default.c @@ -4392,15 +4392,16 @@ static inline void gc_mark_check_t_none(rb_objspace_t *objspace, VALUE obj) { if (RB_UNLIKELY(RB_TYPE_P(obj, T_NONE))) { - char obj_info_buf[256]; - rb_raw_obj_info(obj_info_buf, 256, obj); + enum {info_size = 256}; + char obj_info_buf[info_size]; + rb_raw_obj_info(obj_info_buf, info_size, obj); - char parent_obj_info_buf[256]; + char parent_obj_info_buf[info_size]; if (objspace->rgengc.parent_object == Qfalse) { - strcpy(parent_obj_info_buf, "(none)"); + strlcpy(parent_obj_info_buf, "(none)", info_size); } else { - rb_raw_obj_info(parent_obj_info_buf, 256, objspace->rgengc.parent_object); + rb_raw_obj_info(parent_obj_info_buf, info_size, objspace->rgengc.parent_object); } rb_bug("try to mark T_NONE object (obj: %s, parent: %s)", obj_info_buf, parent_obj_info_buf); diff --git a/imemo.c b/imemo.c index abf101f362f325..7cec33bc1edf00 100644 --- a/imemo.c +++ b/imemo.c @@ -48,27 +48,23 @@ rb_imemo_new(enum imemo_type type, VALUE v0, size_t size) return (VALUE)obj; } -static rb_imemo_tmpbuf_t * +VALUE rb_imemo_tmpbuf_new(void) { - size_t size = sizeof(struct rb_imemo_tmpbuf_struct); VALUE flags = T_IMEMO | (imemo_tmpbuf << FL_USHIFT); - NEWOBJ_OF(obj, struct rb_imemo_tmpbuf_struct, 0, flags, size, 0); + NEWOBJ_OF(obj, rb_imemo_tmpbuf_t, 0, flags, sizeof(rb_imemo_tmpbuf_t), NULL); - return obj; + return (VALUE)obj; } void * rb_alloc_tmp_buffer_with_count(volatile VALUE *store, size_t size, size_t cnt) { - void *ptr; - rb_imemo_tmpbuf_t *tmpbuf; - /* Keep the order; allocate an empty imemo first then xmalloc, to * get rid of potential memory leak */ - tmpbuf = rb_imemo_tmpbuf_new(); + rb_imemo_tmpbuf_t *tmpbuf = (rb_imemo_tmpbuf_t *)rb_imemo_tmpbuf_new(); *store = (VALUE)tmpbuf; - ptr = ruby_xmalloc(size); + void *ptr = ruby_xmalloc(size); tmpbuf->ptr = ptr; tmpbuf->cnt = cnt; @@ -101,7 +97,7 @@ rb_free_tmp_buffer(volatile VALUE *store) rb_imemo_tmpbuf_t * rb_imemo_tmpbuf_parser_heap(void *buf, rb_imemo_tmpbuf_t *old_heap, size_t cnt) { - rb_imemo_tmpbuf_t *tmpbuf = rb_imemo_tmpbuf_new(); + rb_imemo_tmpbuf_t *tmpbuf = (rb_imemo_tmpbuf_t *)rb_imemo_tmpbuf_new(); tmpbuf->ptr = buf; tmpbuf->next = old_heap; tmpbuf->cnt = cnt; diff --git a/include/ruby/internal/core/robject.h b/include/ruby/internal/core/robject.h index 4892faab2b1adf..99f6470ac15c81 100644 --- a/include/ruby/internal/core/robject.h +++ b/include/ruby/internal/core/robject.h @@ -55,7 +55,7 @@ */ enum ruby_robject_flags { /** - * This flag has marks that the object's instance variables are stored in an + * This flag marks that the object's instance variables are stored in an * external heap buffer. * Normally, instance variable references are stored inside the object slot, * but if it overflow, Ruby may have to allocate a separate buffer and spills diff --git a/internal/imemo.h b/internal/imemo.h index f7bd6202384dca..de617d94c1bf5c 100644 --- a/internal/imemo.h +++ b/internal/imemo.h @@ -93,7 +93,6 @@ struct vm_ifunc { struct rb_imemo_tmpbuf_struct { VALUE flags; - VALUE reserved; VALUE *ptr; /* malloc'ed buffer */ struct rb_imemo_tmpbuf_struct *next; /* next imemo */ size_t cnt; /* buffer size in VALUE */ @@ -133,16 +132,15 @@ struct MEMO { #ifndef RUBY_RUBYPARSER_H typedef struct rb_imemo_tmpbuf_struct rb_imemo_tmpbuf_t; #endif +VALUE rb_imemo_tmpbuf_new(void); rb_imemo_tmpbuf_t *rb_imemo_tmpbuf_parser_heap(void *buf, rb_imemo_tmpbuf_t *old_heap, size_t cnt); struct vm_ifunc *rb_vm_ifunc_new(rb_block_call_func_t func, const void *data, int min_argc, int max_argc); static inline enum imemo_type imemo_type(VALUE imemo); static inline int imemo_type_p(VALUE imemo, enum imemo_type imemo_type); static inline bool imemo_throw_data_p(VALUE imemo); static inline struct vm_ifunc *rb_vm_ifunc_proc_new(rb_block_call_func_t func, const void *data); -static inline VALUE rb_imemo_tmpbuf_auto_free_pointer(void); static inline void *RB_IMEMO_TMPBUF_PTR(VALUE v); static inline void *rb_imemo_tmpbuf_set_ptr(VALUE v, void *ptr); -static inline VALUE rb_imemo_tmpbuf_auto_free_pointer_new_from_an_RString(VALUE str); static inline void MEMO_V1_SET(struct MEMO *m, VALUE v); static inline void MEMO_V2_SET(struct MEMO *m, VALUE v); @@ -201,12 +199,6 @@ rb_vm_ifunc_proc_new(rb_block_call_func_t func, const void *data) return rb_vm_ifunc_new(func, data, 0, UNLIMITED_ARGUMENTS); } -static inline VALUE -rb_imemo_tmpbuf_auto_free_pointer(void) -{ - return rb_imemo_new(imemo_tmpbuf, 0, sizeof(rb_imemo_tmpbuf_t)); -} - static inline void * RB_IMEMO_TMPBUF_PTR(VALUE v) { @@ -221,7 +213,7 @@ rb_imemo_tmpbuf_set_ptr(VALUE v, void *ptr) } static inline VALUE -rb_imemo_tmpbuf_auto_free_pointer_new_from_an_RString(VALUE str) +rb_imemo_tmpbuf_new_from_an_RString(VALUE str) { const void *src; VALUE imemo; @@ -231,7 +223,7 @@ rb_imemo_tmpbuf_auto_free_pointer_new_from_an_RString(VALUE str) StringValue(str); /* create tmpbuf to keep the pointer before xmalloc */ - imemo = rb_imemo_tmpbuf_auto_free_pointer(); + imemo = rb_imemo_tmpbuf_new(); tmpbuf = (rb_imemo_tmpbuf_t *)imemo; len = RSTRING_LEN(str); src = RSTRING_PTR(str); diff --git a/jit.c b/jit.c index efecbef35455ca..f233a2f01f1c8e 100644 --- a/jit.c +++ b/jit.c @@ -533,6 +533,131 @@ for_each_iseq_i(void *vstart, void *vend, size_t stride, void *data) return 0; } +uint32_t +rb_jit_get_page_size(void) +{ +#if defined(_SC_PAGESIZE) + long page_size = sysconf(_SC_PAGESIZE); + if (page_size <= 0) rb_bug("jit: failed to get page size"); + + // 1 GiB limit. x86 CPUs with PDPE1GB can do this and anything larger is unexpected. + // Though our design sort of assume we have fine grained control over memory protection + // which require small page sizes. + if (page_size > 0x40000000l) rb_bug("jit page size too large"); + + return (uint32_t)page_size; +#else +#error "JIT supports POSIX only for now" +#endif +} + +#if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE) +// Align the current write position to a multiple of bytes +static uint8_t * +align_ptr(uint8_t *ptr, uint32_t multiple) +{ + // Compute the pointer modulo the given alignment boundary + uint32_t rem = ((uint32_t)(uintptr_t)ptr) % multiple; + + // If the pointer is already aligned, stop + if (rem == 0) + return ptr; + + // Pad the pointer by the necessary amount to align it + uint32_t pad = multiple - rem; + + return ptr + pad; +} +#endif + +// Address space reservation. Memory pages are mapped on an as needed basis. +// See the Rust mm module for details. +uint8_t * +rb_jit_reserve_addr_space(uint32_t mem_size) +{ +#ifndef _WIN32 + uint8_t *mem_block; + + // On Linux + #if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE) + uint32_t const page_size = (uint32_t)sysconf(_SC_PAGESIZE); + uint8_t *const cfunc_sample_addr = (void *)(uintptr_t)&rb_jit_reserve_addr_space; + uint8_t *const probe_region_end = cfunc_sample_addr + INT32_MAX; + // Align the requested address to page size + uint8_t *req_addr = align_ptr(cfunc_sample_addr, page_size); + + // Probe for addresses close to this function using MAP_FIXED_NOREPLACE + // to improve odds of being in range for 32-bit relative call instructions. + do { + mem_block = mmap( + req_addr, + mem_size, + PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, + -1, + 0 + ); + + // If we succeeded, stop + if (mem_block != MAP_FAILED) { + ruby_annotate_mmap(mem_block, mem_size, "Ruby:rb_jit_reserve_addr_space"); + break; + } + + // -4MiB. Downwards to probe away from the heap. (On x86/A64 Linux + // main_code_addr < heap_addr, and in case we are in a shared + // library mapped higher than the heap, downwards is still better + // since it's towards the end of the heap rather than the stack.) + req_addr -= 4 * 1024 * 1024; + } while (req_addr < probe_region_end); + + // On MacOS and other platforms + #else + // Try to map a chunk of memory as executable + mem_block = mmap( + (void *)rb_jit_reserve_addr_space, + mem_size, + PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, + -1, + 0 + ); + #endif + + // Fallback + if (mem_block == MAP_FAILED) { + // Try again without the address hint (e.g., valgrind) + mem_block = mmap( + NULL, + mem_size, + PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, + -1, + 0 + ); + + if (mem_block != MAP_FAILED) { + ruby_annotate_mmap(mem_block, mem_size, "Ruby:rb_jit_reserve_addr_space:fallback"); + } + } + + // Check that the memory mapping was successful + if (mem_block == MAP_FAILED) { + perror("ruby: jit: mmap:"); + if(errno == ENOMEM) { + // No crash report if it's only insufficient memory + exit(EXIT_FAILURE); + } + rb_bug("mmap failed"); + } + + return mem_block; +#else + // Windows not supported for now + return NULL; +#endif +} + // Walk all ISEQs in the heap and invoke the callback - shared between YJIT and ZJIT void rb_jit_for_each_iseq(rb_iseq_callback callback, void *data) @@ -540,3 +665,52 @@ rb_jit_for_each_iseq(rb_iseq_callback callback, void *data) struct iseq_callback_data callback_data = { .callback = callback, .data = data }; rb_objspace_each_objects(for_each_iseq_i, (void *)&callback_data); } + +bool +rb_jit_mark_writable(void *mem_block, uint32_t mem_size) +{ + return mprotect(mem_block, mem_size, PROT_READ | PROT_WRITE) == 0; +} + +void +rb_jit_mark_executable(void *mem_block, uint32_t mem_size) +{ + // Do not call mprotect when mem_size is zero. Some platforms may return + // an error for it. https://github.com/Shopify/ruby/issues/450 + if (mem_size == 0) { + return; + } + if (mprotect(mem_block, mem_size, PROT_READ | PROT_EXEC)) { + rb_bug("Couldn't make JIT page (%p, %lu bytes) executable, errno: %s", + mem_block, (unsigned long)mem_size, strerror(errno)); + } +} + +// Free the specified memory block. +bool +rb_jit_mark_unused(void *mem_block, uint32_t mem_size) +{ + // On Linux, you need to use madvise MADV_DONTNEED to free memory. + // We might not need to call this on macOS, but it's not really documented. + // We generally prefer to do the same thing on both to ease testing too. + madvise(mem_block, mem_size, MADV_DONTNEED); + + // On macOS, mprotect PROT_NONE seems to reduce RSS. + // We also call this on Linux to avoid executing unused pages. + return mprotect(mem_block, mem_size, PROT_NONE) == 0; +} + +// Invalidate icache for arm64. +// `start` is inclusive and `end` is exclusive. +void +rb_jit_icache_invalidate(void *start, void *end) +{ + // Clear/invalidate the instruction cache. Compiles to nothing on x86_64 + // but required on ARM before running freshly written code. + // On Darwin it's the same as calling sys_icache_invalidate(). +#ifdef __GNUC__ + __builtin___clear_cache(start, end); +#elif defined(__aarch64__) +#error No instruction cache clear available with this compiler on Aarch64! +#endif +} diff --git a/jit/Cargo.toml b/jit/Cargo.toml new file mode 100644 index 00000000000000..530fe3674b0345 --- /dev/null +++ b/jit/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "jit" +version = "0.1.0" +edition = "2024" + +[dependencies] diff --git a/jit/src/lib.rs b/jit/src/lib.rs new file mode 100644 index 00000000000000..6079d00f2fd886 --- /dev/null +++ b/jit/src/lib.rs @@ -0,0 +1,37 @@ +//! Shared code between YJIT and ZJIT. + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::alloc::{GlobalAlloc, Layout, System}; + +#[global_allocator] +pub static GLOBAL_ALLOCATOR: StatsAlloc = StatsAlloc { alloc_size: AtomicUsize::new(0) }; + +pub struct StatsAlloc { + pub alloc_size: AtomicUsize, +} + +unsafe impl GlobalAlloc for StatsAlloc { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + self.alloc_size.fetch_add(layout.size(), Ordering::SeqCst); + unsafe { System.alloc(layout) } + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + self.alloc_size.fetch_sub(layout.size(), Ordering::SeqCst); + unsafe { System.dealloc(ptr, layout) } + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + self.alloc_size.fetch_add(layout.size(), Ordering::SeqCst); + unsafe { System.alloc_zeroed(layout) } + } + + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + if new_size > layout.size() { + self.alloc_size.fetch_add(new_size - layout.size(), Ordering::SeqCst); + } else if new_size < layout.size() { + self.alloc_size.fetch_sub(layout.size() - new_size, Ordering::SeqCst); + } + unsafe { System.realloc(ptr, layout, new_size) } + } +} diff --git a/lib/bundler/definition.rb b/lib/bundler/definition.rb index e400c38cec54dc..e177b6e39673c7 100644 --- a/lib/bundler/definition.rb +++ b/lib/bundler/definition.rb @@ -1036,6 +1036,8 @@ def converge_specs(specs) specs.each do |s| name = s.name + next if @gems_to_unlock.include?(name) + dep = @dependencies.find {|d| s.satisfies?(d) } lockfile_source = s.source @@ -1049,12 +1051,13 @@ def converge_specs(specs) # Replace the locked dependency's source with the equivalent source from the Gemfile s.source = replacement_source || default_source + next if s.source_changed? source = s.source next if @sources_to_unlock.include?(source.name) # Path sources have special logic - if source.instance_of?(Source::Path) || source.instance_of?(Source::Gemspec) || (source.instance_of?(Source::Git) && !@gems_to_unlock.include?(name) && deps.include?(dep)) + if source.is_a?(Source::Path) new_spec = source.specs[s].first if new_spec s.runtime_dependencies.replace(new_spec.runtime_dependencies) @@ -1136,7 +1139,7 @@ def lockfiles_equal?(current, proposed, preserve_unknown_sections) def additional_base_requirements_to_prevent_downgrades(resolution_base) return resolution_base unless @locked_gems && !sources.expired_sources?(@locked_gems.sources) @originally_locked_specs.each do |locked_spec| - next if locked_spec.source.is_a?(Source::Path) + next if locked_spec.source.is_a?(Source::Path) || locked_spec.source_changed? name = locked_spec.name next if @changed_dependencies.include?(name) diff --git a/lib/bundler/ruby_dsl.rb b/lib/bundler/ruby_dsl.rb index cd88253f463374..db4d5521e54925 100644 --- a/lib/bundler/ruby_dsl.rb +++ b/lib/bundler/ruby_dsl.rb @@ -57,6 +57,8 @@ def normalize_ruby_file(filename) else file_content.strip end + rescue Errno::ENOENT + raise GemfileError, "Could not find version file #{filename}" end end end diff --git a/lib/erb.rb b/lib/erb.rb index ebf91e4792d67e..d4f43e0772341b 100644 --- a/lib/erb.rb +++ b/lib/erb.rb @@ -17,246 +17,505 @@ require 'erb/def_method' require 'erb/util' +# :markup: markdown # -# = ERB -- Ruby Templating +# Class **ERB** (the name stands for **Embedded Ruby**) +# is an easy-to-use, but also very powerful, [template processor][template processor]. # -# == Introduction +# Like method [sprintf][sprintf], \ERB can format run-time data into a string. +# \ERB, however,s is *much more powerful*. # -# ERB provides an easy to use but powerful templating system for Ruby. Using -# ERB, actual Ruby code can be added to any plain text document for the -# purposes of generating document information details and/or flow control. +# In simplest terms: # -# A very simple example is this: +# - You can create an \ERB object to store a text *template* that includes specially formatted *tags*; +# each tag specifies data that at run-time is to replace the tag in the produced result. +# - You can call instance method ERB#result to get the result, +# which is the string formed by replacing each tag with run-time data. # -# require 'erb' +# \ERB is commonly used to produce: # -# x = 42 -# template = ERB.new <<-EOF -# The value of x is: <%= x %> -# EOF -# puts template.result(binding) +# - Customized or personalized email messages. +# - Customized or personalized web pages. +# - Software code (in code-generating applications). # -# Prints: The value of x is: 42 +# ## Usage # -# More complex examples are given below. +# Before you can use \ERB, you must first require it +# (examples on this page assume that this has been done): # +# ``` +# require 'erb' +# ``` # -# == Recognized Tags +# ## In Brief # -# ERB recognizes certain tags in the provided template and converts them based -# on the rules below: +# ``` +# # Expression tag: begins with '<%', ends with '%>'. +# # This expression does not need the local binding. +# ERB.new('Today is <%= Date::DAYNAMES[Date.today.wday] %>.').result # => "Today is Monday." +# # This expression tag does need the local binding. +# magic_word = 'xyzzy' +# template.result(binding) # => "The magic word is xyzzy." # -# <% Ruby code -- inline with output %> -# <%= Ruby expression -- replace with result %> -# <%# comment -- ignored -- useful in testing %> (`<% #` doesn't work. Don't use Ruby comments.) -# % a line of Ruby code -- treated as <% line %> (optional -- see ERB.new) -# %% replaced with % if first thing on a line and % processing is used -# <%% or %%> -- replace with <% or %> respectively +# Execution tag: begins with '<%=', ends with '%>'. +# s = '<% File.write("t.txt", "Some stuff.") %>' +# ERB.new(s).result +# File.read('t.txt') # => "Some stuff." # -# All other text is passed through ERB filtering unchanged. +# # Comment tag: begins with '<%#', ends with '%>'. +# s = 'Some stuff;<%# Note to self: figure out what the stuff is. %> more stuff.' +# ERB.new(s).result # => "Some stuff; more stuff." +# ``` # # -# == Options +# ## Some Simple Examples # -# There are several settings you can change when you use ERB: -# * the nature of the tags that are recognized; -# * the binding used to resolve local variables in the template. +# Here's a simple example of \ERB in action: # -# See the ERB.new and ERB#result methods for more detail. +# ``` +# s = 'The time is <%= Time.now %>.' +# template = ERB.new(s) +# template.result +# # => "The time is 2025-09-09 10:49:26 -0500." +# ``` # -# == Character encodings +# Details: # -# ERB (or Ruby code generated by ERB) returns a string in the same -# character encoding as the input string. When the input string has -# a magic comment, however, it returns a string in the encoding specified -# by the magic comment. +# 1. A plain-text string is assigned to variable `s`. +# Its embedded [expression tag][expression tags] `'<%= Time.now %>'` includes a Ruby expression, `Time.now`. +# 2. The string is put into a new \ERB object, and stored in variable `template`. +# 4. Method call `template.result` generates a string that contains the run-time value of `Time.now`, +# as computed at the time of the call. # -# # -*- coding: utf-8 -*- -# require 'erb' +# The template may be re-used: # -# template = ERB.new < -# \_\_ENCODING\_\_ is <%= \_\_ENCODING\_\_ %>. -# EOF -# puts template.result +# ``` +# template.result +# # => "The time is 2025-09-09 10:49:33 -0500." +# ``` # -# Prints: \_\_ENCODING\_\_ is Big5. +# Another example: # +# ``` +# s = 'The magic word is <%= magic_word %>.' +# template = ERB.new(s) +# magic_word = 'abracadabra' +# # => "abracadabra" +# template.result(binding) +# # => "The magic word is abracadabra." +# ``` # -# == Examples +# Details: # -# === Plain Text +# 1. As before, a plain-text string is assigned to variable `s`. +# Its embedded [expression tag][expression tags] `'<%= magic_word %>'` has a variable *name*, `magic_word`. +# 2. The string is put into a new \ERB object, and stored in variable `template`; +# note that `magic_word` need not be defined before the \ERB object is created. +# 3. `magic_word = 'abracadabra'` assigns a value to variable `magic_word`. +# 4. Method call `template.result(binding)` generates a string +# that contains the *value* of `magic_word`. # -# ERB is useful for any generic templating situation. Note that in this example, we use the -# convenient "% at start of line" tag, and we quote the template literally with -# %q{...} to avoid trouble with the backslash. +# As before, the template may be re-used: # -# require "erb" +# ``` +# magic_word = 'xyzzy' +# template.result(binding) +# # => "The magic word is xyzzy." +# ``` +# +# ## Bindings +# +# The first example above passed no argument to method `result`; +# the second example passed argument `binding`. +# +# Here's why: +# +# - The first example has tag `<%= Time.now %>`, +# which cites *globally-defined* constant `Time`; +# the default `binding` (details not needed here) includes the binding of global constant `Time` to its value. +# - The second example has tag `<%= magic_word %>`, +# which cites *locally-defined* variable `magic_word`; +# the passed argument `binding` (which is simply a call to method [Kernel#binding][kernel#binding]) +# includes the binding of local variable `magic_word` to its value. +# +# ## Tags +# +# The examples above use expression tags. +# These are the tags available in \ERB: +# +# - [Expression tag][expression tags]: the tag contains a Ruby exprssion; +# in the result, the entire tag is to be replaced with the run-time value of the expression. +# - [Execution tag][execution tags]: the tag contains Ruby code; +# in the result, the entire tag is to be replaced with the run-time value of the code. +# - [Comment tag][comment tags]: the tag contains comment code; +# in the result, the entire tag is to be omitted. +# +# ### Expression Tags +# +# You can embed a Ruby expression in a template using an *expression tag*. +# +# Its syntax is `<%= _expression_ %>`, +# where *expression* is any valid Ruby expression. +# +# When you call method #result, +# the method evaluates the expression and replaces the entire expression tag with the expression's value: +# +# ``` +# ERB.new('Today is <%= Date::DAYNAMES[Date.today.wday] %>.').result +# # => "Today is Monday." +# ERB.new('Tomorrow will be <%= Date::DAYNAMES[Date.today.wday + 1] %>.').result +# # => "Tomorrow will be Tuesday." +# ERB.new('Yesterday was <%= Date::DAYNAMES[Date.today.wday - 1] %>.').result +# # => "Yesterday was Sunday." +# ``` +# +# Note that whitespace before and after the expression +# is allowed but not required, +# and that such whitespace is stripped from the result. +# +# ``` +# ERB.new('My appointment is on <%=Date::DAYNAMES[Date.today.wday + 2]%>.').result +# # => "My appointment is on Wednesday." +# ERB.new('My appointment is on <%= Date::DAYNAMES[Date.today.wday + 2] %>.').result +# # => "My appointment is on Wednesday." +# ``` +# +# ### Execution Tags +# +# You can embed Ruby executable code in template using an *execution tag*. +# +# Its syntax is `<% _code_ %>`, +# where *code* is any valid Ruby code. +# +# When you call method #result, +# the method executes the code and removes the entire execution tag +# (generating no text in the result). +# +# You can interleave text with execution tags to form a control structure +# such as a conditional, a loop, or a `case` statements. +# +# Conditional: +# +# ``` +# s = < +# An error has occurred. +# <% else %> +# Oops! +# <% end %> +# EOT +# template = ERB.new(s) +# verbosity = true +# template.result(binding) +# # => "\nAn error has occurred.\n\n" +# verbosity = false +# template.result(binding) +# # => "\nOops!\n\n" +# ``` +# +# Note that the interleaved text may itself contain expression tags: +# +# Loop: +# +# ``` +# s = < +# <%= dayname %> +# <% end %> +# EOT +# ERB.new(s).result +# # => "\nSun\n\nMon\n\nTue\n\nWed\n\nThu\n\nFri\n\nSat\n\n" +# ``` +# +# Other, non-control, lines of Ruby code may be interleaved with the text, +# and the Ruby code may itself contain regular Ruby comments: +# +# ``` +# s = < +# <%= Time.now %> +# <% sleep(1) # Let's make the times different. %> +# <% end %> +# EOT +# ERB.new(s).result +# # => "\n2025-09-09 11:36:02 -0500\n\n\n2025-09-09 11:36:03 -0500\n\n\n2025-09-09 11:36:04 -0500\n\n\n" +# ``` +# +# The execution tag may also contain multiple lines of code: +# +# ``` +# s = < +# * <%=i%>,<%=j%> +# <% +# end +# end +# %> +# EOT +# ERB.new(s).result +# # => "\n* 0,0\n\n* 0,1\n\n* 0,2\n\n* 1,0\n\n* 1,1\n\n* 1,2\n\n* 2,0\n\n* 2,1\n\n* 2,2\n\n" +# ``` +# +# #### Shorthand Format for Execution Tags +# +# You can give `trim_mode: '%'` to enable a shorthand format for execution tags; +# this example uses the shorthand format `% _code_` instead of `<% _code_ %>`: # -# # Create template. -# template = %q{ -# From: James Edward Gray II -# To: <%= to %> -# Subject: Addressing Needs +# ``` +# s = < +# % end +# EOT +# template = ERB.new(s, trim_mode: '%') +# priorities = [ 'Run Ruby Quiz', +# 'Document Modules', +# 'Answer Questions on Ruby Talk' ] +# puts template.result(binding) +# * Run Ruby Quiz +# * Document Modules +# * Answer Questions on Ruby Talk +# ``` # -# <%= to[/\w+/] %>: +# Note that in the shorthand format, the character `'%'` must be the first character in the code line +# (no leading whitespace). # -# Just wanted to send a quick note assuring that your needs are being -# addressed. +# ### Comment Tags # -# I want you to know that my team will keep working on the issues, -# especially: +# You can embed a comment in a template using a *comment tag*; +# its syntax is `<%# _text_ %>`, +# where *text* is the text of the comment. # -# <%# ignore numerous minor requests -- focus on priorities %> -# % priorities.each do |priority| -# * <%= priority %> -# % end +# When you call method #result, +# it removes the entire comment tag +# (generating no text in the result). # -# Thanks for your patience. +# Example: # -# James Edward Gray II -# }.gsub(/^ /, '') +# ``` +# s = 'Some stuff;<%# Note to self: figure out what the stuff is. %> more stuff.' +# ERB.new(s).result # => "Some stuff; more stuff." +# ``` # -# message = ERB.new(template, trim_mode: "%<>") +# A comment tag may appear anywhere in the template text. # -# # Set up template data. -# to = "Community Spokesman " -# priorities = [ "Run Ruby Quiz", -# "Document Modules", -# "Answer Questions on Ruby Talk" ] +# Note that the beginning of the tag must be `'<%#'`, not `'<% #'`. # -# # Produce result. -# email = message.result -# puts email +# In this example, the tag begins with `'<% #'`, and so is an execution tag, not a comment tag; +# the cited code consists entirely of a Ruby-style comment (which is of course ignored): # -# Generates: +# ``` +# ERB.new('Some stuff;<% # Note to self: figure out what the stuff is. %> more stuff.').result +# # => "Some stuff;" +# ``` # -# From: James Edward Gray II -# To: Community Spokesman -# Subject: Addressing Needs +# ## Encodings # -# Community: +# In general, an \ERB result string (or Ruby code generated by \ERB) +# has the same encoding as the string originally passed to ERB.new; +# see [Encoding][encoding]. # -# Just wanted to send a quick note assuring that your needs are being addressed. +# You can specify the output encoding by adding a [magic comment][magic comments] +# at the top of the given string: # -# I want you to know that my team will keep working on the issues, especially: +# ``` +# s = < # -# * Run Ruby Quiz -# * Document Modules -# * Answer Questions on Ruby Talk +# Some text. +# EOF +# # => "<%#-*- coding: Big5 -*-%>\n\nSome text.\n" +# s.encoding +# # => # +# ERB.new(s).result.encoding +# # => # +# ``` # -# Thanks for your patience. +# ## Plain Text Example # -# James Edward Gray II +# Here's a plain-text string; +# it uses the literal notation `'%q{ ... }'` to define the string +# (see [%q literals][%q literals]); +# this avoids problems with backslashes. # -# === Ruby in HTML +# ``` +# s = %q{ +# From: James Edward Gray II +# To: <%= to %> +# Subject: Addressing Needs # -# ERB is often used in .rhtml files (HTML with embedded Ruby). Notice the need in -# this example to provide a special binding when the template is run, so that the instance -# variables in the Product object can be resolved. +# <%= to[/\w+/] %>: # -# require "erb" +# Just wanted to send a quick note assuring that your needs are being +# addressed. # -# # Build template data class. -# class Product -# def initialize( code, name, desc, cost ) -# @code = code -# @name = name -# @desc = desc -# @cost = cost +# I want you to know that my team will keep working on the issues, +# especially: # -# @features = [ ] -# end +# <%# ignore numerous minor requests -- focus on priorities %> +# % priorities.each do |priority| +# * <%= priority %> +# % end # -# def add_feature( feature ) -# @features << feature -# end +# Thanks for your patience. # -# # Support templating of member data. -# def get_binding -# binding -# end +# James Edward Gray II +# } +# ``` # -# # ... -# end +# The template will need these: # -# # Create template. -# template = %{ -# -# Ruby Toys -- <%= @name %> -# +# ``` +# to = 'Community Spokesman ' +# priorities = [ 'Run Ruby Quiz', +# 'Document Modules', +# 'Answer Questions on Ruby Talk' ] +# ``` # -#

<%= @name %> (<%= @code %>)

-#

<%= @desc %>

+# Finally, make the template and get the result # -#
    -# <% @features.each do |f| %> -#
  • <%= f %>
    • -# <% end %> -#
+# ``` +# template = ERB.new(s, trim_mode: '%<>') +# puts template.result(binding) # -#

-# <% if @cost < 10 %> -# Only <%= @cost %>!!! -# <% else %> -# Call for a price, today! -# <% end %> -#

+# From: James Edward Gray II +# To: Community Spokesman +# Subject: Addressing Needs # -# -# -# }.gsub(/^ /, '') +# Community: # -# rhtml = ERB.new(template) +# Just wanted to send a quick note assuring that your needs are being +# addressed. # -# # Set up template data. -# toy = Product.new( "TZ-1002", -# "Rubysapien", -# "Geek's Best Friend! Responds to Ruby commands...", -# 999.95 ) -# toy.add_feature("Listens for verbal commands in the Ruby language!") -# toy.add_feature("Ignores Perl, Java, and all C variants.") -# toy.add_feature("Karate-Chop Action!!!") -# toy.add_feature("Matz signature on left leg.") -# toy.add_feature("Gem studded eyes... Rubies, of course!") +# I want you to know that my team will keep working on the issues, +# especially: # -# # Produce result. -# rhtml.run(toy.get_binding) +# * Run Ruby Quiz +# * Document Modules +# * Answer Questions on Ruby Talk # -# Generates (some blank lines removed): +# Thanks for your patience. # -# -# Ruby Toys -- Rubysapien -# +# James Edward Gray II +# ``` # -#

Rubysapien (TZ-1002)

-#

Geek's Best Friend! Responds to Ruby commands...

+# ## HTML Example # -#
    -#
  • Listens for verbal commands in the Ruby language!
    • -#
  • Ignores Perl, Java, and all C variants.
    • -#
  • Karate-Chop Action!!!
    • -#
  • Matz signature on left leg.
    • -#
  • Gem studded eyes... Rubies, of course!
    • -#
+# This example shows an HTML template. # -#

-# Call for a price, today! -#

-# -# -# +# First, here's a custom class, `Product`: # +# ``` +# class Product +# def initialize(code, name, desc, cost) +# @code = code +# @name = name +# @desc = desc +# @cost = cost +# @features = [] +# end # -# == Notes +# def add_feature(feature) +# @features << feature +# end # -# There are a variety of templating solutions available in various Ruby projects. -# For example, RDoc, distributed with Ruby, uses its own template engine, which -# can be reused elsewhere. +# # Support templating of member data. +# def get_binding +# binding +# end # -# Other popular engines could be found in the corresponding -# {Category}[https://www.ruby-toolbox.com/categories/template_engines] of -# The Ruby Toolbox. +# end +# ``` +# +# The template below will need these values: +# +# ``` +# toy = Product.new('TZ-1002', +# 'Rubysapien', +# "Geek's Best Friend! Responds to Ruby commands...", +# 999.95 +# ) +# toy.add_feature('Listens for verbal commands in the Ruby language!') +# toy.add_feature('Ignores Perl, Java, and all C variants.') +# toy.add_feature('Karate-Chop Action!!!') +# toy.add_feature('Matz signature on left leg.') +# toy.add_feature('Gem studded eyes... Rubies, of course!') +# ``` +# +# Here's the HTML: +# +# ``` +# s = < +# Ruby Toys -- <%= @name %> +# +#

<%= @name %> (<%= @code %>)

+#

<%= @desc %>

+#
    +# <% @features.each do |f| %> +#
  • <%= f %>
    • +# <% end %> +#
+#

+# <% if @cost < 10 %> +# Only <%= @cost %>!!! +# <% else %> +# Call for a price, today! +# <% end %> +#

+# +# +# EOT +# ``` +# +# Finally, build the template and get the result (omitting some blank lines): +# +# ``` +# template = ERB.new(s) +# puts template.result(toy.get_binding) +# +# Ruby Toys -- Rubysapien +# +#

Rubysapien (TZ-1002)

+#

Geek's Best Friend! Responds to Ruby commands...

+#
    +#
  • Listens for verbal commands in the Ruby language!
    • +#
  • Ignores Perl, Java, and all C variants.
    • +#
  • Karate-Chop Action!!!
    • +#
  • Matz signature on left leg.
    • +#
  • Gem studded eyes... Rubies, of course!
    • +#
+#

+# Call for a price, today! +#

+# +# +# ``` +# +# +# ## Other Template Processors +# +# Various Ruby projects have their own template processors. +# The Ruby Processing System [RDoc][rdoc], for example, has one that can be used elsewhere. +# +# Other popular template processors may found in the [Template Engines][template engines] page +# of the Ruby Toolbox. +# +# [binding object]: https://docs.ruby-lang.org/en/master/Binding.html +# [comment tags]: rdoc-ref:ERB@Comment+Tags +# [encoding]: https://docs.ruby-lang.org/en/master/Encoding.html +# [execution tags]: rdoc-ref:ERB@Execution+Tags +# [expression tags]: rdoc-ref:ERB@Expression+Tags +# [kernel#binding]: https://docs.ruby-lang.org/en/master/Kernel.html#method-i-binding +# [%q literals]: https://docs.ruby-lang.org/en/master/syntax/literals_rdoc.html#label-25q-3A+Non-Interpolable+String+Literals +# [magic comments]: https://docs.ruby-lang.org/en/master/syntax/comments_rdoc.html#label-Magic+Comments +# [rdoc]: https://ruby.github.io/rdoc +# [sprintf]: https://docs.ruby-lang.org/en/master/Kernel.html#method-i-sprintf +# [template engines]: https://www.ruby-toolbox.com/categories/template_engines +# [template processor]: https://en.wikipedia.org/wiki/Template_processor # class ERB Revision = '$Date:: $' # :nodoc: #' @@ -267,69 +526,62 @@ def self.version VERSION end + # :markup: markdown + # + # :call-seq: + # ERB.new(string, trim_mode: nil, eoutvar: '_erbout') + # + # Returns a new \ERB object containing the given +string+. # - # Constructs a new ERB object with the template specified in _str_. + # For details about `string`, its embedded tags, and generated results, see ERB. # - # An ERB object works by building a chunk of Ruby code that will output - # the completed template when run. + # **Keyword Argument `trim_mode`** # - # If _trim_mode_ is passed a String containing one or more of the following - # modifiers, ERB will adjust its code generation as listed: + # When keyword argument `trim_mode` has a string value, + # that value may be one of: # - # % enables Ruby code processing for lines beginning with % - # <> omit newline for lines starting with <% and ending in %> - # > omit newline for lines ending in %> - # - omit blank lines ending in -%> + # - `'%'`: Enable [shorthand format][shorthand format] for execution tags. + # - `'-'`: Omit each blank line ending with `'%>'`. + # - `'>'`: Omit newline for each line ending with `'%>'`. + # - `'<>'`: Omit newline for each line starting with `'<%'` and ending with `'%>'`. # - # _eoutvar_ can be used to set the name of the variable ERB will build up - # its output in. This is useful when you need to run multiple ERB - # templates through the same binding and/or when you want to control where - # output ends up. Pass the name of the variable to be used inside a String. + # The value may also be certain combinations of the above. # - # === Example + # - `'%-'`: Enable shorthand and omit each blank line ending with `'%>'`. + # - `'%>'`: Enable shorthand and omit newline for each line ending with `'%>'`. + # - `'%<>'`: Enable shorthand and omit newline for each line starting with `'<%'` and ending with `'%>'`. # - # require "erb" + # **Keyword Argument `eoutvar`** # - # # build data class - # class Listings - # PRODUCT = { :name => "Chicken Fried Steak", - # :desc => "A well messaged pattie, breaded and fried.", - # :cost => 9.95 } + # The string value of keyword argument `eoutvar` specifies the name of the variable + # that method #result uses to construct its result string. + # This is useful when you need to run multiple \ERB templates through the same binding + # and/or when you want to control where output ends up. # - # attr_reader :product, :price + # It's good practice to choose a variable name that begins with an underscore: `'_'`. # - # def initialize( product = "", price = "" ) - # @product = product - # @price = price - # end + # Backward Compatibility # - # def build - # b = binding - # # create and run templates, filling member data variables - # ERB.new(<<~'END_PRODUCT', trim_mode: "", eoutvar: "@product").result b - # <%= PRODUCT[:name] %> - # <%= PRODUCT[:desc] %> - # END_PRODUCT - # ERB.new(<<~'END_PRICE', trim_mode: "", eoutvar: "@price").result b - # <%= PRODUCT[:name] %> -- <%= PRODUCT[:cost] %> - # <%= PRODUCT[:desc] %> - # END_PRICE - # end - # end + # The calling sequence given above -- which is the one you should use -- + # is a simplified version of the complete formal calling sequence, + # which is: # - # # setup template data - # listings = Listings.new - # listings.build + # ``` + # ERB.new(string, + # safe_level=NOT_GIVEN, legacy_trim_mode=NOT_GIVEN, legacy_eoutvar=NOT_GIVEN, + # trim_mode: nil, eoutvar: '_erbout') + # ``` # - # puts listings.product + "\n" + listings.price + # The second, third, and fourth positional arguments (those in the second line above) are deprecated; + # this method issues warnings if they are given. # - # _Generates_ + # However, their values, if given, are handled thus: # - # Chicken Fried Steak - # A well massaged pattie, breaded and fried. + # - `safe_level`: ignored. + # - `legacy_trim_mode: overrides keyword argument `trim_mode`. + # - `legacy_eoutvar: overrides keyword argument `eoutvar`. # - # Chicken Fried Steak -- 9.95 - # A well massaged pattie, breaded and fried. + # [shorthand format]: rdoc-ref:ERB@Shorthand+Format+for+Execution+Tags # def initialize(str, safe_level=NOT_GIVEN, legacy_trim_mode=NOT_GIVEN, legacy_eoutvar=NOT_GIVEN, trim_mode: nil, eoutvar: '_erbout') # Complex initializer for $SAFE deprecation at [Feature #14256]. Use keyword arguments to pass trim_mode or eoutvar. diff --git a/lib/optparse.rb b/lib/optparse.rb index 06e33db1f533c4..ea6844b9558e5c 100644 --- a/lib/optparse.rb +++ b/lib/optparse.rb @@ -1855,7 +1855,7 @@ def permute(*argv, **keywords) # def permute!(argv = default_argv, **keywords) nonopts = [] - order!(argv, **keywords, &nonopts.method(:<<)) + order!(argv, **keywords) {|nonopt| nonopts << nonopt} argv[0, 0] = nonopts argv end @@ -1908,13 +1908,16 @@ def getopts(*args, symbolize_names: false, **keywords) single_options, *long_options = *args result = {} + setter = (symbolize_names ? + ->(name, val) {result[name.to_sym] = val} + : ->(name, val) {result[name] = val}) single_options.scan(/(.)(:)?/) do |opt, val| if val - result[opt] = nil + setter[opt, nil] define("-#{opt} VAL") else - result[opt] = false + setter[opt, false] define("-#{opt}") end end if single_options @@ -1923,16 +1926,16 @@ def getopts(*args, symbolize_names: false, **keywords) arg, desc = arg.split(';', 2) opt, val = arg.split(':', 2) if val - result[opt] = val.empty? ? nil : val + setter[opt, (val unless val.empty?)] define("--#{opt}=#{result[opt] || "VAL"}", *[desc].compact) else - result[opt] = false + setter[opt, false] define("--#{opt}", *[desc].compact) end end - parse_in_order(argv, result.method(:[]=), **keywords) - symbolize_names ? result.transform_keys(&:to_sym) : result + parse_in_order(argv, setter, **keywords) + result end # @@ -1982,7 +1985,7 @@ def complete(typ, opt, icase = false, *pat) # :nodoc: visit(:complete, typ, opt, icase, *pat) {|o, *sw| return sw} } exc = ambiguous ? AmbiguousOption : InvalidOption - raise exc.new(opt, additional: self.method(:additional_message).curry[typ]) + raise exc.new(opt, additional: proc {|o| additional_message(typ, o)}) end private :complete @@ -2273,9 +2276,10 @@ def recover(argv) argv end + DIR = File.join(__dir__, '') def self.filter_backtrace(array) unless $DEBUG - array.delete_if(&%r"\A#{Regexp.quote(__FILE__)}:"o.method(:=~)) + array.delete_if {|bt| bt.start_with?(DIR)} end array end diff --git a/lib/prism/polyfill/warn.rb b/lib/prism/polyfill/warn.rb index 560380d30807e8..76a4264623ba21 100644 --- a/lib/prism/polyfill/warn.rb +++ b/lib/prism/polyfill/warn.rb @@ -7,17 +7,14 @@ Kernel.prepend( Module.new { def warn(*msgs, uplevel: nil, category: nil) # :nodoc: - uplevel = - case uplevel - when nil - 1 - when Integer - uplevel + 1 - else - uplevel.to_int + 1 - end - - super(*msgs, uplevel: uplevel) + case uplevel + when nil + super(*msgs) + when Integer + super(*msgs, uplevel: uplevel + 1) + else + super(*msgs, uplevel: uplevel.to_int + 1) + end end } ) @@ -25,17 +22,14 @@ def warn(*msgs, uplevel: nil, category: nil) # :nodoc: Object.prepend( Module.new { def warn(*msgs, uplevel: nil, category: nil) # :nodoc: - uplevel = - case uplevel - when nil - 1 - when Integer - uplevel + 1 - else - uplevel.to_int + 1 - end - - super(*msgs, uplevel: uplevel) + case uplevel + when nil + super(*msgs) + when Integer + super(*msgs, uplevel: uplevel + 1) + else + super(*msgs, uplevel: uplevel.to_int + 1) + end end } ) diff --git a/lib/prism/prism.gemspec b/lib/prism/prism.gemspec index 4daa5113007086..65305d0ec1a244 100644 --- a/lib/prism/prism.gemspec +++ b/lib/prism/prism.gemspec @@ -2,7 +2,7 @@ Gem::Specification.new do |spec| spec.name = "prism" - spec.version = "1.4.0" + spec.version = "1.5.1" spec.authors = ["Shopify"] spec.email = ["ruby@shopify.com"] diff --git a/lib/prism/translation/parser.rb b/lib/prism/translation/parser.rb index a7888f77ecced1..1ad7a193c4f18b 100644 --- a/lib/prism/translation/parser.rb +++ b/lib/prism/translation/parser.rb @@ -19,6 +19,13 @@ module Translation # whitequark/parser gem's syntax tree. It inherits from the base parser for # the parser gem, and overrides the parse* methods to parse with prism and # then translate. + # + # Note that this version of the parser always parses using the latest + # version of Ruby syntax supported by Prism. If you want specific version + # support, use one of the version-specific subclasses, such as + # `Prism::Translation::Parser34`. If you want to parse using the same + # version of Ruby syntax as the currently running version of Ruby, use + # `Prism::Translation::ParserCurrent`. class Parser < ::Parser::Base Diagnostic = ::Parser::Diagnostic # :nodoc: private_constant :Diagnostic @@ -77,7 +84,7 @@ def initialize(builder = Prism::Translation::Parser::Builder.new, parser: Prism) end def version # :nodoc: - 34 + 35 end # The default encoding for Ruby files is UTF-8. diff --git a/lib/prism/translation/ruby_parser.rb b/lib/prism/translation/ruby_parser.rb index ac538a2e97ae43..2ca7da0bf2a5d5 100644 --- a/lib/prism/translation/ruby_parser.rb +++ b/lib/prism/translation/ruby_parser.rb @@ -152,7 +152,7 @@ def visit_assoc_splat_node(node) # ^^ # ``` def visit_back_reference_read_node(node) - s(node, :back_ref, node.name.name.delete_prefix("$").to_sym) + s(node, :back_ref, node.name.to_s.delete_prefix("$").to_sym) end # ``` diff --git a/namespace.c b/namespace.c index 55b7580c72e3ab..28e2c63a81940e 100644 --- a/namespace.c +++ b/namespace.c @@ -725,28 +725,32 @@ copy_ext_file(char *src_path, char *dst_path) #define IS_DLEXT(e) (strcmp((e), DLEXT) == 0) static void -fname_without_suffix(char *fname, char *rvalue) +fname_without_suffix(const char *fname, char *rvalue, size_t rsize) { - char *pos; - strcpy(rvalue, fname); - for (pos = rvalue + strlen(fname); pos > rvalue; pos--) { + size_t len = strlen(fname); + const char *pos; + for (pos = fname + len; pos > fname; pos--) { if (IS_SOEXT(pos) || IS_DLEXT(pos)) { - *pos = '\0'; - return; + len = pos - fname; + break; } + if (fname + len - pos > DLEXT_MAXLEN) break; } + if (len > rsize - 1) len = rsize - 1; + memcpy(rvalue, fname, len); + rvalue[len] = '\0'; } static void -escaped_basename(char *path, char *fname, char *rvalue) +escaped_basename(const char *path, const char *fname, char *rvalue, size_t rsize) { - char *pos, *leaf, *found; - leaf = path; + char *pos; + const char *leaf = path, *found; // `leaf + 1` looks uncomfortable (when leaf == path), but fname must not be the top-dir itself while ((found = strstr(leaf + 1, fname)) != NULL) { leaf = found; // find the last occurrence for the path like /etc/my-crazy-lib-dir/etc.so } - strcpy(rvalue, leaf); + strlcpy(rvalue, leaf, rsize); for (pos = rvalue; *pos; pos++) { if (isdirsep(*pos)) { *pos = '+'; @@ -762,8 +766,8 @@ rb_namespace_local_extension(VALUE namespace, VALUE fname, VALUE path) char *src_path = RSTRING_PTR(path), *fname_ptr = RSTRING_PTR(fname); rb_namespace_t *ns = rb_get_namespace_t(namespace); - fname_without_suffix(fname_ptr, fname2); - escaped_basename(src_path, fname2, basename); + fname_without_suffix(fname_ptr, fname2, sizeof(fname2)); + escaped_basename(src_path, fname2, basename, sizeof(basename)); wrote = sprint_ext_filename(ext_path, sizeof(ext_path), ns->ns_id, NAMESPACE_TMP_PREFIX, basename); if (wrote >= (int)sizeof(ext_path)) { diff --git a/numeric.c b/numeric.c index 89cff8a730fc9c..de5b02aaf9eb41 100644 --- a/numeric.c +++ b/numeric.c @@ -6455,7 +6455,7 @@ Init_Numeric(void) * * If the platform supports denormalized numbers, * there are numbers between zero and Float::MIN. - * 0.0.next_float returns the smallest positive floating point number + * +0.0.next_float+ returns the smallest positive floating point number * including denormalized numbers. */ rb_define_const(rb_cFloat, "MIN", DBL2NUM(DBL_MIN)); diff --git a/parse.y b/parse.y index a6c5e2e5b94f3f..6fac3b8ec79760 100644 --- a/parse.y +++ b/parse.y @@ -4198,6 +4198,11 @@ call_args : value_expr(command) $$ = NEW_LIST($1, &@$); /*% ripper: args_add!(args_new!, $:1) %*/ } + | def_endless_method(endless_command) + { + $$ = NEW_LIST($1, &@$); + /*% ripper: args_add!(args_new!, $:1) %*/ + } | args opt_block_arg { $$ = arg_blk_pass($1, $2); @@ -5892,7 +5897,8 @@ strings : string { if (!$1) { $$ = NEW_STR(STRING_NEW0(), &@$); - } else { + } + else { $$ = evstr2dstr(p, $1); } /*% ripper: $:1 %*/ @@ -6917,7 +6923,8 @@ parser_dispatch_delayed_token(struct parser_params *p, enum yytokentype t, int l if (p->keep_tokens) { /* p->delayed.token is freed by rb_parser_tokens_free */ parser_append_tokens(p, p->delayed.token, t, line); - } else { + } + else { rb_parser_string_free(p, p->delayed.token); } @@ -6980,6 +6987,16 @@ is_identchar(struct parser_params *p, const char *ptr, const char *MAYBE_UNUSED( return rb_enc_isalnum((unsigned char)*ptr, enc) || *ptr == '_' || !ISASCII(*ptr); } +static inline bool +peek_word_at(struct parser_params *p, const char *str, size_t len, int at) +{ + const char *ptr = p->lex.pcur + at; + if (lex_eol_ptr_n_p(p, ptr, len-1)) return false; + if (memcmp(ptr, str, len)) return false; + if (lex_eol_ptr_n_p(p, ptr, len)) return true; + return !is_identchar(p, ptr+len, p->lex.pend, p->enc); +} + static inline int parser_is_identchar(struct parser_params *p) { @@ -10551,7 +10568,24 @@ parser_yylex(struct parser_params *p) token_flush(p); } goto retry; + case 'a': + if (peek_word_at(p, "nd", 2, 0)) goto leading_logical; + goto bol; + case 'o': + if (peek_word_at(p, "r", 1, 0)) goto leading_logical; + goto bol; + case '|': + if (peek(p, '|')) goto leading_logical; + goto bol; case '&': + if (peek(p, '&')) { + leading_logical: + pushback(p, c); + dispatch_delayed_token(p, tIGNORED_NL); + cmd_state = FALSE; + goto retry; + } + /* fall through */ case '.': { dispatch_delayed_token(p, tIGNORED_NL); if (peek(p, '.') == (c == '&')) { @@ -10560,6 +10594,7 @@ parser_yylex(struct parser_params *p) goto retry; } } + bol: default: p->ruby_sourceline--; p->lex.nextline = p->lex.lastline; diff --git a/pathname_builtin.rb b/pathname_builtin.rb index 52b7003a64b8bf..16ed219ec38c06 100644 --- a/pathname_builtin.rb +++ b/pathname_builtin.rb @@ -321,7 +321,7 @@ def sub_ext(repl) end if File.dirname('A:') == 'A:.' # DOSish drive letter - # Regexp that matches an absoltute path. + # Regexp that matches an absolute path. ABSOLUTE_PATH = /\A(?:[A-Za-z]:|#{SEPARATOR_PAT})/ else ABSOLUTE_PATH = /\A#{SEPARATOR_PAT}/ diff --git a/prism/config.yml b/prism/config.yml index b37b98cbdfe252..3366b6235d003c 100644 --- a/prism/config.yml +++ b/prism/config.yml @@ -992,8 +992,19 @@ nodes: - name: constant type: node? kind: - - ConstantReadNode - ConstantPathNode + - ConstantReadNode + comment: | + Represents the optional constant preceding the Array + + foo in Bar[] + ^^^ + + foo in Bar[1, 2, 3] + ^^^ + + foo in Bar::Baz[1, 2, 3] + ^^^^^^^^ - name: requireds type: node[] kind: pattern expression @@ -2409,23 +2420,68 @@ nodes: - name: constant type: node? kind: - - ConstantReadNode - ConstantPathNode + - ConstantReadNode + comment: | + Represents the optional constant preceding the pattern + + foo in Foo(*bar, baz, *qux) + ^^^ - name: left type: node kind: SplatNode + comment: | + Represents the first wildcard node in the pattern. + + foo in *bar, baz, *qux + ^^^^ + + foo in Foo(*bar, baz, *qux) + ^^^^ - name: requireds type: node[] kind: pattern expression + comment: | + Represents the nodes in between the wildcards. + + foo in *bar, baz, *qux + ^^^ + + foo in Foo(*bar, baz, 1, *qux) + ^^^^^^ - name: right type: node kind: - SplatNode - on error: MissingNode + comment: | + Represents the second wildcard node in the pattern. + + foo in *bar, baz, *qux + ^^^^ + + foo in Foo(*bar, baz, *qux) + ^^^^ - name: opening_loc type: location? + comment: | + The location of the opening brace. + + foo in [*bar, baz, *qux] + ^ + + foo in Foo(*bar, baz, *qux) + ^ - name: closing_loc type: location? + comment: | + The location of the closing brace. + + foo in [*bar, baz, *qux] + ^ + + foo in Foo(*bar, baz, *qux) + ^ comment: | Represents a find pattern in pattern matching. @@ -2437,6 +2493,9 @@ nodes: foo in Foo(*bar, baz, *qux) ^^^^^^^^^^^^^^^^^^^^ + + foo => *bar, baz, *qux + ^^^^^^^^^^^^^^^ - name: FlipFlopNode flags: RangeFlags fields: @@ -2714,20 +2773,60 @@ nodes: - name: constant type: node? kind: - - ConstantReadNode - ConstantPathNode + - ConstantReadNode + comment: | + Represents the optional constant preceding the Hash. + + foo => Bar[a: 1, b: 2] + ^^^ + + foo => Bar::Baz[a: 1, b: 2] + ^^^^^^^^ - name: elements type: node[] kind: AssocNode + comment: | + Represents the explicit named hash keys and values. + + foo => { a: 1, b:, ** } + ^^^^^^^^ - name: rest type: node? kind: - AssocSplatNode - NoKeywordsParameterNode + comment: | + Represents the rest of the Hash keys and values. This can be named, unnamed, or explicitly forbidden via `**nil`, this last one results in a `NoKeywordsParameterNode`. + + foo => { a: 1, b:, **c } + ^^^ + + foo => { a: 1, b:, ** } + ^^ + + foo => { a: 1, b:, **nil } + ^^^^^ - name: opening_loc type: location? + comment: | + The location of the opening brace. + + foo => { a: 1 } + ^ + + foo => Bar[a: 1] + ^ - name: closing_loc type: location? + comment: | + The location of the closing brace. + + foo => { a: 1 } + ^ + + foo => Bar[a: 1] + ^ comment: | Represents a hash pattern in pattern matching. @@ -2736,6 +2835,12 @@ nodes: foo => { a: 1, b: 2, **c } ^^^^^^^^^^^^^^^^^^^ + + foo => Bar[a: 1, b: 2] + ^^^^^^^^^^^^^^^ + + foo in { a: 1, b: 2 } + ^^^^^^^^^^^^^^ - name: IfNode fields: - name: if_keyword_loc @@ -3388,6 +3493,9 @@ nodes: foo, bar = baz ^^^ ^^^ + + foo => baz + ^^^ - name: LocalVariableWriteNode fields: - name: name @@ -3478,11 +3586,65 @@ nodes: - name: value type: node kind: non-void expression + comment: | + Represents the left-hand side of the operator. + + foo => bar + ^^^ - name: pattern type: node kind: pattern expression + comment: | + Represents the right-hand side of the operator. The type of the node depends on the expression. + + Anything that looks like a local variable name (including `_`) will result in a `LocalVariableTargetNode`. + + foo => a # This is equivalent to writing `a = foo` + ^ + + Using an explicit `Array` or combining expressions with `,` will result in a `ArrayPatternNode`. This can be preceded by a constant. + + foo => [a] + ^^^ + + foo => a, b + ^^^^ + + foo => Bar[a, b] + ^^^^^^^^^ + + If the array pattern contains at least two wildcard matches, a `FindPatternNode` is created instead. + + foo => *, 1, *a + ^^^^^ + + Using an explicit `Hash` or a constant with square brackets and hash keys in the square brackets will result in a `HashPatternNode`. + + foo => { a: 1, b: } + + foo => Bar[a: 1, b:] + + foo => Bar[**] + + To use any variable that needs run time evaluation, pinning is required. This results in a `PinnedVariableNode` + + foo => ^a + ^^ + + Similar, any expression can be used with pinning. This results in a `PinnedExpressionNode`. + + foo => ^(a + 1) + + Anything else will result in the regular node for that expression, for example a `ConstantReadNode`. + + foo => CONST - name: operator_loc type: location + comment: | + The location of the operator. + + foo => bar + ^^ comment: | Represents the use of the `=>` operator. @@ -3912,12 +4074,32 @@ nodes: - name: expression type: node kind: non-void expression + comment: | + The expression used in the pinned expression + + foo in ^(bar) + ^^^ - name: operator_loc type: location + comment: | + The location of the `^` operator + + foo in ^(bar) + ^ - name: lparen_loc type: location + comment: | + The location of the opening parenthesis. + + foo in ^(bar) + ^ - name: rparen_loc type: location + comment: | + The location of the closing parenthesis. + + foo in ^(bar) + ^ comment: | Represents the use of the `^` operator for pinning an expression in a pattern matching expression. @@ -3936,8 +4118,18 @@ nodes: - NumberedReferenceReadNode # foo in ^$1 - ItLocalVariableReadNode # proc { 1 in ^it } - on error: MissingNode # foo in ^Bar + comment: | + The variable used in the pinned expression + + foo in ^bar + ^^^ - name: operator_loc type: location + comment: | + The location of the `^` operator + + foo in ^bar + ^ comment: | Represents the use of the `^` operator for pinning a variable in a pattern matching expression. diff --git a/prism/extension.h b/prism/extension.h index 506da2fd6f079e..b18e770d9213f7 100644 --- a/prism/extension.h +++ b/prism/extension.h @@ -1,7 +1,7 @@ #ifndef PRISM_EXT_NODE_H #define PRISM_EXT_NODE_H -#define EXPECTED_PRISM_VERSION "1.4.0" +#define EXPECTED_PRISM_VERSION "1.5.1" #include #include diff --git a/prism/options.h b/prism/options.h index 092fda4f07878a..1a92c470f1ea7d 100644 --- a/prism/options.h +++ b/prism/options.h @@ -237,6 +237,8 @@ static const uint8_t PM_OPTIONS_COMMAND_LINE_X = 0x20; * @param shebang_callback The shebang callback to set. * @param shebang_callback_data Any additional data that should be passed along * to the callback. + * + * \public \memberof pm_options */ PRISM_EXPORTED_FUNCTION void pm_options_shebang_callback_set(pm_options_t *options, pm_options_shebang_callback_t shebang_callback, void *shebang_callback_data); @@ -245,6 +247,8 @@ PRISM_EXPORTED_FUNCTION void pm_options_shebang_callback_set(pm_options_t *optio * * @param options The options struct to set the filepath on. * @param filepath The filepath to set. + * + * \public \memberof pm_options */ PRISM_EXPORTED_FUNCTION void pm_options_filepath_set(pm_options_t *options, const char *filepath); @@ -253,6 +257,8 @@ PRISM_EXPORTED_FUNCTION void pm_options_filepath_set(pm_options_t *options, cons * * @param options The options struct to set the line on. * @param line The line to set. + * + * \public \memberof pm_options */ PRISM_EXPORTED_FUNCTION void pm_options_line_set(pm_options_t *options, int32_t line); @@ -261,6 +267,8 @@ PRISM_EXPORTED_FUNCTION void pm_options_line_set(pm_options_t *options, int32_t * * @param options The options struct to set the encoding on. * @param encoding The encoding to set. + * + * \public \memberof pm_options */ PRISM_EXPORTED_FUNCTION void pm_options_encoding_set(pm_options_t *options, const char *encoding); @@ -269,6 +277,8 @@ PRISM_EXPORTED_FUNCTION void pm_options_encoding_set(pm_options_t *options, cons * * @param options The options struct to set the encoding_locked value on. * @param encoding_locked The encoding_locked value to set. + * + * \public \memberof pm_options */ PRISM_EXPORTED_FUNCTION void pm_options_encoding_locked_set(pm_options_t *options, bool encoding_locked); @@ -277,6 +287,8 @@ PRISM_EXPORTED_FUNCTION void pm_options_encoding_locked_set(pm_options_t *option * * @param options The options struct to set the frozen string literal value on. * @param frozen_string_literal The frozen string literal value to set. + * + * \public \memberof pm_options */ PRISM_EXPORTED_FUNCTION void pm_options_frozen_string_literal_set(pm_options_t *options, bool frozen_string_literal); @@ -285,6 +297,8 @@ PRISM_EXPORTED_FUNCTION void pm_options_frozen_string_literal_set(pm_options_t * * * @param options The options struct to set the command line option on. * @param command_line The command_line value to set. + * + * \public \memberof pm_options */ PRISM_EXPORTED_FUNCTION void pm_options_command_line_set(pm_options_t *options, uint8_t command_line); @@ -297,6 +311,8 @@ PRISM_EXPORTED_FUNCTION void pm_options_command_line_set(pm_options_t *options, * @param version The version to set. * @param length The length of the version string. * @return Whether or not the version was parsed successfully. + * + * \public \memberof pm_options */ PRISM_EXPORTED_FUNCTION bool pm_options_version_set(pm_options_t *options, const char *version, size_t length); @@ -305,6 +321,8 @@ PRISM_EXPORTED_FUNCTION bool pm_options_version_set(pm_options_t *options, const * * @param options The options struct to set the main script value on. * @param main_script The main script value to set. + * + * \public \memberof pm_options */ PRISM_EXPORTED_FUNCTION void pm_options_main_script_set(pm_options_t *options, bool main_script); @@ -313,6 +331,8 @@ PRISM_EXPORTED_FUNCTION void pm_options_main_script_set(pm_options_t *options, b * * @param options The options struct to set the partial script value on. * @param partial_script The partial script value to set. + * + * \public \memberof pm_options */ PRISM_EXPORTED_FUNCTION void pm_options_partial_script_set(pm_options_t *options, bool partial_script); @@ -321,6 +341,8 @@ PRISM_EXPORTED_FUNCTION void pm_options_partial_script_set(pm_options_t *options * * @param options The options struct to set the freeze value on. * @param freeze The freeze value to set. + * + * \public \memberof pm_options */ PRISM_EXPORTED_FUNCTION void pm_options_freeze_set(pm_options_t *options, bool freeze); @@ -330,6 +352,8 @@ PRISM_EXPORTED_FUNCTION void pm_options_freeze_set(pm_options_t *options, bool f * @param options The options struct to initialize the scopes array on. * @param scopes_count The number of scopes to allocate. * @return Whether or not the scopes array was initialized successfully. + * + * \public \memberof pm_options */ PRISM_EXPORTED_FUNCTION bool pm_options_scopes_init(pm_options_t *options, size_t scopes_count); @@ -339,6 +363,8 @@ PRISM_EXPORTED_FUNCTION bool pm_options_scopes_init(pm_options_t *options, size_ * @param options The options struct to get the scope from. * @param index The index of the scope to get. * @return A pointer to the scope at the given index. + * + * \public \memberof pm_options */ PRISM_EXPORTED_FUNCTION const pm_options_scope_t * pm_options_scope_get(const pm_options_t *options, size_t index); @@ -349,6 +375,8 @@ PRISM_EXPORTED_FUNCTION const pm_options_scope_t * pm_options_scope_get(const pm * @param scope The scope struct to initialize. * @param locals_count The number of locals to allocate. * @return Whether or not the scope was initialized successfully. + * + * \public \memberof pm_options */ PRISM_EXPORTED_FUNCTION bool pm_options_scope_init(pm_options_scope_t *scope, size_t locals_count); @@ -358,6 +386,8 @@ PRISM_EXPORTED_FUNCTION bool pm_options_scope_init(pm_options_scope_t *scope, si * @param scope The scope struct to get the local from. * @param index The index of the local to get. * @return A pointer to the local at the given index. + * + * \public \memberof pm_options */ PRISM_EXPORTED_FUNCTION const pm_string_t * pm_options_scope_local_get(const pm_options_scope_t *scope, size_t index); @@ -366,6 +396,8 @@ PRISM_EXPORTED_FUNCTION const pm_string_t * pm_options_scope_local_get(const pm_ * * @param scope The scope struct to set the forwarding on. * @param forwarding The forwarding value to set. + * + * \public \memberof pm_options */ PRISM_EXPORTED_FUNCTION void pm_options_scope_forwarding_set(pm_options_scope_t *scope, uint8_t forwarding); @@ -373,6 +405,8 @@ PRISM_EXPORTED_FUNCTION void pm_options_scope_forwarding_set(pm_options_scope_t * Free the internal memory associated with the options. * * @param options The options struct whose internal memory should be freed. + * + * \public \memberof pm_options */ PRISM_EXPORTED_FUNCTION void pm_options_free(pm_options_t *options); diff --git a/prism/prism.c b/prism/prism.c index 2f7ce0b9866438..2e202c37456ea5 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -5279,10 +5279,6 @@ pm_interpolated_string_node_append(pm_interpolated_string_node_t *node, pm_node_ switch (PM_NODE_TYPE(part)) { case PM_STRING_NODE: - // If inner string is not frozen, clear flags for this string - if (!PM_NODE_FLAG_P(part, PM_STRING_FLAGS_FROZEN)) { - CLEAR_FLAGS(node); - } part->flags = (pm_node_flags_t) ((part->flags | PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN) & ~PM_STRING_FLAGS_MUTABLE); break; case PM_INTERPOLATED_STRING_NODE: @@ -10834,14 +10830,37 @@ parser_lex(pm_parser_t *parser) { following = next_newline(following, parser->end - following); } - // If the lex state was ignored, or we hit a '.' or a '&.', - // we will lex the ignored newline + // If the lex state was ignored, we will lex the + // ignored newline. + if (lex_state_ignored_p(parser)) { + if (!lexed_comment) parser_lex_ignored_newline(parser); + lexed_comment = false; + goto lex_next_token; + } + + // If we hit a '.' or a '&.' we will lex the ignored + // newline. + if (following && ( + (peek_at(parser, following) == '.') || + (peek_at(parser, following) == '&' && peek_at(parser, following + 1) == '.') + )) { + if (!lexed_comment) parser_lex_ignored_newline(parser); + lexed_comment = false; + goto lex_next_token; + } + + + // If we are parsing as CRuby 3.5 or later and we + // hit a '&&' or a '||' then we will lex the ignored + // newline. if ( - lex_state_ignored_p(parser) || - (following && ( - (peek_at(parser, following) == '.') || - (peek_at(parser, following) == '&' && peek_at(parser, following + 1) == '.') - )) + (parser->version >= PM_OPTIONS_VERSION_CRUBY_3_5) && + following && ( + (peek_at(parser, following) == '&' && peek_at(parser, following + 1) == '&') || + (peek_at(parser, following) == '|' && peek_at(parser, following + 1) == '|') || + (peek_at(parser, following) == 'a' && peek_at(parser, following + 1) == 'n' && peek_at(parser, following + 2) == 'd' && !char_is_identifier(parser, following + 3, parser->end - (following + 3))) || + (peek_at(parser, following) == 'o' && peek_at(parser, following + 1) == 'r' && !char_is_identifier(parser, following + 2, parser->end - (following + 2))) + ) ) { if (!lexed_comment) parser_lex_ignored_newline(parser); lexed_comment = false; @@ -10881,6 +10900,63 @@ parser_lex(pm_parser_t *parser) { parser->next_start = NULL; LEX(PM_TOKEN_AMPERSAND_DOT); } + + if (parser->version >= PM_OPTIONS_VERSION_CRUBY_3_5) { + // If we hit an && then we are in a logical chain + // and we need to return the logical operator. + if (peek_at(parser, next_content) == '&' && peek_at(parser, next_content + 1) == '&') { + if (!lexed_comment) parser_lex_ignored_newline(parser); + lex_state_set(parser, PM_LEX_STATE_BEG); + parser->current.start = next_content; + parser->current.end = next_content + 2; + parser->next_start = NULL; + LEX(PM_TOKEN_AMPERSAND_AMPERSAND); + } + + // If we hit a || then we are in a logical chain and + // we need to return the logical operator. + if (peek_at(parser, next_content) == '|' && peek_at(parser, next_content + 1) == '|') { + if (!lexed_comment) parser_lex_ignored_newline(parser); + lex_state_set(parser, PM_LEX_STATE_BEG); + parser->current.start = next_content; + parser->current.end = next_content + 2; + parser->next_start = NULL; + LEX(PM_TOKEN_PIPE_PIPE); + } + + // If we hit an 'and' then we are in a logical chain + // and we need to return the logical operator. + if ( + peek_at(parser, next_content) == 'a' && + peek_at(parser, next_content + 1) == 'n' && + peek_at(parser, next_content + 2) == 'd' && + !char_is_identifier(parser, next_content + 3, parser->end - (next_content + 3)) + ) { + if (!lexed_comment) parser_lex_ignored_newline(parser); + lex_state_set(parser, PM_LEX_STATE_BEG); + parser->current.start = next_content; + parser->current.end = next_content + 3; + parser->next_start = NULL; + parser->command_start = true; + LEX(PM_TOKEN_KEYWORD_AND); + } + + // If we hit a 'or' then we are in a logical chain + // and we need to return the logical operator. + if ( + peek_at(parser, next_content) == 'o' && + peek_at(parser, next_content + 1) == 'r' && + !char_is_identifier(parser, next_content + 2, parser->end - (next_content + 2)) + ) { + if (!lexed_comment) parser_lex_ignored_newline(parser); + lex_state_set(parser, PM_LEX_STATE_BEG); + parser->current.start = next_content; + parser->current.end = next_content + 2; + parser->next_start = NULL; + parser->command_start = true; + LEX(PM_TOKEN_KEYWORD_OR); + } + } } // At this point we know this is a regular newline, and we can set the @@ -18415,20 +18491,28 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b return (pm_node_t *) node; } case PM_TOKEN_CHARACTER_LITERAL: { - parser_lex(parser); - - pm_token_t opening = parser->previous; - opening.type = PM_TOKEN_STRING_BEGIN; - opening.end = opening.start + 1; - - pm_token_t content = parser->previous; - content.type = PM_TOKEN_STRING_CONTENT; - content.start = content.start + 1; - pm_token_t closing = not_provided(parser); - pm_node_t *node = (pm_node_t *) pm_string_node_create_current_string(parser, &opening, &content, &closing); + pm_node_t *node = (pm_node_t *) pm_string_node_create_current_string( + parser, + &(pm_token_t) { + .type = PM_TOKEN_STRING_BEGIN, + .start = parser->current.start, + .end = parser->current.start + 1 + }, + &(pm_token_t) { + .type = PM_TOKEN_STRING_CONTENT, + .start = parser->current.start + 1, + .end = parser->current.end + }, + &closing + ); + pm_node_flag_set(node, parse_unescaped_encoding(parser)); + // Skip past the character literal here, since now we have handled + // parser->explicit_encoding correctly. + parser_lex(parser); + // Characters can be followed by strings in which case they are // automatically concatenated. if (match1(parser, PM_TOKEN_STRING_BEGIN)) { @@ -19505,13 +19589,13 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_do_loop_stack_push(parser, false); statements = (pm_node_t *) pm_statements_node_create(parser); - // In endless method bodies, we need to handle command calls carefully. - // We want to allow command calls in assignment context but maintain - // the same binding power to avoid changing how operators are parsed. - // Note that we're intentionally NOT allowing code like `private def foo = puts "Hello"` - // because the original parser, parse.y, can't handle it and we want to maintain the same behavior - bool allow_command_call = (binding_power == PM_BINDING_POWER_ASSIGNMENT) || - (binding_power < PM_BINDING_POWER_COMPOSITION); + bool allow_command_call; + if (parser->version >= PM_OPTIONS_VERSION_CRUBY_3_5) { + allow_command_call = accepts_command_call; + } else { + // Allow `def foo = puts "Hello"` but not `private def foo = puts "Hello"` + allow_command_call = binding_power == PM_BINDING_POWER_ASSIGNMENT || binding_power < PM_BINDING_POWER_COMPOSITION; + } pm_node_t *statement = parse_expression(parser, PM_BINDING_POWER_DEFINED + 1, allow_command_call, false, PM_ERR_DEF_ENDLESS, (uint16_t) (depth + 1)); diff --git a/prism/prism.h b/prism/prism.h index a6f22f1a5ad39f..dc31f26e786a5c 100644 --- a/prism/prism.h +++ b/prism/prism.h @@ -49,10 +49,15 @@ PRISM_EXPORTED_FUNCTION const char * pm_version(void); /** * Initialize a parser with the given start and end pointers. * + * The resulting parser must eventually be freed with `pm_parser_free()`. + * * @param parser The parser to initialize. * @param source The source to parse. * @param size The size of the source. - * @param options The optional options to use when parsing. + * @param options The optional options to use when parsing. These options must + * live for the whole lifetime of this parser. + * + * \public \memberof pm_parser */ PRISM_EXPORTED_FUNCTION void pm_parser_init(pm_parser_t *parser, const uint8_t *source, size_t size, const pm_options_t *options); @@ -62,13 +67,20 @@ PRISM_EXPORTED_FUNCTION void pm_parser_init(pm_parser_t *parser, const uint8_t * * * @param parser The parser to register the callback with. * @param callback The callback to register. + * + * \public \memberof pm_parser */ PRISM_EXPORTED_FUNCTION void pm_parser_register_encoding_changed_callback(pm_parser_t *parser, pm_encoding_changed_callback_t callback); /** * Free any memory associated with the given parser. * + * This does not free the `pm_options_t` object that was used to initialize the + * parser. + * * @param parser The parser to free. + * + * \public \memberof pm_parser */ PRISM_EXPORTED_FUNCTION void pm_parser_free(pm_parser_t *parser); @@ -77,11 +89,13 @@ PRISM_EXPORTED_FUNCTION void pm_parser_free(pm_parser_t *parser); * * @param parser The parser to use. * @return The AST representing the source. + * + * \public \memberof pm_parser */ PRISM_EXPORTED_FUNCTION pm_node_t * pm_parse(pm_parser_t *parser); /** - * This function is used in pm_parse_stream to retrieve a line of input from a + * This function is used in pm_parse_stream() to retrieve a line of input from a * stream. It closely mirrors that of fgets so that fgets can be used as the * default implementation. */ @@ -104,6 +118,8 @@ typedef int (pm_parse_stream_feof_t)(void *stream); * @param stream_feof The function to use to determine if the stream has hit eof. * @param options The optional options to use when parsing. * @return The AST representing the source. + * + * \public \memberof pm_parser */ PRISM_EXPORTED_FUNCTION pm_node_t * pm_parse_stream(pm_parser_t *parser, pm_buffer_t *buffer, void *stream, pm_parse_stream_fgets_t *stream_fgets, pm_parse_stream_feof_t *stream_feof, const pm_options_t *options); @@ -316,10 +332,10 @@ PRISM_EXPORTED_FUNCTION pm_string_query_t pm_string_query_method_name(const uint * to want to use and be aware of are: * * * `pm_parser_t` - the main parser structure - * * `pm_parser_init` - initialize a parser - * * `pm_parse` - parse and return the root node - * * `pm_node_destroy` - deallocate the root node returned by `pm_parse` - * * `pm_parser_free` - free the internal memory of the parser + * * `pm_parser_init()` - initialize a parser + * * `pm_parse()` - parse and return the root node + * * `pm_node_destroy()` - deallocate the root node returned by `pm_parse()` + * * `pm_parser_free()` - free the internal memory of the parser * * Putting all of this together would look something like: * @@ -336,8 +352,8 @@ PRISM_EXPORTED_FUNCTION pm_string_query_t pm_string_query_method_name(const uint * } * ``` * - * All of the nodes "inherit" from `pm_node_t` by embedding those structures as - * their first member. This means you can downcast and upcast any node in the + * All of the nodes "inherit" from `pm_node_t` by embedding those structures + * as their first member. This means you can downcast and upcast any node in the * tree to a `pm_node_t`. * * @section serializing Serializing @@ -349,9 +365,9 @@ PRISM_EXPORTED_FUNCTION pm_string_query_t pm_string_query_method_name(const uint * use and be aware of are: * * * `pm_buffer_t` - a small buffer object that will hold the serialized AST - * * `pm_buffer_free` - free the memory associated with the buffer - * * `pm_serialize` - serialize the AST into a buffer - * * `pm_serialize_parse` - parse and serialize the AST into a buffer + * * `pm_buffer_free()` - free the memory associated with the buffer + * * `pm_serialize()` - serialize the AST into a buffer + * * `pm_serialize_parse()` - parse and serialize the AST into a buffer * * Putting all of this together would look something like: * @@ -369,7 +385,7 @@ PRISM_EXPORTED_FUNCTION pm_string_query_t pm_string_query_method_name(const uint * @section inspecting Inspecting * * Prism provides the ability to inspect the AST by pretty-printing nodes. You - * can do this with the `pm_prettyprint` function, which you would use like: + * can do this with the `pm_prettyprint()` function, which you would use like: * * ```c * void prettyprint(const uint8_t *source, size_t length) { diff --git a/prism/regexp.h b/prism/regexp.h index c0b3163e93b649..5366b5a5a0d359 100644 --- a/prism/regexp.h +++ b/prism/regexp.h @@ -17,12 +17,12 @@ #include /** - * This callback is called when a named capture group is found. + * This callback is called by pm_regexp_parse() when a named capture group is found. */ typedef void (*pm_regexp_name_callback_t)(const pm_string_t *name, void *data); /** - * This callback is called when a parse error is found. + * This callback is called by pm_regexp_parse() when a parse error is found. */ typedef void (*pm_regexp_error_callback_t)(const uint8_t *start, const uint8_t *end, const char *message, void *data); diff --git a/prism/srcs.mk b/prism/srcs.mk index aa5c0fa2b5ee33..ff4ddff0722e32 100644 --- a/prism/srcs.mk +++ b/prism/srcs.mk @@ -4,7 +4,9 @@ PRISM_CONFIG = $(PRISM_SRCDIR)/config.yml srcs uncommon.mk: prism/.srcs.mk.time -prism/.srcs.mk.time: +prism/.srcs.mk.time: $(order_only) $(PRISM_BUILD_DIR)/.time +prism/$(HAVE_BASERUBY:no=.srcs.mk.time): + touch $@ prism/$(HAVE_BASERUBY:yes=.srcs.mk.time): \ $(PRISM_SRCDIR)/templates/template.rb \ $(PRISM_SRCDIR)/srcs.mk.in diff --git a/prism/srcs.mk.in b/prism/srcs.mk.in index 655de155d5e97b..e337eff8ea00e5 100644 --- a/prism/srcs.mk.in +++ b/prism/srcs.mk.in @@ -12,7 +12,9 @@ PRISM_CONFIG = $(PRISM_SRCDIR)/config.yml srcs <%=%><%=mk%>: prism/.srcs.mk.time -prism/.srcs.mk.time: +prism/.srcs.mk.time: $(order_only) $(PRISM_BUILD_DIR)/.time +prism/$(HAVE_BASERUBY:no=.srcs.mk.time): + touch $@ prism/$(HAVE_BASERUBY:yes=.srcs.mk.time): \ $(PRISM_SRCDIR)/templates/template.rb \ $(PRISM_SRCDIR)/<%=%><%=script%> diff --git a/prism/templates/lib/prism/serialize.rb.erb b/prism/templates/lib/prism/serialize.rb.erb index 104b60f4842015..366878f709250d 100644 --- a/prism/templates/lib/prism/serialize.rb.erb +++ b/prism/templates/lib/prism/serialize.rb.erb @@ -10,11 +10,11 @@ module Prism # The minor version of prism that we are expecting to find in the serialized # strings. - MINOR_VERSION = 4 + MINOR_VERSION = 5 # The patch version of prism that we are expecting to find in the serialized # strings. - PATCH_VERSION = 0 + PATCH_VERSION = 1 # Deserialize the dumped output from a request to parse or parse_file. # diff --git a/prism/util/pm_buffer.h b/prism/util/pm_buffer.h index f3c20ab2a5ad10..cb80f8b3ce7d01 100644 --- a/prism/util/pm_buffer.h +++ b/prism/util/pm_buffer.h @@ -51,6 +51,8 @@ bool pm_buffer_init_capacity(pm_buffer_t *buffer, size_t capacity); * * @param buffer The buffer to initialize. * @returns True if the buffer was initialized successfully, false otherwise. + * + * \public \memberof pm_buffer_t */ PRISM_EXPORTED_FUNCTION bool pm_buffer_init(pm_buffer_t *buffer); @@ -59,6 +61,8 @@ PRISM_EXPORTED_FUNCTION bool pm_buffer_init(pm_buffer_t *buffer); * * @param buffer The buffer to get the value of. * @returns The value of the buffer. + * + * \public \memberof pm_buffer_t */ PRISM_EXPORTED_FUNCTION char * pm_buffer_value(const pm_buffer_t *buffer); @@ -67,6 +71,8 @@ PRISM_EXPORTED_FUNCTION char * pm_buffer_value(const pm_buffer_t *buffer); * * @param buffer The buffer to get the length of. * @returns The length of the buffer. + * + * \public \memberof pm_buffer_t */ PRISM_EXPORTED_FUNCTION size_t pm_buffer_length(const pm_buffer_t *buffer); @@ -222,6 +228,8 @@ void pm_buffer_insert(pm_buffer_t *buffer, size_t index, const char *value, size * Free the memory associated with the buffer. * * @param buffer The buffer to free. + * + * \public \memberof pm_buffer_t */ PRISM_EXPORTED_FUNCTION void pm_buffer_free(pm_buffer_t *buffer); diff --git a/prism/util/pm_integer.h b/prism/util/pm_integer.h index a9e2966703408b..304665e6205dea 100644 --- a/prism/util/pm_integer.h +++ b/prism/util/pm_integer.h @@ -112,6 +112,8 @@ void pm_integers_reduce(pm_integer_t *numerator, pm_integer_t *denominator); * * @param buffer The buffer to append the string to. * @param integer The integer to convert to a string. + * + * \public \memberof pm_integer_t */ PRISM_EXPORTED_FUNCTION void pm_integer_string(pm_buffer_t *buffer, const pm_integer_t *integer); @@ -120,6 +122,8 @@ PRISM_EXPORTED_FUNCTION void pm_integer_string(pm_buffer_t *buffer, const pm_int * the integer exceeds the size of a single node in the linked list. * * @param integer The integer to free. + * + * \public \memberof pm_integer_t */ PRISM_EXPORTED_FUNCTION void pm_integer_free(pm_integer_t *integer); diff --git a/prism/util/pm_list.h b/prism/util/pm_list.h index 3512dee979aa52..f544bb2943d3ff 100644 --- a/prism/util/pm_list.h +++ b/prism/util/pm_list.h @@ -68,6 +68,8 @@ typedef struct { * * @param list The list to check. * @return True if the given list is empty, otherwise false. + * + * \public \memberof pm_list_t */ PRISM_EXPORTED_FUNCTION bool pm_list_empty_p(pm_list_t *list); @@ -76,6 +78,8 @@ PRISM_EXPORTED_FUNCTION bool pm_list_empty_p(pm_list_t *list); * * @param list The list to check. * @return The size of the list. + * + * \public \memberof pm_list_t */ PRISM_EXPORTED_FUNCTION size_t pm_list_size(pm_list_t *list); @@ -91,6 +95,8 @@ void pm_list_append(pm_list_t *list, pm_list_node_t *node); * Deallocate the internal state of the given list. * * @param list The list to free. + * + * \public \memberof pm_list_t */ PRISM_EXPORTED_FUNCTION void pm_list_free(pm_list_t *list); diff --git a/prism/util/pm_string.c b/prism/util/pm_string.c index 75422fbdf2c41d..cf79885fddbdff 100644 --- a/prism/util/pm_string.c +++ b/prism/util/pm_string.c @@ -182,7 +182,7 @@ pm_string_mapped_init(pm_string_t *string, const char *filepath) { if (size == 0) { close(fd); - const uint8_t source[] = ""; + static const uint8_t source[] = ""; *string = (pm_string_t) { .type = PM_STRING_CONSTANT, .source = source, .length = 0 }; return PM_STRING_INIT_SUCCESS; } @@ -278,7 +278,7 @@ pm_string_file_init(pm_string_t *string, const char *filepath) { size_t size = (size_t) sb.st_size; if (size == 0) { close(fd); - const uint8_t source[] = ""; + static const uint8_t source[] = ""; *string = (pm_string_t) { .type = PM_STRING_CONSTANT, .source = source, .length = 0 }; return PM_STRING_INIT_SUCCESS; } diff --git a/prism/util/pm_string.h b/prism/util/pm_string.h index f99f1abdf38c9b..d8456ff2947eb8 100644 --- a/prism/util/pm_string.h +++ b/prism/util/pm_string.h @@ -45,11 +45,11 @@ typedef struct { /** This is a slice of another string, and should not be freed. */ PM_STRING_SHARED, - /** This string owns its memory, and should be freed using `pm_string_free`. */ + /** This string owns its memory, and should be freed using `pm_string_free()`. */ PM_STRING_OWNED, #ifdef PRISM_HAS_MMAP - /** This string is a memory-mapped file, and should be freed using `pm_string_free`. */ + /** This string is a memory-mapped file, and should be freed using `pm_string_free()`. */ PM_STRING_MAPPED #endif } type; @@ -130,6 +130,8 @@ typedef enum { * @param string The string to initialize. * @param filepath The filepath to read. * @return The success of the read, indicated by the value of the enum. + * + * \public \memberof pm_string_t */ PRISM_EXPORTED_FUNCTION pm_string_init_result_t pm_string_mapped_init(pm_string_t *string, const char *filepath); @@ -141,6 +143,8 @@ PRISM_EXPORTED_FUNCTION pm_string_init_result_t pm_string_mapped_init(pm_string_ * @param string The string to initialize. * @param filepath The filepath to read. * @return The success of the read, indicated by the value of the enum. + * + * \public \memberof pm_string_t */ PRISM_EXPORTED_FUNCTION pm_string_init_result_t pm_string_file_init(pm_string_t *string, const char *filepath); @@ -169,6 +173,8 @@ int pm_string_compare(const pm_string_t *left, const pm_string_t *right); * * @param string The string to get the length of. * @return The length of the string. + * + * \public \memberof pm_string_t */ PRISM_EXPORTED_FUNCTION size_t pm_string_length(const pm_string_t *string); @@ -177,6 +183,8 @@ PRISM_EXPORTED_FUNCTION size_t pm_string_length(const pm_string_t *string); * * @param string The string to get the start pointer of. * @return The start pointer of the string. + * + * \public \memberof pm_string_t */ PRISM_EXPORTED_FUNCTION const uint8_t * pm_string_source(const pm_string_t *string); @@ -184,6 +192,8 @@ PRISM_EXPORTED_FUNCTION const uint8_t * pm_string_source(const pm_string_t *stri * Free the associated memory of the given string. * * @param string The string to free. + * + * \public \memberof pm_string_t */ PRISM_EXPORTED_FUNCTION void pm_string_free(pm_string_t *string); diff --git a/prism/version.h b/prism/version.h index 0a2a8c8fce5a25..697ba0647df1ec 100644 --- a/prism/version.h +++ b/prism/version.h @@ -14,16 +14,16 @@ /** * The minor version of the Prism library as an int. */ -#define PRISM_VERSION_MINOR 4 +#define PRISM_VERSION_MINOR 5 /** * The patch version of the Prism library as an int. */ -#define PRISM_VERSION_PATCH 0 +#define PRISM_VERSION_PATCH 1 /** * The version of the Prism library as a constant string. */ -#define PRISM_VERSION "1.4.0" +#define PRISM_VERSION "1.5.1" #endif diff --git a/prism_compile.c b/prism_compile.c index 37909e49e01444..578e6f240f98dc 100644 --- a/prism_compile.c +++ b/prism_compile.c @@ -1,4 +1,5 @@ #include "prism.h" +#include "ruby/version.h" /** * This compiler defines its own concept of the location of a node. We do this @@ -6739,6 +6740,12 @@ pm_compile_scope_node(rb_iseq_t *iseq, pm_scope_node_t *scope_node, const pm_nod body->param.flags.has_lead = true; } + // Fill in the anonymous `it` parameter, if it exists + if (scope_node->parameters && PM_NODE_TYPE_P(scope_node->parameters, PM_IT_PARAMETERS_NODE)) { + body->param.lead_num = 1; + body->param.flags.has_lead = true; + } + //********END OF STEP 3********** //********STEP 4********** @@ -10620,7 +10627,26 @@ pm_parse_errors_format_line(const pm_parser_t *parser, const pm_newline_list_t * // Here we determine if we should truncate the end of the line. bool truncate_end = false; if ((column_end != 0) && ((end - (start + column_end)) >= PM_ERROR_TRUNCATE)) { - end = start + column_end + PM_ERROR_TRUNCATE; + const uint8_t *end_candidate = start + column_end + PM_ERROR_TRUNCATE; + + for (const uint8_t *ptr = start; ptr < end_candidate;) { + size_t char_width = parser->encoding->char_width(ptr, parser->end - ptr); + + // If we failed to decode a character, then just bail out and + // truncate at the fixed width. + if (char_width == 0) break; + + // If this next character would go past the end candidate, + // then we need to truncate before it. + if (ptr + char_width > end_candidate) { + end_candidate = ptr; + break; + } + + ptr += char_width; + } + + end = end_candidate; truncate_end = true; } @@ -11352,6 +11378,8 @@ pm_parse_file(pm_parse_result_t *result, VALUE filepath, VALUE *script_lines) pm_options_filepath_set(&result->options, RSTRING_PTR(filepath)); RB_GC_GUARD(filepath); + pm_options_version_for_current_ruby_set(&result->options); + pm_parser_init(&result->parser, pm_string_source(&result->input), pm_string_length(&result->input), &result->options); pm_node_t *node = pm_parse(&result->parser); @@ -11410,6 +11438,8 @@ pm_parse_string(pm_parse_result_t *result, VALUE source, VALUE filepath, VALUE * pm_options_filepath_set(&result->options, RSTRING_PTR(filepath)); RB_GC_GUARD(filepath); + pm_options_version_for_current_ruby_set(&result->options); + pm_parser_init(&result->parser, pm_string_source(&result->input), pm_string_length(&result->input), &result->options); pm_node_t *node = pm_parse(&result->parser); @@ -11492,6 +11522,13 @@ pm_parse_stdin(pm_parse_result_t *result) return pm_parse_process(result, node, NULL); } +#define PM_VERSION_FOR_RELEASE(major, minor) PM_VERSION_FOR_RELEASE_IMPL(major, minor) +#define PM_VERSION_FOR_RELEASE_IMPL(major, minor) PM_OPTIONS_VERSION_CRUBY_##major##_##minor + +void pm_options_version_for_current_ruby_set(pm_options_t *options) { + options->version = PM_VERSION_FOR_RELEASE(RUBY_API_VERSION_MAJOR, RUBY_API_VERSION_MINOR); +} + #undef NEW_ISEQ #define NEW_ISEQ OLD_ISEQ diff --git a/prism_compile.h b/prism_compile.h index c032449bd65ca9..e588122205948b 100644 --- a/prism_compile.h +++ b/prism_compile.h @@ -94,6 +94,7 @@ VALUE pm_parse_file(pm_parse_result_t *result, VALUE filepath, VALUE *script_lin VALUE pm_load_parse_file(pm_parse_result_t *result, VALUE filepath, VALUE *script_lines); VALUE pm_parse_string(pm_parse_result_t *result, VALUE source, VALUE filepath, VALUE *script_lines); VALUE pm_parse_stdin(pm_parse_result_t *result); +void pm_options_version_for_current_ruby_set(pm_options_t *options); void pm_parse_result_free(pm_parse_result_t *result); rb_iseq_t *pm_iseq_new(pm_scope_node_t *node, VALUE name, VALUE path, VALUE realpath, const rb_iseq_t *parent, enum rb_iseq_type, int *error_state); diff --git a/process.c b/process.c index da9ce74027ce3c..0fb727db8af9f8 100644 --- a/process.c +++ b/process.c @@ -2635,7 +2635,7 @@ rb_exec_fillarg(VALUE prog, int argc, VALUE *argv, VALUE env, VALUE opthash, VAL } rb_str_buf_cat(argv_str, (char *)&null, sizeof(null)); /* terminator for execve. */ eargp->invoke.cmd.argv_str = - rb_imemo_tmpbuf_auto_free_pointer_new_from_an_RString(argv_str); + rb_imemo_tmpbuf_new_from_an_RString(argv_str); } RB_GC_GUARD(execarg_obj); } @@ -2726,7 +2726,7 @@ open_func(void *ptr) static void rb_execarg_allocate_dup2_tmpbuf(struct rb_execarg *eargp, long len) { - VALUE tmpbuf = rb_imemo_tmpbuf_auto_free_pointer(); + VALUE tmpbuf = rb_imemo_tmpbuf_new(); rb_imemo_tmpbuf_set_ptr(tmpbuf, ruby_xmalloc(run_exec_dup2_tmpbuf_size(len))); eargp->dup2_tmpbuf = tmpbuf; } @@ -2830,7 +2830,7 @@ rb_execarg_parent_start1(VALUE execarg_obj) p = NULL; rb_str_buf_cat(envp_str, (char *)&p, sizeof(p)); eargp->envp_str = - rb_imemo_tmpbuf_auto_free_pointer_new_from_an_RString(envp_str); + rb_imemo_tmpbuf_new_from_an_RString(envp_str); eargp->envp_buf = envp_buf; /* diff --git a/jit.rs b/ruby.rs similarity index 100% rename from jit.rs rename to ruby.rs diff --git a/spec/bundler/bundler/ruby_dsl_spec.rb b/spec/bundler/bundler/ruby_dsl_spec.rb index 2607f746e76653..0d02542fb595b3 100644 --- a/spec/bundler/bundler/ruby_dsl_spec.rb +++ b/spec/bundler/bundler/ruby_dsl_spec.rb @@ -210,6 +210,16 @@ class MockDSL it_behaves_like "it stores the ruby version" end end + + context "when the file does not exist" do + let(:ruby_version_file_path) { nil } + let(:ruby_version_arg) { nil } + let(:file) { "nonexistent.txt" } + + it "raises an error" do + expect { subject }.to raise_error(Bundler::GemfileError, /Could not find version file nonexistent.txt/) + end + end end end end diff --git a/spec/bundler/lock/git_spec.rb b/spec/bundler/lock/git_spec.rb index 49c0a2af1c20fc..4e416518305aa8 100644 --- a/spec/bundler/lock/git_spec.rb +++ b/spec/bundler/lock/git_spec.rb @@ -220,4 +220,39 @@ expect(lockfile).to include("securerandom (0.3.2)") end + + it "does not lock versions that don't exist in the repository when changing a GIT direct dep to a GEM direct dep" do + build_repo4 do + build_gem "ruby-lsp", "0.16.1" + end + + path = lib_path("ruby-lsp") + revision = build_git("ruby-lsp", "0.16.2", path: path).ref_for("HEAD") + + lockfile <<~L + GIT + remote: #{path} + revision: #{revision} + specs: + ruby-lsp (0.16.2) + + PLATFORMS + #{lockfile_platforms} + + DEPENDENCIES + ruby-lsp! + + BUNDLED WITH + #{Bundler::VERSION} + L + + gemfile <<~G + source "https://gem.repo4" + gem "ruby-lsp" + G + + bundle "lock" + + expect(lockfile).to include("ruby-lsp (0.16.1)") + end end diff --git a/spec/ruby/core/regexp/shared/new.rb b/spec/ruby/core/regexp/shared/new.rb index 921736d2990853..12c3d7c9c265e9 100644 --- a/spec/ruby/core/regexp/shared/new.rb +++ b/spec/ruby/core/regexp/shared/new.rb @@ -195,190 +195,14 @@ def obj.to_int() ScratchPad.record(:called) end -> { Regexp.send(@method, "\\\\") }.should_not raise_error(RegexpError) end - it "accepts a backspace followed by a character" do + it "accepts a backspace followed by a non-special character" do Regexp.send(@method, "\\N").should == /#{"\x5c"+"N"}/ end - it "accepts a one-digit octal value" do - Regexp.send(@method, "\0").should == /#{"\x00"}/ - end - - it "accepts a two-digit octal value" do - Regexp.send(@method, "\11").should == /#{"\x09"}/ - end - - it "accepts a one-digit hexadecimal value" do - Regexp.send(@method, "\x9n").should == /#{"\x09n"}/ - end - - it "accepts a two-digit hexadecimal value" do - Regexp.send(@method, "\x23").should == /#{"\x23"}/ - end - - it "interprets a digit following a two-digit hexadecimal value as a character" do - Regexp.send(@method, "\x420").should == /#{"\x420"}/ - end - it "raises a RegexpError if \\x is not followed by any hexadecimal digits" do -> { Regexp.send(@method, "\\" + "xn") }.should raise_error(RegexpError, Regexp.new(Regexp.escape("invalid hex escape: /\\xn/"))) end - it "accepts an escaped string interpolation" do - Regexp.send(@method, "\#{abc}").should == /#{"\#{abc}"}/ - end - - it "accepts '\\n'" do - Regexp.send(@method, "\n").should == /#{"\x0a"}/ - end - - it "accepts '\\t'" do - Regexp.send(@method, "\t").should == /#{"\x09"}/ - end - - it "accepts '\\r'" do - Regexp.send(@method, "\r").should == /#{"\x0d"}/ - end - - it "accepts '\\f'" do - Regexp.send(@method, "\f").should == /#{"\x0c"}/ - end - - it "accepts '\\v'" do - Regexp.send(@method, "\v").should == /#{"\x0b"}/ - end - - it "accepts '\\a'" do - Regexp.send(@method, "\a").should == /#{"\x07"}/ - end - - it "accepts '\\e'" do - Regexp.send(@method, "\e").should == /#{"\x1b"}/ - end - - it "accepts '\\C-\\n'" do - Regexp.send(@method, "\C-\n").should == /#{"\x0a"}/ - end - - it "accepts '\\C-\\t'" do - Regexp.send(@method, "\C-\t").should == /#{"\x09"}/ - end - - it "accepts '\\C-\\r'" do - Regexp.send(@method, "\C-\r").should == /#{"\x0d"}/ - end - - it "accepts '\\C-\\f'" do - Regexp.send(@method, "\C-\f").should == /#{"\x0c"}/ - end - - it "accepts '\\C-\\v'" do - Regexp.send(@method, "\C-\v").should == /#{"\x0b"}/ - end - - it "accepts '\\C-\\a'" do - Regexp.send(@method, "\C-\a").should == /#{"\x07"}/ - end - - it "accepts '\\C-\\e'" do - Regexp.send(@method, "\C-\e").should == /#{"\x1b"}/ - end - - it "accepts multiple consecutive '\\' characters" do - Regexp.send(@method, "\\\\\\N").should == /#{"\\\\\\"+"N"}/ - end - - it "accepts characters and escaped octal digits" do - Regexp.send(@method, "abc\076").should == /#{"abc\x3e"}/ - end - - it "accepts escaped octal digits and characters" do - Regexp.send(@method, "\076abc").should == /#{"\x3eabc"}/ - end - - it "accepts characters and escaped hexadecimal digits" do - Regexp.send(@method, "abc\x42").should == /#{"abc\x42"}/ - end - - it "accepts escaped hexadecimal digits and characters" do - Regexp.send(@method, "\x3eabc").should == /#{"\x3eabc"}/ - end - - it "accepts escaped hexadecimal and octal digits" do - Regexp.send(@method, "\061\x42").should == /#{"\x31\x42"}/ - end - - it "accepts \\u{H} for a single Unicode codepoint" do - Regexp.send(@method, "\u{f}").should == /#{"\x0f"}/ - end - - it "accepts \\u{HH} for a single Unicode codepoint" do - Regexp.send(@method, "\u{7f}").should == /#{"\x7f"}/ - end - - it "accepts \\u{HHH} for a single Unicode codepoint" do - Regexp.send(@method, "\u{07f}").should == /#{"\x7f"}/ - end - - it "accepts \\u{HHHH} for a single Unicode codepoint" do - Regexp.send(@method, "\u{0000}").should == /#{"\x00"}/ - end - - it "accepts \\u{HHHHH} for a single Unicode codepoint" do - Regexp.send(@method, "\u{00001}").should == /#{"\x01"}/ - end - - it "accepts \\u{HHHHHH} for a single Unicode codepoint" do - Regexp.send(@method, "\u{000000}").should == /#{"\x00"}/ - end - - it "accepts characters followed by \\u{HHHH}" do - Regexp.send(@method, "abc\u{3042}").should == /#{"abc\u3042"}/ - end - - it "accepts \\u{HHHH} followed by characters" do - Regexp.send(@method, "\u{3042}abc").should == /#{"\u3042abc"}/ - end - - it "accepts escaped hexadecimal digits followed by \\u{HHHH}" do - Regexp.send(@method, "\x42\u{3042}").should == /#{"\x42\u3042"}/ - end - - it "accepts escaped octal digits followed by \\u{HHHH}" do - Regexp.send(@method, "\056\u{3042}").should == /#{"\x2e\u3042"}/ - end - - it "accepts a combination of escaped octal and hexadecimal digits and \\u{HHHH}" do - Regexp.send(@method, "\056\x42\u{3042}\x52\076").should == /#{"\x2e\x42\u3042\x52\x3e"}/ - end - - it "accepts \\uHHHH for a single Unicode codepoint" do - Regexp.send(@method, "\u3042").should == /#{"\u3042"}/ - end - - it "accepts characters followed by \\uHHHH" do - Regexp.send(@method, "abc\u3042").should == /#{"abc\u3042"}/ - end - - it "accepts \\uHHHH followed by characters" do - Regexp.send(@method, "\u3042abc").should == /#{"\u3042abc"}/ - end - - it "accepts escaped hexadecimal digits followed by \\uHHHH" do - Regexp.send(@method, "\x42\u3042").should == /#{"\x42\u3042"}/ - end - - it "accepts escaped octal digits followed by \\uHHHH" do - Regexp.send(@method, "\056\u3042").should == /#{"\x2e\u3042"}/ - end - - it "accepts a combination of escaped octal and hexadecimal digits and \\uHHHH" do - Regexp.send(@method, "\056\x42\u3042\x52\076").should == /#{"\x2e\x42\u3042\x52\x3e"}/ - end - - it "accepts a multiple byte character which need not be escaped" do - Regexp.send(@method, "\").should == /#{""}/ - end - it "raises a RegexpError if less than four digits are given for \\uHHHH" do -> { Regexp.send(@method, "\\" + "u304") }.should raise_error(RegexpError, Regexp.new(Regexp.escape("invalid Unicode escape: /\\u304/"))) end @@ -433,69 +257,6 @@ def obj.to_int() ScratchPad.record(:called) end describe :regexp_new_string_binary, shared: true do describe "with escaped characters" do - it "accepts a three-digit octal value" do - Regexp.send(@method, "\315").should == /#{"\xcd"}/ - end - - it "interprets a digit following a three-digit octal value as a character" do - Regexp.send(@method, "\3762").should == /#{"\xfe2"}/ - end - - it "accepts '\\M-\\n'" do - Regexp.send(@method, "\M-\n").should == /#{"\x8a"}/ - end - - it "accepts '\\M-\\t'" do - Regexp.send(@method, "\M-\t").should == /#{"\x89"}/ - end - - it "accepts '\\M-\\r'" do - Regexp.send(@method, "\M-\r").should == /#{"\x8d"}/ - end - - it "accepts '\\M-\\f'" do - Regexp.send(@method, "\M-\f").should == /#{"\x8c"}/ - end - - it "accepts '\\M-\\v'" do - Regexp.send(@method, "\M-\v").should == /#{"\x8b"}/ - end - - it "accepts '\\M-\\a'" do - Regexp.send(@method, "\M-\a").should == /#{"\x87"}/ - end - - it "accepts '\\M-\\e'" do - Regexp.send(@method, "\M-\e").should == /#{"\x9b"}/ - end - - it "accepts '\\M-\\C-\\n'" do - Regexp.send(@method, "\M-\C-\n").should == /#{"\x8a"}/ - end - - it "accepts '\\M-\\C-\\t'" do - Regexp.send(@method, "\M-\C-\t").should == /#{"\x89"}/ - end - - it "accepts '\\M-\\C-\\r'" do - Regexp.send(@method, "\M-\C-\r").should == /#{"\x8d"}/ - end - - it "accepts '\\M-\\C-\\f'" do - Regexp.send(@method, "\M-\C-\f").should == /#{"\x8c"}/ - end - - it "accepts '\\M-\\C-\\v'" do - Regexp.send(@method, "\M-\C-\v").should == /#{"\x8b"}/ - end - - it "accepts '\\M-\\C-\\a'" do - Regexp.send(@method, "\M-\C-\a").should == /#{"\x87"}/ - end - - it "accepts '\\M-\\C-\\e'" do - Regexp.send(@method, "\M-\C-\e").should == /#{"\x9b"}/ - end end end diff --git a/spec/ruby/optional/capi/string_spec.rb b/spec/ruby/optional/capi/string_spec.rb index be9cb9015f0d34..605c43769ddb0b 100644 --- a/spec/ruby/optional/capi/string_spec.rb +++ b/spec/ruby/optional/capi/string_spec.rb @@ -191,11 +191,19 @@ def inspect end it "returns a new String object filled with \\0 bytes" do - s = @s.rb_str_tmp_new(4) - s.encoding.should == Encoding::BINARY - s.bytesize.should == 4 - s.size.should == 4 - s.should == "\x00\x00\x00\x00" + lens = [4] + + ruby_version_is "3.5" do + lens << 100 + end + + lens.each do |len| + s = @s.rb_str_tmp_new(len) + s.encoding.should == Encoding::BINARY + s.bytesize.should == len + s.size.should == len + s.should == "\x00" * len + end end end diff --git a/string.c b/string.c index 20873a35a5579a..834641cfdbf0c1 100644 --- a/string.c +++ b/string.c @@ -1066,6 +1066,9 @@ str_enc_new(VALUE klass, const char *ptr, long len, rb_encoding *enc) if (ptr) { memcpy(RSTRING_PTR(str), ptr, len); } + else { + memset(RSTRING_PTR(str), 0, len); + } STR_SET_LEN(str, len); TERM_FILL(RSTRING_PTR(str) + len, termlen); @@ -5067,34 +5070,36 @@ static VALUE get_pat(VALUE); * match(pattern, offset = 0) -> matchdata or nil * match(pattern, offset = 0) {|matchdata| ... } -> object * - * Returns a MatchData object (or +nil+) based on +self+ and the given +pattern+. - * - * Note: also updates Regexp@Global+Variables. + * Creates a MatchData object based on +self+ and the given arguments; + * updates {Regexp Global Variables}[rdoc-ref:Regexp@Global+Variables]. * * - Computes +regexp+ by converting +pattern+ (if not already a Regexp). + * * regexp = Regexp.new(pattern) + * * - Computes +matchdata+, which will be either a MatchData object or +nil+ * (see Regexp#match): - * matchdata = regexp.match(self) - * - * With no block given, returns the computed +matchdata+: * - * 'foo'.match('f') # => # - * 'foo'.match('o') # => # - * 'foo'.match('x') # => nil + * matchdata = regexp.match(self[offset..]) * - * If Integer argument +offset+ is given, the search begins at index +offset+: + * With no block given, returns the computed +matchdata+ or +nil+: * + * 'foo'.match('f') # => # + * 'foo'.match('o') # => # + * 'foo'.match('x') # => nil * 'foo'.match('f', 1) # => nil * 'foo'.match('o', 1) # => # * - * With a block given, calls the block with the computed +matchdata+ - * and returns the block's return value: + * With a block given and computed +matchdata+ non-nil, calls the block with +matchdata+; + * returns the block's return value: * * 'foo'.match(/o/) {|matchdata| matchdata } # => # - * 'foo'.match(/x/) {|matchdata| matchdata } # => nil - * 'foo'.match(/f/, 1) {|matchdata| matchdata } # => nil * + * With a block given and +nil+ +matchdata+, does not call the block: + * + * 'foo'.match(/x/) {|matchdata| fail 'Cannot happen' } # => nil + * + * Related: see {Querying}[rdoc-ref:String@Querying]. */ static VALUE @@ -5116,24 +5121,23 @@ rb_str_match_m(int argc, VALUE *argv, VALUE str) * call-seq: * match?(pattern, offset = 0) -> true or false * - * Returns +true+ or +false+ based on whether a match is found for +self+ and +pattern+. + * Returns whether a match is found for +self+ and the given arguments; + * does not update {Regexp Global Variables}[rdoc-ref:Regexp@Global+Variables]. * - * Note: does not update Regexp@Global+Variables. + * Computes +regexp+ by converting +pattern+ (if not already a Regexp): * - * Computes +regexp+ by converting +pattern+ (if not already a Regexp). * regexp = Regexp.new(pattern) * - * Returns +true+ if self+.match(regexp) returns a MatchData object, + * Returns +true+ if self[offset..].match(regexp) returns a MatchData object, * +false+ otherwise: * * 'foo'.match?(/o/) # => true * 'foo'.match?('o') # => true * 'foo'.match?(/x/) # => false - * - * If Integer argument +offset+ is given, the search begins at index +offset+: * 'foo'.match?('f', 1) # => false * 'foo'.match?('o', 1) # => true * + * Related: see {Querying}[rdoc-ref:String@Querying]. */ static VALUE @@ -10401,10 +10405,11 @@ rb_str_lstrip_bang(VALUE str) * * whitespace = "\x00\t\n\v\f\r " * s = whitespace + 'abc' + whitespace - * s # => "\u0000\t\n\v\f\r abc\u0000\t\n\v\f\r " - * s.lstrip # => "abc\u0000\t\n\v\f\r " + * # => "\u0000\t\n\v\f\r abc\u0000\t\n\v\f\r " + * s.lstrip + * # => "abc\u0000\t\n\v\f\r " * - * Related: String#rstrip, String#strip. + * Related: see {Converting to New String}[rdoc-ref:String@Converting+to+New+String]. */ static VALUE @@ -10739,20 +10744,79 @@ rb_str_hex(VALUE str) * call-seq: * oct -> integer * - * Interprets the leading substring of +self+ as a string of octal digits - * (with an optional sign) and returns the corresponding number; - * returns zero if there is no such leading substring: + * Interprets the leading substring of +self+ as octal, binary, decimal, or hexadecimal, possibly signed; + * returns their value as an integer. + * + * In brief: + * + * # Interpreted as octal. + * '777'.oct # => 511 + * '777x'.oct # => 511 + * '0777'.oct # => 511 + * '0o777'.oct # => 511 + * '-777'.oct # => -511 + * # Not interpreted as octal. + * '0b111'.oct # => 7 # Interpreted as binary. + * '0d999'.oct # => 999 # Interpreted as decimal. + * '0xfff'.oct # => 4095 # Interpreted as hexadecimal. * - * '123'.oct # => 83 - * '-377'.oct # => -255 - * '0377non-numeric'.oct # => 255 - * 'non-numeric'.oct # => 0 + * The leading substring is interpreted as octal when it begins with: * - * If +self+ starts with 0, radix indicators are honored; - * see Kernel#Integer. + * - One or more character representing octal digits + * (each in the range '0'..'7'); + * the string to be interpreted ends at the first character that does not represent an octal digit: * - * Related: String#hex. + * '7'.oct @ => 7 + * '11'.oct # => 9 + * '777'.oct # => 511 + * '0777'.oct # => 511 + * '7778'.oct # => 511 + * '777x'.oct # => 511 * + * - '0o', followed by one or more octal digits: + * + * '0o777'.oct # => 511 + * '0o7778'.oct # => 511 + * + * The leading substring is _not_ interpreted as octal when it begins with: + * + * - '0b', followed by one or more characters representing binary digits + * (each in the range '0'..'1'); + * the string to be interpreted ends at the first character that does not represent a binary digit. + * the string is interpreted as binary digits (base 2): + * + * '0b111'.oct # => 7 + * '0b1112'.oct # => 7 + * + * - '0d', followed by one or more characters representing decimal digits + * (each in the range '0'..'9'); + * the string to be interpreted ends at the first character that does not represent a decimal digit. + * the string is interpreted as decimal digits (base 10): + * + * '0d999'.oct # => 999 + * '0d999x'.oct # => 999 + * + * - '0x', followed by one or more characters representing hexadecimal digits + * (each in one of the ranges '0'..'9', 'a'..'f', or 'A'..'F'); + * the string to be interpreted ends at the first character that does not represent a hexadecimal digit. + * the string is interpreted as hexadecimal digits (base 16): + * + * '0xfff'.oct # => 4095 + * '0xfffg'.oct # => 4095 + * + * Any of the above may prefixed with '-', which negates the interpreted value: + * + * '-777'.oct # => -511 + * '-0777'.oct # => -511 + * '-0b111'.oct # => -7 + * '-0xfff'.oct # => -4095 + * + * For any substring not described above, returns zero: + * + * 'foo'.oct # => 0 + * ''.oct # => 0 + * + * Related: see {Converting to Non-String}[rdoc-ref:String@Converting+to+Non--5CString]. */ static VALUE diff --git a/test/-ext-/bug_reporter/test_bug_reporter.rb b/test/-ext-/bug_reporter/test_bug_reporter.rb index 1350675a48d3ed..83fdba22829e97 100644 --- a/test/-ext-/bug_reporter/test_bug_reporter.rb +++ b/test/-ext-/bug_reporter/test_bug_reporter.rb @@ -23,7 +23,7 @@ def test_bug_reporter_add # We want the printed description to match this process's RUBY_DESCRIPTION args.push("--yjit") if JITSupport.yjit_enabled? args.push("--zjit") if JITSupport.zjit_enabled? - args.unshift({"RUBY_ON_BUG" => nil}) + args.unshift({"RUBY_ON_BUG" => nil, "RUBY_CRASH_REPORT" => nil}) stdin = "#{no_core}register_sample_bug_reporter(12345); Process.kill :SEGV, $$" assert_in_out_err(args, stdin, [], expected_stderr, encoding: "ASCII-8BIT") ensure diff --git a/test/.excludes/JSONGenericObjectTest.rb b/test/.excludes/JSONGenericObjectTest.rb new file mode 100644 index 00000000000000..820a6a01200182 --- /dev/null +++ b/test/.excludes/JSONGenericObjectTest.rb @@ -0,0 +1,4 @@ +# ostruct will be loaded when JSON::GenericObject is autoloaded. By +# removing all test methods, the autoload in `setup` is not triggered. + +exclude /test_/, 'JSON::GenericObject needs ostruct gem' diff --git a/test/json/test_helper.rb b/test/json/test_helper.rb index cf592debfd8c43..a788804ddeca4e 100644 --- a/test/json/test_helper.rb +++ b/test/json/test_helper.rb @@ -1,5 +1,8 @@ $LOAD_PATH.unshift(File.expand_path('../../../ext', __FILE__), File.expand_path('../../../lib', __FILE__)) +require 'coverage' +Coverage.start + begin require 'simplecov' rescue LoadError diff --git a/test/openssl/test_pkey.rb b/test/openssl/test_pkey.rb index 71f5da81d145e7..24a595333b4ea3 100644 --- a/test/openssl/test_pkey.rb +++ b/test/openssl/test_pkey.rb @@ -60,6 +60,122 @@ def test_s_generate_key assert_not_equal nil, pkey.private_key end + def test_s_read_pem_unknown_block + # A PEM-encoded certificate and a PEM-encoded private key are combined. + # Check that OSSL_STORE doesn't stop after the first PEM block. + orig = Fixtures.pkey("rsa-1") + subject = OpenSSL::X509::Name.new([["CN", "test"]]) + cert = issue_cert(subject, orig, 1, [], nil, nil) + + input = cert.to_text + cert.to_pem + orig.to_text + orig.private_to_pem + pkey = OpenSSL::PKey.read(input) + assert_equal(orig.private_to_der, pkey.private_to_der) + end + + def test_s_read_der_then_pem + # If the input is valid as both DER and PEM (which allows garbage data + # before and after the block), it is read as DER + # + # TODO: Garbage data after DER should not be allowed, but it is currently + # ignored + orig1 = Fixtures.pkey("rsa-1") + orig2 = Fixtures.pkey("rsa-2") + pkey = OpenSSL::PKey.read(orig1.public_to_der + orig2.private_to_pem) + assert_equal(orig1.public_to_der, pkey.public_to_der) + assert_not_predicate(pkey, :private?) + end + + def test_s_read_passphrase + orig = Fixtures.pkey("rsa-1") + encrypted_pem = orig.private_to_pem("AES-256-CBC", "correct_passphrase") + assert_match(/\A-----BEGIN ENCRYPTED PRIVATE KEY-----/, encrypted_pem) + + # Correct passphrase passed as the second argument + pkey1 = OpenSSL::PKey.read(encrypted_pem, "correct_passphrase") + assert_equal(orig.private_to_der, pkey1.private_to_der) + + # Correct passphrase returned by the block. The block gets false + called = 0 + flag = nil + pkey2 = OpenSSL::PKey.read(encrypted_pem) { |f| + called += 1 + flag = f + "correct_passphrase" + } + assert_equal(orig.private_to_der, pkey2.private_to_der) + assert_equal(1, called) + assert_false(flag) + + # Incorrect passphrase passed. The block is not called + called = 0 + assert_raise(OpenSSL::PKey::PKeyError) { + OpenSSL::PKey.read(encrypted_pem, "incorrect_passphrase") { + called += 1 + } + } + assert_equal(0, called) + + # Incorrect passphrase returned by the block. The block is called only once + called = 0 + assert_raise(OpenSSL::PKey::PKeyError) { + OpenSSL::PKey.read(encrypted_pem) { + called += 1 + "incorrect_passphrase" + } + } + assert_equal(1, called) + + # Incorrect passphrase returned by the block. The input contains two PEM + # blocks. + called = 0 + assert_raise(OpenSSL::PKey::PKeyError) { + OpenSSL::PKey.read(encrypted_pem + encrypted_pem) { + called += 1 + "incorrect_passphrase" + } + } + assert_equal(1, called) + end + + def test_s_read_passphrase_tty + omit "https://github.com/aws/aws-lc/pull/2555" if aws_lc? + + orig = Fixtures.pkey("rsa-1") + encrypted_pem = orig.private_to_pem("AES-256-CBC", "correct_passphrase") + + # Correct passphrase passed to OpenSSL's prompt + script = <<~"end;" + require "openssl" + Process.setsid + OpenSSL::PKey.read(#{encrypted_pem.dump}) + puts "ok" + end; + assert_in_out_err([*$:.map { |l| "-I#{l}" }, "-e#{script}"], + "correct_passphrase\n") { |stdout, stderr| + assert_equal(["Enter PEM pass phrase:"], stderr) + assert_equal(["ok"], stdout) + } + + # Incorrect passphrase passed to OpenSSL's prompt + script = <<~"end;" + require "openssl" + Process.setsid + begin + OpenSSL::PKey.read(#{encrypted_pem.dump}) + rescue OpenSSL::PKey::PKeyError + puts "ok" + else + puts "expected OpenSSL::PKey::PKeyError" + end + end; + stdin = "incorrect_passphrase\n" * 5 + assert_in_out_err([*$:.map { |l| "-I#{l}" }, "-e#{script}"], + stdin) { |stdout, stderr| + assert_equal(1, stderr.count("Enter PEM pass phrase:")) + assert_equal(["ok"], stdout) + } + end if ENV["OSSL_TEST_ALL"] == "1" && Process.respond_to?(:setsid) + def test_hmac_sign_verify pkey = OpenSSL::PKey.generate_key("HMAC", { "key" => "abcd" }) diff --git a/test/prism/errors/endless_method_command_call.txt b/test/prism/errors/endless_method_command_call.txt new file mode 100644 index 00000000000000..e6a328c2944a68 --- /dev/null +++ b/test/prism/errors/endless_method_command_call.txt @@ -0,0 +1,3 @@ +private :m, def hello = puts "Hello" + ^ unexpected string literal, expecting end-of-input + diff --git a/test/prism/errors/private_endless_method.txt b/test/prism/errors/private_endless_method.txt deleted file mode 100644 index 8aae5e0cd39035..00000000000000 --- a/test/prism/errors/private_endless_method.txt +++ /dev/null @@ -1,3 +0,0 @@ -private def foo = puts "Hello" - ^ unexpected string literal, expecting end-of-input - diff --git a/test/prism/errors_test.rb b/test/prism/errors_test.rb index 62bbd8458b2ca9..9dd4aea72865d9 100644 --- a/test/prism/errors_test.rb +++ b/test/prism/errors_test.rb @@ -100,6 +100,15 @@ def foo(bar: bar) = 42 end end + def test_private_endless_method + source = <<~RUBY + private def foo = puts "Hello" + RUBY + + assert_predicate Prism.parse(source, version: "3.4"), :failure? + assert_predicate Prism.parse(source), :success? + end + private def assert_errors(filepath) diff --git a/test/prism/fixtures/character_literal.txt b/test/prism/fixtures/character_literal.txt new file mode 100644 index 00000000000000..920332123f13ac --- /dev/null +++ b/test/prism/fixtures/character_literal.txt @@ -0,0 +1,2 @@ +# encoding: Windows-31J +p ?\u3042"" diff --git a/test/prism/fixtures/endless_methods_command_call.txt b/test/prism/fixtures/endless_methods_command_call.txt new file mode 100644 index 00000000000000..91a9d156d5538b --- /dev/null +++ b/test/prism/fixtures/endless_methods_command_call.txt @@ -0,0 +1,8 @@ +private def foo = puts "Hello" +private def foo = puts "Hello", "World" +private def foo = puts "Hello" do expr end +private def foo() = puts "Hello" +private def foo(x) = puts x +private def obj.foo = puts "Hello" +private def obj.foo() = puts "Hello" +private def obj.foo(x) = puts x diff --git a/test/prism/fixtures/leading_logical.txt b/test/prism/fixtures/leading_logical.txt new file mode 100644 index 00000000000000..feb5ee245c8b2f --- /dev/null +++ b/test/prism/fixtures/leading_logical.txt @@ -0,0 +1,21 @@ +1 +&& 2 +&& 3 + +1 +|| 2 +|| 3 + +1 +and 2 +and 3 + +1 +or 2 +or 3 + +1 +andfoo + +2 +orfoo diff --git a/test/prism/fixtures_test.rb b/test/prism/fixtures_test.rb index 3b4a502b9010f1..b4b656fcf49b48 100644 --- a/test/prism/fixtures_test.rb +++ b/test/prism/fixtures_test.rb @@ -8,7 +8,6 @@ module Prism class FixturesTest < TestCase except = [] - if RUBY_VERSION < "3.3.0" # Ruby < 3.3.0 cannot parse heredocs where there are leading whitespace # characters in the heredoc start. @@ -25,6 +24,10 @@ class FixturesTest < TestCase except << "whitequark/ruby_bug_19281.txt" end + # Leaving these out until they are supported by parse.y. + except << "leading_logical.txt" + except << "endless_methods_command_call.txt" + Fixture.each(except: except) do |fixture| define_method(fixture.test_name) { assert_valid_syntax(fixture.read) } end diff --git a/test/prism/lex_test.rb b/test/prism/lex_test.rb index d34c3d9dd3cb2f..4eacbab3e1170d 100644 --- a/test/prism/lex_test.rb +++ b/test/prism/lex_test.rb @@ -42,6 +42,12 @@ class LexTest < TestCase except << "whitequark/ruby_bug_19281.txt" end + # https://bugs.ruby-lang.org/issues/20925 + except << "leading_logical.txt" + + # https://bugs.ruby-lang.org/issues/17398#note-12 + except << "endless_methods_command_call.txt" + Fixture.each(except: except) do |fixture| define_method(fixture.test_name) { assert_lex(fixture) } end diff --git a/test/prism/locals_test.rb b/test/prism/locals_test.rb index e0e9a458559759..950e7118af526a 100644 --- a/test/prism/locals_test.rb +++ b/test/prism/locals_test.rb @@ -29,7 +29,11 @@ class LocalsTest < TestCase except = [ # Skip this fixture because it has a different number of locals because # CRuby is eliminating dead code. - "whitequark/ruby_bug_10653.txt" + "whitequark/ruby_bug_10653.txt", + + # Leaving these out until they are supported by parse.y. + "leading_logical.txt", + "endless_methods_command_call.txt" ] Fixture.each(except: except) do |fixture| diff --git a/test/prism/result/static_literals_test.rb b/test/prism/result/static_literals_test.rb index cc070279169aba..dcfc692897cd66 100644 --- a/test/prism/result/static_literals_test.rb +++ b/test/prism/result/static_literals_test.rb @@ -4,11 +4,6 @@ module Prism class StaticLiteralsTest < TestCase - def test_concatenanted_string_literal_is_not_static - node = Prism.parse_statement("'a' 'b'") - refute_predicate node, :static_literal? - end - def test_static_literals assert_warning("1") assert_warning("0xA", "10", "10") diff --git a/test/prism/ruby/parser_test.rb b/test/prism/ruby/parser_test.rb index 2a4ea981bdb5c3..98740f09734043 100644 --- a/test/prism/ruby/parser_test.rb +++ b/test/prism/ruby/parser_test.rb @@ -64,6 +64,12 @@ class ParserTest < TestCase # 1.. && 2 "ranges.txt", + + # Cannot yet handling leading logical operators. + "leading_logical.txt", + + # Ruby >= 3.5 specific syntax + "endless_methods_command_call.txt", ] # These files contain code that is being parsed incorrectly by the parser diff --git a/test/prism/ruby/ripper_test.rb b/test/prism/ruby/ripper_test.rb index 5c37178889ecf5..39325137ba07f2 100644 --- a/test/prism/ruby/ripper_test.rb +++ b/test/prism/ruby/ripper_test.rb @@ -8,6 +8,9 @@ module Prism class RipperTest < TestCase # Skip these tests that Ripper is reporting the wrong results for. incorrect = [ + # Not yet supported. + "leading_logical.txt", + # Ripper incorrectly attributes the block to the keyword. "seattlerb/block_break.txt", "seattlerb/block_next.txt", @@ -26,7 +29,10 @@ class RipperTest < TestCase "whitequark/lvar_injecting_match.txt", # Ripper fails to understand some structures that span across heredocs. - "spanning_heredoc.txt" + "spanning_heredoc.txt", + + # https://bugs.ruby-lang.org/issues/17398#note-12 + "endless_methods_command_call.txt", ] # Skip these tests that we haven't implemented yet. diff --git a/test/prism/ruby/ruby_parser_test.rb b/test/prism/ruby/ruby_parser_test.rb index 960e7f63e46385..b21ad81391ed1e 100644 --- a/test/prism/ruby/ruby_parser_test.rb +++ b/test/prism/ruby/ruby_parser_test.rb @@ -16,6 +16,7 @@ module Prism class RubyParserTest < TestCase todos = [ + "character_literal.txt", "encoding_euc_jp.txt", "regex_char_width.txt", "seattlerb/masgn_colon3.txt", @@ -38,6 +39,7 @@ class RubyParserTest < TestCase "dos_endings.txt", "heredocs_with_fake_newlines.txt", "heredocs_with_ignored_newlines.txt", + "leading_logical.txt", "method_calls.txt", "methods.txt", "multi_write.txt", @@ -73,7 +75,10 @@ class RubyParserTest < TestCase "whitequark/ruby_bug_11989.txt", "whitequark/ruby_bug_18878.txt", "whitequark/ruby_bug_19281.txt", - "whitequark/slash_newline_in_heredocs.txt" + "whitequark/slash_newline_in_heredocs.txt", + + # Ruby >= 3.5 specific syntax + "endless_methods_command_call.txt", ] Fixture.each(except: failures) do |fixture| diff --git a/test/ruby/lbtest.rb b/test/ruby/lbtest.rb index c7822c9e9a2cce..9b6abe870e0c7e 100644 --- a/test/ruby/lbtest.rb +++ b/test/ruby/lbtest.rb @@ -1,5 +1,8 @@ # frozen_string_literal: false +require 'fiddle' +Fiddle::Pointer.new(1)[0] + class LocalBarrier def initialize(n) @wait = Thread::Queue.new diff --git a/test/ruby/test_ast.rb b/test/ruby/test_ast.rb index 55e5915d821546..c7a946dec868bb 100644 --- a/test/ruby/test_ast.rb +++ b/test/ruby/test_ast.rb @@ -48,7 +48,7 @@ def initialize(path, src: nil) @path = path @errors = [] @debug = false - @ast = RubyVM::AbstractSyntaxTree.parse(src) if src + @ast = EnvUtil.suppress_warning { RubyVM::AbstractSyntaxTree.parse(src) } if src end def validate_range @@ -67,7 +67,7 @@ def validate_not_cared def ast return @ast if defined?(@ast) - @ast = RubyVM::AbstractSyntaxTree.parse_file(@path) + @ast = EnvUtil.suppress_warning { RubyVM::AbstractSyntaxTree.parse_file(@path) } end private @@ -135,7 +135,7 @@ def validate_not_cared0(node) Dir.glob("test/**/*.rb", base: SRCDIR).each do |path| define_method("test_all_tokens:#{path}") do - node = RubyVM::AbstractSyntaxTree.parse_file("#{SRCDIR}/#{path}", keep_tokens: true) + node = EnvUtil.suppress_warning { RubyVM::AbstractSyntaxTree.parse_file("#{SRCDIR}/#{path}", keep_tokens: true) } tokens = node.all_tokens.sort_by { [_1.last[0], _1.last[1]] } tokens_bytes = tokens.map { _1[2]}.join.bytes source_bytes = File.read("#{SRCDIR}/#{path}").bytes diff --git a/test/ruby/test_object.rb b/test/ruby/test_object.rb index 9074e54df5ed50..cccd7359e1108e 100644 --- a/test/ruby/test_object.rb +++ b/test/ruby/test_object.rb @@ -280,6 +280,12 @@ def test_methods_prepend_singleton assert_equal([:foo], k.private_methods(false)) end + class ToStrCounter + def initialize(str = "@foo") @str = str; @count = 0; end + def to_str; @count += 1; @str; end + def count; @count; end + end + def test_instance_variable_get o = Object.new o.instance_eval { @foo = :foo } @@ -291,9 +297,7 @@ def test_instance_variable_get assert_raise(NameError) { o.instance_variable_get("bar") } assert_raise(TypeError) { o.instance_variable_get(1) } - n = Object.new - def n.to_str; @count = defined?(@count) ? @count + 1 : 1; "@foo"; end - def n.count; @count; end + n = ToStrCounter.new assert_equal(:foo, o.instance_variable_get(n)) assert_equal(1, n.count) end @@ -308,9 +312,7 @@ def test_instance_variable_set assert_raise(NameError) { o.instance_variable_set("bar", 1) } assert_raise(TypeError) { o.instance_variable_set(1, 1) } - n = Object.new - def n.to_str; @count = defined?(@count) ? @count + 1 : 1; "@foo"; end - def n.count; @count; end + n = ToStrCounter.new o.instance_variable_set(n, :bar) assert_equal(:bar, o.instance_eval { @foo }) assert_equal(1, n.count) @@ -327,9 +329,7 @@ def test_instance_variable_defined assert_raise(NameError) { o.instance_variable_defined?("bar") } assert_raise(TypeError) { o.instance_variable_defined?(1) } - n = Object.new - def n.to_str; @count = defined?(@count) ? @count + 1 : 1; "@foo"; end - def n.count; @count; end + n = ToStrCounter.new assert_equal(true, o.instance_variable_defined?(n)) assert_equal(1, n.count) end diff --git a/test/ruby/test_optimization.rb b/test/ruby/test_optimization.rb index e39eafa5e50bb9..089c5fbd1d0dc6 100644 --- a/test/ruby/test_optimization.rb +++ b/test/ruby/test_optimization.rb @@ -606,11 +606,11 @@ def foo(n) end class Bug10557 - def [](_) + def [](_, &) block_given? end - def []=(_, _) + def []=(_, _, &) block_given? end end @@ -1256,6 +1256,9 @@ def initialize a, **kw insn = iseq.disasm assert_match(/opt_new/, insn) assert_match(/OptNewFoo:.+@a=1, @b=2/, iseq.eval.inspect) + # clean up to avoid warnings + Object.send :remove_const, :OptNewFoo + Object.remove_method :optnew_foo if defined?(optnew_foo) end [ 'def optnew_foo(&) = OptNewFoo.new(&)', diff --git a/test/ruby/test_rubyoptions.rb b/test/ruby/test_rubyoptions.rb index 2c9f8534b19aad..2b7cf1c8999a4c 100644 --- a/test/ruby/test_rubyoptions.rb +++ b/test/ruby/test_rubyoptions.rb @@ -855,6 +855,7 @@ def assert_segv(args, message=nil, list: SEGVTest::ExpectedStderrList, **opt, &b args.unshift("--yjit") if JITSupport.yjit_enabled? args.unshift("--zjit") if JITSupport.zjit_enabled? env.update({'RUBY_ON_BUG' => nil}) + env['RUBY_CRASH_REPORT'] ||= nil # default to not passing down parent setting # ASAN registers a segv handler which prints out "AddressSanitizer: DEADLYSIGNAL" when # catching sigsegv; we don't expect that output, so suppress it. env.update({'ASAN_OPTIONS' => 'handle_segv=0'}) diff --git a/test/ruby/test_syntax.rb b/test/ruby/test_syntax.rb index b7e021a4ff4976..94a2e03940bf5b 100644 --- a/test/ruby/test_syntax.rb +++ b/test/ruby/test_syntax.rb @@ -1259,6 +1259,56 @@ def test_fluent_dot assert_valid_syntax("a #\n#\n&.foo\n") end + def test_fluent_and + omit if /\+PRISM\b/ =~ RUBY_DESCRIPTION + + assert_valid_syntax("a\n" "&& foo") + assert_valid_syntax("a\n" "and foo") + + assert_equal(:ok, eval("#{<<~"begin;"}\n#{<<~'end;'}")) + begin; + a = true + if a + && (a = :ok; true) + a + end + end; + + assert_equal(:ok, eval("#{<<~"begin;"}\n#{<<~'end;'}")) + begin; + a = true + if a + and (a = :ok; true) + a + end + end; + end + + def test_fluent_or + omit if /\+PRISM\b/ =~ RUBY_DESCRIPTION + + assert_valid_syntax("a\n" "|| foo") + assert_valid_syntax("a\n" "or foo") + + assert_equal(:ok, eval("#{<<~"begin;"}\n#{<<~'end;'}")) + begin; + a = false + if a + || (a = :ok; true) + a + end + end; + + assert_equal(:ok, eval("#{<<~"begin;"}\n#{<<~'end;'}")) + begin; + a = false + if a + or (a = :ok; true) + a + end + end; + end + def test_safe_call_in_massign_lhs assert_syntax_error("*a&.x=0", /multiple assignment destination/) assert_syntax_error("a&.x,=0", /multiple assignment destination/) @@ -1794,15 +1844,12 @@ def test_methoddef_endless_command assert_equal("class ok", k.rescued("ok")) assert_equal("instance ok", k.new.rescued("ok")) - # Current technical limitation: cannot prepend "private" or something for command endless def - error = /(syntax error,|\^~*) unexpected string literal/ - error2 = /(syntax error,|\^~*) unexpected local variable or method/ - assert_syntax_error('private def foo = puts "Hello"', error) - assert_syntax_error('private def foo() = puts "Hello"', error) - assert_syntax_error('private def foo(x) = puts x', error2) - assert_syntax_error('private def obj.foo = puts "Hello"', error) - assert_syntax_error('private def obj.foo() = puts "Hello"', error) - assert_syntax_error('private def obj.foo(x) = puts x', error2) + assert_valid_syntax('private def foo = puts "Hello"') + assert_valid_syntax('private def foo() = puts "Hello"') + assert_valid_syntax('private def foo(x) = puts x') + assert_valid_syntax('private def obj.foo = puts "Hello"') + assert_valid_syntax('private def obj.foo() = puts "Hello"') + assert_valid_syntax('private def obj.foo(x) = puts x') end def test_methoddef_in_cond @@ -1957,6 +2004,19 @@ def test_it assert_equal(/9/, eval('9.then { /#{it}/o }')) end + def test_it_with_splat_super_method + bug21256 = '[ruby-core:121592] [Bug #21256]' + + a = Class.new do + define_method(:foo) { it } + end + b = Class.new(a) do + def foo(*args) = super + end + + assert_equal(1, b.new.foo(1), bug21256) + end + def test_value_expr_in_condition mesg = /void value expression/ assert_syntax_error("tap {a = (true ? next : break)}", mesg) @@ -2027,10 +2087,11 @@ def obj1.bar(*args, **kws, &block) end obj4 = obj1.clone obj5 = obj1.clone + obj6 = obj1.clone obj1.instance_eval('def foo(...) bar(...) end', __FILE__, __LINE__) - obj1.instance_eval('def foo(...) eval("bar(...)") end', __FILE__, __LINE__) obj4.instance_eval("def foo ...\n bar(...)\n""end", __FILE__, __LINE__) obj5.instance_eval("def foo ...; bar(...); end", __FILE__, __LINE__) + obj6.instance_eval('def foo(...) eval("bar(...)") end', __FILE__, __LINE__) klass = Class.new { def foo(*args, **kws, &block) @@ -2059,7 +2120,7 @@ def obj3.bar(*args, &block) end obj3.instance_eval('def foo(...) bar(...) end', __FILE__, __LINE__) - [obj1, obj2, obj3, obj4, obj5].each do |obj| + [obj1, obj2, obj3, obj4, obj5, obj6].each do |obj| assert_warning('') { assert_equal([[1, 2, 3], {k1: 4, k2: 5}], obj.foo(1, 2, 3, k1: 4, k2: 5) {|*x| x}) } diff --git a/test/ruby/test_variable.rb b/test/ruby/test_variable.rb index df0c6f1f094578..68434e0b6c479b 100644 --- a/test/ruby/test_variable.rb +++ b/test/ruby/test_variable.rb @@ -512,7 +512,7 @@ def test_genivar_cache instance.instance_variable_set(:@a3, 3) instance.instance_variable_set(:@a4, 4) end.resume - assert_equal 4, instance.instance_variable_get(:@a4) + assert_equal 4, instance.instance_variable_get(:@a4), bug21547 end private diff --git a/test/ruby/test_vm_dump.rb b/test/ruby/test_vm_dump.rb index a3e7b69913b7d2..fe358fb65af395 100644 --- a/test/ruby/test_vm_dump.rb +++ b/test/ruby/test_vm_dump.rb @@ -5,7 +5,8 @@ class TestVMDump < Test::Unit::TestCase def assert_darwin_vm_dump_works(args, timeout=nil) - assert_in_out_err(args, "", [], /^\[IMPORTANT\]/, timeout: timeout || 300) + env = {"RUBY_ON_BUG" => nil, "RUBY_CRASH_REPORT" => nil} + assert_in_out_err(env, args, "", [], /^\[IMPORTANT\]/, timeout: timeout || 300) end def test_darwin_invalid_call diff --git a/tool/auto-style.rb b/tool/auto-style.rb index b673e3d1771042..259ed377bc7036 100755 --- a/tool/auto-style.rb +++ b/tool/auto-style.rb @@ -11,9 +11,11 @@ class Git def initialize(oldrev, newrev, branch = nil) @oldrev = oldrev - @newrev = newrev.empty? ? 'HEAD' : newrev + @newrev = !newrev || newrev.empty? ? 'HEAD' : newrev @branch = branch + return unless oldrev + # GitHub may not fetch github.event.pull_request.base.sha at checkout git('log', '--format=%H', '-1', @oldrev, out: IO::NULL, err: [:child, :out]) or git('fetch', '--depth=1', 'origin', @oldrev) @@ -59,7 +61,7 @@ def commit(log, *files) end def push - git('push', 'origin', @branch) + git('push', 'origin', @branch) if @branch end def diff @@ -181,85 +183,97 @@ def with_clean_env addr2line.c io_buffer.c prism*.c scheduler.c ] -oldrev, newrev, pushref = ARGV -unless dry_run = pushref.empty? - branch = IO.popen(['git', 'rev-parse', '--symbolic', '--abbrev-ref', pushref], &:read).strip -end -git = Git.new(oldrev, newrev, branch) +def adjust_styles(files) + trailing = eofnewline = expandtab = indent = false -updated_files = git.updated_paths -files = updated_files.select {|l| - /^\d/ !~ l and /\.bat\z/ !~ l and - (/\A(?:config|[Mm]akefile|GNUmakefile|README)/ =~ File.basename(l) or - /\A\z|\.(?:[chsy]|\d+|e?rb|tmpl|bas[eh]|z?sh|in|ma?k|def|src|trans|rdoc|ja|en|el|sed|awk|p[ly]|scm|mspec|html|rs)\z/ =~ File.extname(l)) -} -files.select! {|n| File.file?(n) } -files.reject! do |f| - IGNORED_FILES.any? { |re| f.match(re) } -end -if files.empty? - puts "No files are an auto-style target:\n#{updated_files.join("\n")}" - exit -end + edited_files = files.select do |f| + src = File.binread(f) rescue next + eofnewline = eofnewline0 = true if src.sub!(/(? 0x40000000l) rb_bug("yjit page size too large"); - - return (uint32_t)page_size; -#else -#error "YJIT supports POSIX only for now" -#endif -} - -#if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE) -// Align the current write position to a multiple of bytes -static uint8_t * -align_ptr(uint8_t *ptr, uint32_t multiple) -{ - // Compute the pointer modulo the given alignment boundary - uint32_t rem = ((uint32_t)(uintptr_t)ptr) % multiple; - - // If the pointer is already aligned, stop - if (rem == 0) - return ptr; - - // Pad the pointer by the necessary amount to align it - uint32_t pad = multiple - rem; - - return ptr + pad; -} -#endif - -// Address space reservation. Memory pages are mapped on an as needed basis. -// See the Rust mm module for details. -uint8_t * -rb_yjit_reserve_addr_space(uint32_t mem_size) -{ -#ifndef _WIN32 - uint8_t *mem_block; - - // On Linux - #if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE) - uint32_t const page_size = (uint32_t)sysconf(_SC_PAGESIZE); - uint8_t *const cfunc_sample_addr = (void *)(uintptr_t)&rb_yjit_reserve_addr_space; - uint8_t *const probe_region_end = cfunc_sample_addr + INT32_MAX; - // Align the requested address to page size - uint8_t *req_addr = align_ptr(cfunc_sample_addr, page_size); - - // Probe for addresses close to this function using MAP_FIXED_NOREPLACE - // to improve odds of being in range for 32-bit relative call instructions. - do { - mem_block = mmap( - req_addr, - mem_size, - PROT_NONE, - MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, - -1, - 0 - ); - - // If we succeeded, stop - if (mem_block != MAP_FAILED) { - ruby_annotate_mmap(mem_block, mem_size, "Ruby:rb_yjit_reserve_addr_space"); - break; - } - - // -4MiB. Downwards to probe away from the heap. (On x86/A64 Linux - // main_code_addr < heap_addr, and in case we are in a shared - // library mapped higher than the heap, downwards is still better - // since it's towards the end of the heap rather than the stack.) - req_addr -= 4 * 1024 * 1024; - } while (req_addr < probe_region_end); - - // On MacOS and other platforms - #else - // Try to map a chunk of memory as executable - mem_block = mmap( - (void *)rb_yjit_reserve_addr_space, - mem_size, - PROT_NONE, - MAP_PRIVATE | MAP_ANONYMOUS, - -1, - 0 - ); - #endif - - // Fallback - if (mem_block == MAP_FAILED) { - // Try again without the address hint (e.g., valgrind) - mem_block = mmap( - NULL, - mem_size, - PROT_NONE, - MAP_PRIVATE | MAP_ANONYMOUS, - -1, - 0 - ); - - if (mem_block != MAP_FAILED) { - ruby_annotate_mmap(mem_block, mem_size, "Ruby:rb_yjit_reserve_addr_space:fallback"); - } - } - - // Check that the memory mapping was successful - if (mem_block == MAP_FAILED) { - perror("ruby: yjit: mmap:"); - if(errno == ENOMEM) { - // No crash report if it's only insufficient memory - exit(EXIT_FAILURE); - } - rb_bug("mmap failed"); - } - - return mem_block; -#else - // Windows not supported for now - return NULL; -#endif -} - // Is anyone listening for :c_call and :c_return event currently? bool rb_c_method_tracing_currently_enabled(const rb_execution_context_t *ec) diff --git a/yjit/Cargo.toml b/yjit/Cargo.toml index ad7dd35ecfb173..e2f1d84ffd3d8c 100644 --- a/yjit/Cargo.toml +++ b/yjit/Cargo.toml @@ -13,6 +13,7 @@ publish = false # Don't publish to crates.io # No required dependencies to simplify build process. TODO: Link to yet to be # written rationale. Optional For development and testing purposes capstone = { version = "0.13.0", optional = true } +jit = { version = "0.1.0", path = "../jit" } # NOTE: Development builds select a set of these via configure.ac # For debugging, `make V=1` shows exact cargo invocation. @@ -24,3 +25,4 @@ disasm = ["capstone"] # from cfg!(debug_assertions) so that we can see disasm of the code # that would run in the release mode. runtime_checks = [] +stats_allocator = [] diff --git a/yjit/bindgen/src/main.rs b/yjit/bindgen/src/main.rs index 2fc85431e0439d..29b17346cd90ae 100644 --- a/yjit/bindgen/src/main.rs +++ b/yjit/bindgen/src/main.rs @@ -65,35 +65,24 @@ fn main() { // Import YARV bytecode instruction constants .allowlist_type("ruby_vminsn_type") - // From include/ruby/internal/special_consts.h .allowlist_type("ruby_special_consts") - - // From include/ruby/internal/intern/string.h .allowlist_function("rb_utf8_str_new") .allowlist_function("rb_str_buf_append") .allowlist_function("rb_str_dup") - - // From encindex.h .allowlist_type("ruby_preserved_encindex") - - // From include/ruby/ruby.h .allowlist_function("rb_class2name") // This struct is public to Ruby C extensions - // From include/ruby/internal/core/rbasic.h .allowlist_type("RBasic") - // From include/ruby/internal/core/rstring.h .allowlist_type("ruby_rstring_flags") - // From internal.h // This function prints info about a value and is useful for debugging .allowlist_function("rb_obj_info_dump") // For crashing .allowlist_function("rb_bug") - // From shape.h .allowlist_function("rb_obj_shape_id") .allowlist_function("rb_shape_id_offset") .allowlist_function("rb_shape_get_iv_index") @@ -103,36 +92,20 @@ fn main() { .allowlist_function("rb_yjit_shape_index") .allowlist_var("SHAPE_ID_NUM_BITS") .allowlist_var("SHAPE_ID_HAS_IVAR_MASK") - - // From ruby/internal/eval.h .allowlist_function("rb_funcall") - - // From ruby/internal/intern/object.h .allowlist_function("rb_obj_is_kind_of") .allowlist_function("rb_obj_frozen_p") - - // From ruby/internal/encoding/encoding.h .allowlist_type("ruby_encoding_consts") - - // From include/hash.h .allowlist_function("rb_hash_new") - - // From internal/hash.h .allowlist_function("rb_hash_new_with_size") .allowlist_function("rb_hash_resurrect") .allowlist_function("rb_hash_stlike_foreach") .allowlist_function("rb_to_hash_type") - - // From include/ruby/st.h .allowlist_type("st_retval") - - // From include/ruby/internal/intern/hash.h .allowlist_function("rb_hash_aset") .allowlist_function("rb_hash_aref") .allowlist_function("rb_hash_bulk_insert") .allowlist_function("rb_hash_stlike_lookup") - - // From include/ruby/internal/intern/array.h .allowlist_function("rb_ary_new_capa") .allowlist_function("rb_ary_store") .allowlist_function("rb_ary_resurrect") @@ -142,26 +115,17 @@ fn main() { .allowlist_function("rb_ary_push") .allowlist_function("rb_ary_unshift_m") .allowlist_function("rb_yjit_rb_ary_subseq_length") - - // From internal/array.h .allowlist_function("rb_ec_ary_new_from_values") .allowlist_function("rb_ary_tmp_new_from_values") - - // From include/ruby/internal/intern/class.h .allowlist_function("rb_class_attached_object") .allowlist_function("rb_singleton_class") - - // From include/ruby/internal/core/rclass.h .allowlist_function("rb_class_get_superclass") - - // From include/ruby/internal/gc.h .allowlist_function("rb_gc_mark") .allowlist_function("rb_gc_mark_movable") .allowlist_function("rb_gc_location") .allowlist_function("rb_gc_writebarrier") // VALUE variables for Ruby class objects - // From include/ruby/internal/globals.h .allowlist_var("rb_cBasicObject") .allowlist_var("rb_cModule") .allowlist_var("rb_cNilClass") @@ -178,84 +142,53 @@ fn main() { .allowlist_var("rb_cHash") .allowlist_var("rb_cClass") - // From include/ruby/internal/fl_type.h .allowlist_type("ruby_fl_type") .allowlist_type("ruby_fl_ushift") - - // From include/ruby/internal/core/robject.h .allowlist_type("ruby_robject_flags") - - // From include/ruby/internal/core/rarray.h .allowlist_type("ruby_rarray_flags") .allowlist_type("ruby_rarray_consts") - - // From include/ruby/internal/core/rclass.h .allowlist_type("ruby_rmodule_flags") - - // From ruby/internal/globals.h .allowlist_var("rb_mKernel") - - // From vm_callinfo.h .allowlist_type("vm_call_flag_bits") .allowlist_type("rb_call_data") .blocklist_type("rb_callcache.*") // Not used yet - opaque to make it easy to import rb_call_data .opaque_type("rb_callcache.*") .allowlist_type("rb_callinfo") - - // From vm_insnhelper.h .allowlist_var("VM_ENV_DATA_INDEX_ME_CREF") .allowlist_var("rb_block_param_proxy") - - // From include/ruby/internal/intern/range.h .allowlist_function("rb_range_new") - - // From include/ruby/internal/symbol.h .allowlist_function("rb_intern") .allowlist_function("rb_intern2") .allowlist_function("rb_id2sym") .allowlist_function("rb_id2name") .allowlist_function("rb_sym2id") .allowlist_function("rb_str_intern") - - // From internal/numeric.h .allowlist_function("rb_fix_aref") .allowlist_function("rb_float_plus") .allowlist_function("rb_float_minus") .allowlist_function("rb_float_mul") .allowlist_function("rb_float_div") - - // From internal/string.h .allowlist_type("ruby_rstring_private_flags") .allowlist_function("rb_ec_str_resurrect") .allowlist_function("rb_str_concat_literals") .allowlist_function("rb_obj_as_string_result") .allowlist_function("rb_str_byte_substr") .allowlist_function("rb_str_substr_two_fixnums") - - // From include/ruby/internal/intern/parse.h .allowlist_function("rb_backref_get") - - // From include/ruby/internal/intern/re.h .allowlist_function("rb_reg_last_match") .allowlist_function("rb_reg_match_pre") .allowlist_function("rb_reg_match_post") .allowlist_function("rb_reg_match_last") .allowlist_function("rb_reg_nth_match") - - // From internal/re.h .allowlist_function("rb_reg_new_ary") // `ruby_value_type` is a C enum and this stops it from // prefixing all the members with the name of the type .prepend_enum_name(false) .translate_enum_integer_types(true) // so we get fixed width Rust types for members - // From include/ruby/internal/value_type.h .allowlist_type("ruby_value_type") // really old C extension API - // From include/ruby/internal/hash.h .allowlist_type("ruby_rhash_flags") // really old C extension API - - // From method.h .allowlist_type("rb_method_visibility_t") .allowlist_type("rb_method_type_t") .allowlist_type("method_optimized_type") @@ -266,11 +199,7 @@ fn main() { .blocklist_type("rb_method_cfunc_t") .blocklist_type("rb_method_definition_.*") // Large struct with a bitfield and union of many types - don't import (yet?) .opaque_type("rb_method_definition_.*") - - // From numeric.c .allowlist_function("rb_float_new") - - // From vm_core.h .allowlist_var("rb_cRubyVM") .allowlist_var("rb_mRubyVMFrozenCore") .allowlist_var("VM_BLOCK_HANDLER_NONE") @@ -309,19 +238,17 @@ fn main() { .allowlist_type("vm_check_match_type") .allowlist_type("vm_opt_newarray_send_type") .allowlist_type("rb_iseq_type") - - // From yjit.c .allowlist_function("rb_object_shape_count") .allowlist_function("rb_ivar_get_at") .allowlist_function("rb_ivar_get_at_no_ractor_check") .allowlist_function("rb_iseq_(get|set)_yjit_payload") .allowlist_function("rb_iseq_pc_at_idx") .allowlist_function("rb_iseq_opcode_at_pc") - .allowlist_function("rb_yjit_reserve_addr_space") - .allowlist_function("rb_yjit_mark_writable") - .allowlist_function("rb_yjit_mark_executable") - .allowlist_function("rb_yjit_mark_unused") - .allowlist_function("rb_yjit_get_page_size") + .allowlist_function("rb_jit_reserve_addr_space") + .allowlist_function("rb_jit_mark_writable") + .allowlist_function("rb_jit_mark_executable") + .allowlist_function("rb_jit_mark_unused") + .allowlist_function("rb_jit_get_page_size") .allowlist_function("rb_yjit_iseq_builtin_attrs") .allowlist_function("rb_yjit_iseq_inspect") .allowlist_function("rb_yjit_builtin_function") @@ -340,15 +267,13 @@ fn main() { .allowlist_function("rb_ENCODING_GET") .allowlist_function("rb_yjit_get_proc_ptr") .allowlist_function("rb_yjit_exit_locations_dict") - .allowlist_function("rb_yjit_icache_invalidate") + .allowlist_function("rb_jit_icache_invalidate") .allowlist_function("rb_optimized_call") .allowlist_function("rb_yjit_sendish_sp_pops") .allowlist_function("rb_yjit_invokeblock_sp_pops") .allowlist_function("rb_yjit_set_exception_return") .allowlist_function("rb_yjit_str_concat_codepoint") .allowlist_type("rstring_offsets") - - // From jit.c .allowlist_function("rb_assert_holding_vm_lock") .allowlist_function("rb_jit_shape_too_complex_p") .allowlist_function("rb_jit_multi_ractor_p") @@ -356,58 +281,35 @@ fn main() { .allowlist_function("rb_jit_vm_unlock") .allowlist_function("rb_jit_for_each_iseq") .allowlist_type("robject_offsets") - - // from vm_sync.h .allowlist_function("rb_vm_barrier") // Not sure why it's picking these up, but don't. .blocklist_type("FILE") .blocklist_type("_IO_.*") - // From internal/compile.h .allowlist_function("rb_vm_insn_decode") - - // from internal/cont.h .allowlist_function("rb_jit_cont_each_iseq") - - // From iseq.h .allowlist_function("rb_vm_insn_addr2opcode") .allowlist_function("rb_iseqw_to_iseq") .allowlist_function("rb_iseq_label") .allowlist_function("rb_iseq_line_no") .allowlist_type("defined_type") - - // From builtin.h .allowlist_type("rb_builtin_function.*") - - // From internal/variable.h .allowlist_function("rb_gvar_(get|set)") .allowlist_function("rb_ensure_iv_list_size") - - // From include/ruby/internal/intern/variable.h .allowlist_function("rb_attr_get") .allowlist_function("rb_ivar_defined") .allowlist_function("rb_ivar_get") .allowlist_function("rb_mod_name") .allowlist_function("rb_const_get") - - // From internal/vm.h .allowlist_var("rb_vm_insn_count") - - // From include/ruby/internal/intern/vm.h .allowlist_function("rb_get_alloc_func") - - // From internal/object.h .allowlist_function("rb_class_allocate_instance") .allowlist_function("rb_obj_equal") .allowlist_function("rb_class_new_instance_pass_kw") .allowlist_function("rb_obj_alloc") - - // From gc.h and internal/gc.h .allowlist_function("rb_obj_info") .allowlist_function("ruby_xfree") - - // From include/ruby/debug.h .allowlist_function("rb_profile_frames") // Functions used for code generation @@ -491,7 +393,6 @@ fn main() { // We define VALUE manually, don't import it .blocklist_type("VALUE") - // From iseq.h .opaque_type("rb_iseq_t") .blocklist_type("rb_iseq_t") diff --git a/yjit/src/backend/arm64/mod.rs b/yjit/src/backend/arm64/mod.rs index 66e333f867d451..6fa8fef627df0c 100644 --- a/yjit/src/backend/arm64/mod.rs +++ b/yjit/src/backend/arm64/mod.rs @@ -98,7 +98,7 @@ fn emit_jmp_ptr_with_invalidation(cb: &mut CodeBlock, dst_ptr: CodePtr) { #[cfg(not(test))] { let end = cb.get_write_ptr(); - unsafe { rb_yjit_icache_invalidate(start.raw_ptr(cb) as _, end.raw_ptr(cb) as _) }; + unsafe { rb_jit_icache_invalidate(start.raw_ptr(cb) as _, end.raw_ptr(cb) as _) }; } } @@ -1361,7 +1361,7 @@ impl Assembler #[cfg(not(test))] cb.without_page_end_reserve(|cb| { for (start, end) in cb.writable_addrs(start_ptr, cb.get_write_ptr()) { - unsafe { rb_yjit_icache_invalidate(start as _, end as _) }; + unsafe { rb_jit_icache_invalidate(start as _, end as _) }; } }); diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs index 2bac85e62b57af..85fed25d24e48f 100644 --- a/yjit/src/codegen.rs +++ b/yjit/src/codegen.rs @@ -10945,7 +10945,7 @@ impl CodegenGlobals { #[cfg(not(test))] let (mut cb, mut ocb) = { - let virt_block: *mut u8 = unsafe { rb_yjit_reserve_addr_space(exec_mem_size as u32) }; + let virt_block: *mut u8 = unsafe { rb_jit_reserve_addr_space(exec_mem_size as u32) }; // Memory protection syscalls need page-aligned addresses, so check it here. Assuming // `virt_block` is page-aligned, `second_half` should be page-aligned as long as the @@ -10954,7 +10954,7 @@ impl CodegenGlobals { // // Basically, we don't support x86-64 2MiB and 1GiB pages. ARMv8 can do up to 64KiB // (2¹⁶ bytes) pages, which should be fine. 4KiB pages seem to be the most popular though. - let page_size = unsafe { rb_yjit_get_page_size() }; + let page_size = unsafe { rb_jit_get_page_size() }; assert_eq!( virt_block as usize % page_size.as_usize(), 0, "Start of virtual address block should be page-aligned", diff --git a/yjit/src/cruby_bindings.inc.rs b/yjit/src/cruby_bindings.inc.rs index a681d479770948..d9ef2d494f6d55 100644 --- a/yjit/src/cruby_bindings.inc.rs +++ b/yjit/src/cruby_bindings.inc.rs @@ -1164,21 +1164,12 @@ extern "C" { lines: *mut ::std::os::raw::c_int, ) -> ::std::os::raw::c_int; pub fn rb_jit_cont_each_iseq(callback: rb_iseq_callback, data: *mut ::std::os::raw::c_void); - pub fn rb_yjit_mark_writable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32) -> bool; - pub fn rb_yjit_mark_executable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32); - pub fn rb_yjit_mark_unused(mem_block: *mut ::std::os::raw::c_void, mem_size: u32) -> bool; pub fn rb_yjit_array_len(a: VALUE) -> ::std::os::raw::c_long; - pub fn rb_yjit_icache_invalidate( - start: *mut ::std::os::raw::c_void, - end: *mut ::std::os::raw::c_void, - ); pub fn rb_yjit_exit_locations_dict( yjit_raw_samples: *mut VALUE, yjit_line_samples: *mut ::std::os::raw::c_int, samples_len: ::std::os::raw::c_int, ) -> VALUE; - pub fn rb_yjit_get_page_size() -> u32; - pub fn rb_yjit_reserve_addr_space(mem_size: u32) -> *mut u8; pub fn rb_c_method_tracing_currently_enabled(ec: *const rb_execution_context_t) -> bool; pub fn rb_full_cfunc_return(ec: *mut rb_execution_context_t, return_value: VALUE); pub fn rb_iseq_get_yjit_payload(iseq: *const rb_iseq_t) -> *mut ::std::os::raw::c_void; @@ -1325,5 +1316,14 @@ extern "C" { line: ::std::os::raw::c_int, ); pub fn rb_iseq_reset_jit_func(iseq: *const rb_iseq_t); + pub fn rb_jit_get_page_size() -> u32; + pub fn rb_jit_reserve_addr_space(mem_size: u32) -> *mut u8; pub fn rb_jit_for_each_iseq(callback: rb_iseq_callback, data: *mut ::std::os::raw::c_void); + pub fn rb_jit_mark_writable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32) -> bool; + pub fn rb_jit_mark_executable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32); + pub fn rb_jit_mark_unused(mem_block: *mut ::std::os::raw::c_void, mem_size: u32) -> bool; + pub fn rb_jit_icache_invalidate( + start: *mut ::std::os::raw::c_void, + end: *mut ::std::os::raw::c_void, + ); } diff --git a/yjit/src/stats.rs b/yjit/src/stats.rs index ea6130973d22e8..09971c5b3afb48 100644 --- a/yjit/src/stats.rs +++ b/yjit/src/stats.rs @@ -1,9 +1,8 @@ //! Everything related to the collection of runtime stats in YJIT //! See the --yjit-stats command-line option -use std::alloc::{GlobalAlloc, Layout, System}; use std::ptr::addr_of_mut; -use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::atomic::Ordering; use std::time::Instant; use std::collections::HashMap; @@ -12,6 +11,10 @@ use crate::cruby::*; use crate::options::*; use crate::yjit::{yjit_enabled_p, YJIT_INIT_TIME}; +#[cfg(feature = "stats_allocator")] +#[path = "../../jit/src/lib.rs"] +mod jit; + /// Running total of how many ISeqs are in the system. #[no_mangle] pub static mut rb_yjit_live_iseq_count: u64 = 0; @@ -20,43 +23,9 @@ pub static mut rb_yjit_live_iseq_count: u64 = 0; #[no_mangle] pub static mut rb_yjit_iseq_alloc_count: u64 = 0; -/// A middleware to count Rust-allocated bytes as yjit_alloc_size. -#[global_allocator] -static GLOBAL_ALLOCATOR: StatsAlloc = StatsAlloc { alloc_size: AtomicUsize::new(0) }; - -pub struct StatsAlloc { - alloc_size: AtomicUsize, -} - -unsafe impl GlobalAlloc for StatsAlloc { - unsafe fn alloc(&self, layout: Layout) -> *mut u8 { - self.alloc_size.fetch_add(layout.size(), Ordering::SeqCst); - System.alloc(layout) - } - - unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { - self.alloc_size.fetch_sub(layout.size(), Ordering::SeqCst); - System.dealloc(ptr, layout) - } - - unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { - self.alloc_size.fetch_add(layout.size(), Ordering::SeqCst); - System.alloc_zeroed(layout) - } - - unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { - if new_size > layout.size() { - self.alloc_size.fetch_add(new_size - layout.size(), Ordering::SeqCst); - } else if new_size < layout.size() { - self.alloc_size.fetch_sub(layout.size() - new_size, Ordering::SeqCst); - } - System.realloc(ptr, layout, new_size) - } -} - /// The number of bytes YJIT has allocated on the Rust heap. pub fn yjit_alloc_size() -> usize { - GLOBAL_ALLOCATOR.alloc_size.load(Ordering::SeqCst) + jit::GLOBAL_ALLOCATOR.alloc_size.load(Ordering::SeqCst) } /// Mapping of C function / ISEQ name to integer indices diff --git a/yjit/src/virtualmem.rs b/yjit/src/virtualmem.rs index aa6d21f21035d4..58c9d9a7fd380a 100644 --- a/yjit/src/virtualmem.rs +++ b/yjit/src/virtualmem.rs @@ -7,6 +7,9 @@ use std::{cell::RefCell, ptr::NonNull}; use crate::{backend::ir::Target, stats::yjit_alloc_size, utils::IntoUsize}; +#[cfg(test)] +use crate::options::get_option; + #[cfg(not(test))] pub type VirtualMem = VirtualMemory; @@ -316,15 +319,15 @@ mod sys { impl super::Allocator for SystemAllocator { fn mark_writable(&mut self, ptr: *const u8, size: u32) -> bool { - unsafe { rb_yjit_mark_writable(ptr as VoidPtr, size) } + unsafe { rb_jit_mark_writable(ptr as VoidPtr, size) } } fn mark_executable(&mut self, ptr: *const u8, size: u32) { - unsafe { rb_yjit_mark_executable(ptr as VoidPtr, size) } + unsafe { rb_jit_mark_executable(ptr as VoidPtr, size) } } fn mark_unused(&mut self, ptr: *const u8, size: u32) -> bool { - unsafe { rb_yjit_mark_unused(ptr as VoidPtr, size) } + unsafe { rb_jit_mark_unused(ptr as VoidPtr, size) } } } } @@ -411,7 +414,7 @@ pub mod tests { PAGE_SIZE.try_into().unwrap(), NonNull::new(mem_start as *mut u8).unwrap(), mem_size, - 128 * 1024 * 1024, + get_option!(mem_size), ) } diff --git a/yjit/yjit.mk b/yjit/yjit.mk index 6b22a15960c1f3..cf68edb29770b8 100644 --- a/yjit/yjit.mk +++ b/yjit/yjit.mk @@ -6,6 +6,7 @@ YJIT_SRC_FILES = $(wildcard \ $(top_srcdir)/yjit/src/*/*.rs \ $(top_srcdir)/yjit/src/*/*/*.rs \ $(top_srcdir)/yjit/src/*/*/*/*.rs \ + $(top_srcdir)/jit/src/lib.rs \ ) # Because of Cargo cache, if the actual binary is not changed from the diff --git a/zjit.c b/zjit.c index 5d50775fc72c05..565a362a8a6aef 100644 --- a/zjit.c +++ b/zjit.c @@ -31,131 +31,6 @@ #include -uint32_t -rb_zjit_get_page_size(void) -{ -#if defined(_SC_PAGESIZE) - long page_size = sysconf(_SC_PAGESIZE); - if (page_size <= 0) rb_bug("zjit: failed to get page size"); - - // 1 GiB limit. x86 CPUs with PDPE1GB can do this and anything larger is unexpected. - // Though our design sort of assume we have fine grained control over memory protection - // which require small page sizes. - if (page_size > 0x40000000l) rb_bug("zjit page size too large"); - - return (uint32_t)page_size; -#else -#error "ZJIT supports POSIX only for now" -#endif -} - -#if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE) -// Align the current write position to a multiple of bytes -static uint8_t * -align_ptr(uint8_t *ptr, uint32_t multiple) -{ - // Compute the pointer modulo the given alignment boundary - uint32_t rem = ((uint32_t)(uintptr_t)ptr) % multiple; - - // If the pointer is already aligned, stop - if (rem == 0) - return ptr; - - // Pad the pointer by the necessary amount to align it - uint32_t pad = multiple - rem; - - return ptr + pad; -} -#endif - -// Address space reservation. Memory pages are mapped on an as needed basis. -// See the Rust mm module for details. -uint8_t * -rb_zjit_reserve_addr_space(uint32_t mem_size) -{ -#ifndef _WIN32 - uint8_t *mem_block; - - // On Linux - #if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE) - uint32_t const page_size = (uint32_t)sysconf(_SC_PAGESIZE); - uint8_t *const cfunc_sample_addr = (void *)(uintptr_t)&rb_zjit_reserve_addr_space; - uint8_t *const probe_region_end = cfunc_sample_addr + INT32_MAX; - // Align the requested address to page size - uint8_t *req_addr = align_ptr(cfunc_sample_addr, page_size); - - // Probe for addresses close to this function using MAP_FIXED_NOREPLACE - // to improve odds of being in range for 32-bit relative call instructions. - do { - mem_block = mmap( - req_addr, - mem_size, - PROT_NONE, - MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, - -1, - 0 - ); - - // If we succeeded, stop - if (mem_block != MAP_FAILED) { - ruby_annotate_mmap(mem_block, mem_size, "Ruby:rb_zjit_reserve_addr_space"); - break; - } - - // -4MiB. Downwards to probe away from the heap. (On x86/A64 Linux - // main_code_addr < heap_addr, and in case we are in a shared - // library mapped higher than the heap, downwards is still better - // since it's towards the end of the heap rather than the stack.) - req_addr -= 4 * 1024 * 1024; - } while (req_addr < probe_region_end); - - // On MacOS and other platforms - #else - // Try to map a chunk of memory as executable - mem_block = mmap( - (void *)rb_zjit_reserve_addr_space, - mem_size, - PROT_NONE, - MAP_PRIVATE | MAP_ANONYMOUS, - -1, - 0 - ); - #endif - - // Fallback - if (mem_block == MAP_FAILED) { - // Try again without the address hint (e.g., valgrind) - mem_block = mmap( - NULL, - mem_size, - PROT_NONE, - MAP_PRIVATE | MAP_ANONYMOUS, - -1, - 0 - ); - - if (mem_block != MAP_FAILED) { - ruby_annotate_mmap(mem_block, mem_size, "Ruby:rb_zjit_reserve_addr_space:fallback"); - } - } - - // Check that the memory mapping was successful - if (mem_block == MAP_FAILED) { - perror("ruby: zjit: mmap:"); - if(errno == ENOMEM) { - // No crash report if it's only insufficient memory - exit(EXIT_FAILURE); - } - rb_bug("mmap failed"); - } - - return mem_block; -#else - // Windows not supported for now - return NULL; -#endif -} - void rb_zjit_profile_disable(const rb_iseq_t *iseq); void @@ -185,56 +60,6 @@ rb_zjit_constcache_shareable(const struct iseq_inline_constant_cache_entry *ice) return (ice->flags & IMEMO_CONST_CACHE_SHAREABLE) != 0; } - -bool -rb_zjit_mark_writable(void *mem_block, uint32_t mem_size) -{ - return mprotect(mem_block, mem_size, PROT_READ | PROT_WRITE) == 0; -} - -void -rb_zjit_mark_executable(void *mem_block, uint32_t mem_size) -{ - // Do not call mprotect when mem_size is zero. Some platforms may return - // an error for it. https://github.com/Shopify/ruby/issues/450 - if (mem_size == 0) { - return; - } - if (mprotect(mem_block, mem_size, PROT_READ | PROT_EXEC)) { - rb_bug("Couldn't make JIT page (%p, %lu bytes) executable, errno: %s", - mem_block, (unsigned long)mem_size, strerror(errno)); - } -} - -// Free the specified memory block. -bool -rb_zjit_mark_unused(void *mem_block, uint32_t mem_size) -{ - // On Linux, you need to use madvise MADV_DONTNEED to free memory. - // We might not need to call this on macOS, but it's not really documented. - // We generally prefer to do the same thing on both to ease testing too. - madvise(mem_block, mem_size, MADV_DONTNEED); - - // On macOS, mprotect PROT_NONE seems to reduce RSS. - // We also call this on Linux to avoid executing unused pages. - return mprotect(mem_block, mem_size, PROT_NONE) == 0; -} - -// Invalidate icache for arm64. -// `start` is inclusive and `end` is exclusive. -void -rb_zjit_icache_invalidate(void *start, void *end) -{ - // Clear/invalidate the instruction cache. Compiles to nothing on x86_64 - // but required on ARM before running freshly written code. - // On Darwin it's the same as calling sys_icache_invalidate(). -#ifdef __GNUC__ - __builtin___clear_cache(start, end); -#elif defined(__aarch64__) -#error No instruction cache clear available with this compiler on Aarch64! -#endif -} - // Convert a given ISEQ's instructions to zjit_* instructions void rb_zjit_profile_enable(const rb_iseq_t *iseq) diff --git a/zjit.rb b/zjit.rb index 5e647bfbeeeb89..d6f7a4069267b8 100644 --- a/zjit.rb +++ b/zjit.rb @@ -44,6 +44,7 @@ def stats_string print_counters_with_prefix(prefix: 'unhandled_yarv_insn_', prompt: 'unhandled YARV insns', buf:, stats:, limit: 20) print_counters_with_prefix(prefix: 'compile_error_', prompt: 'compile error reasons', buf:, stats:, limit: 20) print_counters_with_prefix(prefix: 'exit_', prompt: 'side exit reasons', buf:, stats:, limit: 20) + print_counters_with_prefix(prefix: 'dynamic_send_type_', prompt: 'dynamic send types', buf:, stats:, limit: 20) # Show the most important stats ratio_in_zjit at the end print_counters([ diff --git a/zjit/Cargo.toml b/zjit/Cargo.toml index 7334d465c28ba8..c97c845a6eff7f 100644 --- a/zjit/Cargo.toml +++ b/zjit/Cargo.toml @@ -9,6 +9,7 @@ publish = false # Don't publish to crates.io # No required dependencies to simplify build process. TODO: Link to yet to be # written rationale. Optional For development and testing purposes capstone = { version = "0.13.0", optional = true } +jit = { version = "0.1.0", path = "../jit" } [dev-dependencies] insta = "1.43.1" @@ -19,3 +20,4 @@ insta = "1.43.1" # Support --yjit-dump-disasm and RubyVM::YJIT.disasm using libcapstone. disasm = ["capstone"] runtime_checks = [] +stats_allocator = [] diff --git a/zjit/bindgen/src/main.rs b/zjit/bindgen/src/main.rs index 56400e20cd742e..7d66bf0ecf3f16 100644 --- a/zjit/bindgen/src/main.rs +++ b/zjit/bindgen/src/main.rs @@ -69,32 +69,21 @@ fn main() { // Import YARV bytecode instruction constants .allowlist_type("ruby_vminsn_type") - // From include/ruby/internal/special_consts.h .allowlist_type("ruby_special_consts") - - // From include/ruby/internal/intern/string.h .allowlist_function("rb_utf8_str_new") .allowlist_function("rb_str_buf_append") .allowlist_function("rb_str_dup") - - // From encindex.h .allowlist_type("ruby_preserved_encindex") - - // From include/ruby/ruby.h .allowlist_function("rb_class2name") // This struct is public to Ruby C extensions - // From include/ruby/internal/core/rbasic.h .allowlist_type("RBasic") - // From include/ruby/internal/core/rstring.h .allowlist_type("ruby_rstring_flags") - // From internal.h // This function prints info about a value and is useful for debugging .allowlist_function("rb_obj_info_dump") - // For testing .allowlist_function("ruby_init") .allowlist_function("ruby_init_stack") .allowlist_function("ruby_options") @@ -106,40 +95,25 @@ fn main() { // For crashing .allowlist_function("rb_bug") - // From shape.h .allowlist_function("rb_obj_shape_id") .allowlist_function("rb_shape_id_offset") .allowlist_function("rb_shape_get_iv_index") .allowlist_function("rb_shape_transition_add_ivar_no_warnings") .allowlist_var("SHAPE_ID_NUM_BITS") - - // From ruby/internal/intern/object.h .allowlist_function("rb_obj_is_kind_of") .allowlist_function("rb_obj_frozen_p") .allowlist_function("rb_class_inherited_p") - - // From ruby/internal/encoding/encoding.h .allowlist_type("ruby_encoding_consts") - - // From include/hash.h .allowlist_function("rb_hash_new") - - // From internal/hash.h .allowlist_function("rb_hash_new_with_size") .allowlist_function("rb_hash_resurrect") .allowlist_function("rb_hash_stlike_foreach") .allowlist_function("rb_to_hash_type") - - // From include/ruby/st.h .allowlist_type("st_retval") - - // From include/ruby/internal/intern/hash.h .allowlist_function("rb_hash_aset") .allowlist_function("rb_hash_aref") .allowlist_function("rb_hash_bulk_insert") .allowlist_function("rb_hash_stlike_lookup") - - // From include/ruby/internal/intern/array.h .allowlist_function("rb_ary_new_capa") .allowlist_function("rb_ary_store") .allowlist_function("rb_ary_resurrect") @@ -149,20 +123,12 @@ fn main() { .allowlist_function("rb_ary_dup") .allowlist_function("rb_ary_push") .allowlist_function("rb_ary_unshift_m") - - // From internal/array.h .allowlist_function("rb_ec_ary_new_from_values") .allowlist_function("rb_ary_tmp_new_from_values") - - // From include/ruby/internal/intern/class.h .allowlist_function("rb_class_attached_object") .allowlist_function("rb_singleton_class") .allowlist_function("rb_define_class") - - // From include/ruby/internal/core/rclass.h .allowlist_function("rb_class_get_superclass") - - // From include/ruby/internal/gc.h .allowlist_function("rb_gc_mark") .allowlist_function("rb_gc_mark_movable") .allowlist_function("rb_gc_location") @@ -170,7 +136,6 @@ fn main() { .allowlist_function("rb_gc_writebarrier_remember") // VALUE variables for Ruby class objects - // From include/ruby/internal/globals.h .allowlist_var("rb_cBasicObject") .allowlist_var("rb_cObject") .allowlist_var("rb_cModule") @@ -192,38 +157,21 @@ fn main() { .allowlist_var("rb_cRegexp") .allowlist_var("rb_cISeq") - // From include/ruby/internal/fl_type.h .allowlist_type("ruby_fl_type") .allowlist_type("ruby_fl_ushift") - - // From include/ruby/internal/core/robject.h .allowlist_type("ruby_robject_flags") - - // From include/ruby/internal/core/rarray.h .allowlist_type("ruby_rarray_flags") .allowlist_type("ruby_rarray_consts") - - // From include/ruby/internal/core/rclass.h .allowlist_type("ruby_rmodule_flags") - - // From ruby/internal/globals.h .allowlist_var("rb_mKernel") - - // From vm_callinfo.h .allowlist_type("vm_call_flag_bits") .allowlist_type("rb_call_data") .blocklist_type("rb_callcache.*") // Not used yet - opaque to make it easy to import rb_call_data .opaque_type("rb_callcache.*") .allowlist_type("rb_callinfo") - - // From vm_insnhelper.h .allowlist_var("VM_ENV_DATA_INDEX_ME_CREF") .allowlist_var("rb_block_param_proxy") - - // From include/ruby/internal/intern/range.h .allowlist_function("rb_range_new") - - // From include/ruby/internal/symbol.h .allowlist_function("rb_intern") .allowlist_function("rb_intern2") .allowlist_function("rb_id2sym") @@ -231,38 +179,26 @@ fn main() { .allowlist_function("rb_str_intern") .allowlist_function("rb_id2str") .allowlist_function("rb_sym2str") - - // From internal/numeric.h .allowlist_function("rb_fix_aref") .allowlist_function("rb_float_plus") .allowlist_function("rb_float_minus") .allowlist_function("rb_float_mul") .allowlist_function("rb_float_div") - - // From internal/string.h .allowlist_type("ruby_rstring_private_flags") .allowlist_function("rb_ec_str_resurrect") .allowlist_function("rb_str_concat_literals") .allowlist_function("rb_obj_as_string_result") .allowlist_function("rb_str_byte_substr") .allowlist_function("rb_str_substr_two_fixnums") - - // From include/ruby/internal/intern/parse.h .allowlist_function("rb_backref_get") - - // From include/ruby/internal/intern/re.h .allowlist_function("rb_reg_last_match") .allowlist_function("rb_reg_match_pre") .allowlist_function("rb_reg_match_post") .allowlist_function("rb_reg_match_last") .allowlist_function("rb_reg_nth_match") - - // From internal/re.h .allowlist_function("rb_reg_new_ary") .allowlist_var("ARG_ENCODING_FIXED") .allowlist_var("ARG_ENCODING_NONE") - - // From include/ruby/onigmo.h .allowlist_var("ONIG_OPTION_IGNORECASE") .allowlist_var("ONIG_OPTION_EXTEND") .allowlist_var("ONIG_OPTION_MULTILINE") @@ -271,13 +207,9 @@ fn main() { // prefixing all the members with the name of the type .prepend_enum_name(false) .translate_enum_integer_types(true) // so we get fixed width Rust types for members - // From include/ruby/internal/value_type.h .allowlist_type("ruby_value_type") // really old C extension API - // From include/ruby/internal/hash.h .allowlist_type("ruby_rhash_flags") // really old C extension API - - // From method.h .allowlist_type("rb_method_visibility_t") .allowlist_type("rb_method_type_t") .allowlist_type("method_optimized_type") @@ -288,11 +220,7 @@ fn main() { .blocklist_type("rb_method_cfunc_t") .blocklist_type("rb_method_definition_.*") // Large struct with a bitfield and union of many types - don't import (yet?) .opaque_type("rb_method_definition_.*") - - // From numeric.c .allowlist_function("rb_float_new") - - // From vm_core.h .allowlist_var("rb_mRubyVMFrozenCore") .allowlist_var("VM_BLOCK_HANDLER_NONE") .allowlist_type("vm_frame_env_flags") @@ -331,17 +259,15 @@ fn main() { .allowlist_type("vm_opt_newarray_send_type") .allowlist_type("rb_iseq_type") .allowlist_type("rb_event_flag_t") - - // From zjit.c .allowlist_function("rb_object_shape_count") .allowlist_function("rb_iseq_(get|set)_zjit_payload") .allowlist_function("rb_iseq_pc_at_idx") .allowlist_function("rb_iseq_opcode_at_pc") - .allowlist_function("rb_zjit_reserve_addr_space") - .allowlist_function("rb_zjit_mark_writable") - .allowlist_function("rb_zjit_mark_executable") - .allowlist_function("rb_zjit_mark_unused") - .allowlist_function("rb_zjit_get_page_size") + .allowlist_function("rb_jit_reserve_addr_space") + .allowlist_function("rb_jit_mark_writable") + .allowlist_function("rb_jit_mark_executable") + .allowlist_function("rb_jit_mark_unused") + .allowlist_function("rb_jit_get_page_size") .allowlist_function("rb_zjit_iseq_builtin_attrs") .allowlist_function("rb_zjit_iseq_inspect") .allowlist_function("rb_zjit_iseq_insn_set") @@ -354,7 +280,7 @@ fn main() { .allowlist_function("rb_RSTRING_LEN") .allowlist_function("rb_ENCODING_GET") .allowlist_function("rb_optimized_call") - .allowlist_function("rb_zjit_icache_invalidate") + .allowlist_function("rb_jit_icache_invalidate") .allowlist_function("rb_zjit_print_exception") .allowlist_function("rb_zjit_singleton_class_p") .allowlist_function("rb_zjit_defined_ivar") @@ -362,8 +288,6 @@ fn main() { .allowlist_type("robject_offsets") .allowlist_type("rstring_offsets") .allowlist_var("RB_INVALID_SHAPE_ID") - - // From jit.c .allowlist_function("rb_assert_holding_vm_lock") .allowlist_function("rb_jit_shape_too_complex_p") .allowlist_function("rb_jit_multi_ractor_p") @@ -372,59 +296,36 @@ fn main() { .allowlist_function("rb_jit_for_each_iseq") .allowlist_function("rb_iseq_reset_jit_func") .allowlist_type("robject_offsets") - - // from vm_sync.h .allowlist_function("rb_vm_barrier") // Not sure why it's picking these up, but don't. .blocklist_type("FILE") .blocklist_type("_IO_.*") - // From internal/compile.h .allowlist_function("rb_vm_insn_decode") - - // from internal/cont.h .allowlist_function("rb_jit_cont_each_iseq") - - // From iseq.h .allowlist_function("rb_vm_insn_addr2opcode") .allowlist_function("rb_iseqw_to_iseq") .allowlist_function("rb_iseq_label") .allowlist_function("rb_iseq_line_no") .allowlist_function("rb_iseq_defined_string") .allowlist_type("defined_type") - - // From builtin.h .allowlist_type("rb_builtin_function.*") - - // From internal/variable.h .allowlist_function("rb_gvar_(get|set)") .allowlist_function("rb_ensure_iv_list_size") - - // From include/ruby/internal/intern/variable.h .allowlist_function("rb_attr_get") .allowlist_function("rb_ivar_defined") .allowlist_function("rb_ivar_get") .allowlist_function("rb_ivar_set") .allowlist_function("rb_mod_name") - - // From internal/vm.h .allowlist_var("rb_vm_insn_count") - - // From include/ruby/internal/intern/vm.h .allowlist_function("rb_get_alloc_func") - - // From internal/object.h .allowlist_function("rb_class_allocate_instance") .allowlist_function("rb_obj_equal") .allowlist_function("rb_class_new_instance_pass_kw") .allowlist_function("rb_obj_alloc") - - // From gc.h and internal/gc.h .allowlist_function("rb_obj_info") .allowlist_function("ruby_xfree") - - // From include/ruby/debug.h .allowlist_function("rb_profile_frames") // Functions used for code generation @@ -505,7 +406,6 @@ fn main() { .blocklist_type("VALUE") .blocklist_type("ID") - // From iseq.h .opaque_type("rb_iseq_t") .blocklist_type("rb_iseq_t") diff --git a/zjit/src/backend/arm64/mod.rs b/zjit/src/backend/arm64/mod.rs index 6e040e5214f0c8..50e7074de11222 100644 --- a/zjit/src/backend/arm64/mod.rs +++ b/zjit/src/backend/arm64/mod.rs @@ -99,7 +99,7 @@ fn emit_jmp_ptr_with_invalidation(cb: &mut CodeBlock, dst_ptr: CodePtr) { let start = cb.get_write_ptr(); emit_jmp_ptr(cb, dst_ptr, true); let end = cb.get_write_ptr(); - unsafe { rb_zjit_icache_invalidate(start.raw_ptr(cb) as _, end.raw_ptr(cb) as _) }; + unsafe { rb_jit_icache_invalidate(start.raw_ptr(cb) as _, end.raw_ptr(cb) as _) }; } fn emit_jmp_ptr(cb: &mut CodeBlock, dst_ptr: CodePtr, padding: bool) { @@ -1382,7 +1382,7 @@ impl Assembler cb.link_labels(); // Invalidate icache for newly written out region so we don't run stale code. - unsafe { rb_zjit_icache_invalidate(start_ptr.raw_ptr(cb) as _, cb.get_write_ptr().raw_ptr(cb) as _) }; + unsafe { rb_jit_icache_invalidate(start_ptr.raw_ptr(cb) as _, cb.get_write_ptr().raw_ptr(cb) as _) }; Ok((start_ptr, gc_offsets)) } else { diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs index 2a581d8baddac8..1f71ad9db3b2b8 100644 --- a/zjit/src/codegen.rs +++ b/zjit/src/codegen.rs @@ -370,7 +370,7 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio gen_send_without_block(jit, asm, *cd, &function.frame_state(*state)), Insn::SendWithoutBlockDirect { cme, iseq, recv, args, state, .. } => gen_send_without_block_direct(cb, jit, asm, *cme, *iseq, opnd!(recv), opnds!(args), &function.frame_state(*state)), &Insn::InvokeSuper { cd, blockiseq, state, .. } => gen_invokesuper(jit, asm, cd, blockiseq, &function.frame_state(state)), - Insn::InvokeBlock { cd, state, .. } => gen_invoke_block(jit, asm, *cd, &function.frame_state(*state)), + Insn::InvokeBlock { cd, state, .. } => gen_invokeblock(jit, asm, *cd, &function.frame_state(*state)), // Ensure we have enough room fit ec, self, and arguments // TODO remove this check when we have stack args (we can use Time.new to test it) Insn::InvokeBuiltin { bf, state, .. } if bf.argc + 2 > (C_ARG_OPNDS.len() as i32) => return Err(*state), @@ -981,6 +981,7 @@ fn gen_send( state: &FrameState, ) -> lir::Opnd { gen_incr_counter(asm, Counter::dynamic_send_count); + gen_incr_counter(asm, Counter::dynamic_send_type_send); // Save PC and SP gen_prepare_call_with_gc(asm, state); @@ -1008,6 +1009,7 @@ fn gen_send_without_block( state: &FrameState, ) -> lir::Opnd { gen_incr_counter(asm, Counter::dynamic_send_count); + gen_incr_counter(asm, Counter::dynamic_send_type_send_without_block); // Note that it's incorrect to use this frame state to side exit because // the state might not be on the boundary of an interpreter instruction. @@ -1108,13 +1110,14 @@ fn gen_send_without_block_direct( } /// Compile for invokeblock -fn gen_invoke_block( +fn gen_invokeblock( jit: &mut JITState, asm: &mut Assembler, cd: *const rb_call_data, state: &FrameState, ) -> lir::Opnd { gen_incr_counter(asm, Counter::dynamic_send_count); + gen_incr_counter(asm, Counter::dynamic_send_type_invokeblock); // Save PC and SP, spill locals and stack gen_prepare_call_with_gc(asm, state); @@ -1141,6 +1144,7 @@ fn gen_invokesuper( state: &FrameState, ) -> lir::Opnd { gen_incr_counter(asm, Counter::dynamic_send_count); + gen_incr_counter(asm, Counter::dynamic_send_type_invokesuper); // Save PC and SP gen_prepare_call_with_gc(asm, state); diff --git a/zjit/src/cruby_bindings.inc.rs b/zjit/src/cruby_bindings.inc.rs index bab40e50ee32cc..9ee4b1bb743958 100644 --- a/zjit/src/cruby_bindings.inc.rs +++ b/zjit/src/cruby_bindings.inc.rs @@ -922,18 +922,9 @@ unsafe extern "C" { lines: *mut ::std::os::raw::c_int, ) -> ::std::os::raw::c_int; pub fn rb_jit_cont_each_iseq(callback: rb_iseq_callback, data: *mut ::std::os::raw::c_void); - pub fn rb_zjit_get_page_size() -> u32; - pub fn rb_zjit_reserve_addr_space(mem_size: u32) -> *mut u8; pub fn rb_zjit_profile_disable(iseq: *const rb_iseq_t); pub fn rb_vm_base_ptr(cfp: *mut rb_control_frame_struct) -> *mut VALUE; pub fn rb_zjit_constcache_shareable(ice: *const iseq_inline_constant_cache_entry) -> bool; - pub fn rb_zjit_mark_writable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32) -> bool; - pub fn rb_zjit_mark_executable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32); - pub fn rb_zjit_mark_unused(mem_block: *mut ::std::os::raw::c_void, mem_size: u32) -> bool; - pub fn rb_zjit_icache_invalidate( - start: *mut ::std::os::raw::c_void, - end: *mut ::std::os::raw::c_void, - ); pub fn rb_zjit_iseq_insn_set( iseq: *const rb_iseq_t, insn_idx: ::std::os::raw::c_uint, @@ -1037,5 +1028,14 @@ unsafe extern "C" { line: ::std::os::raw::c_int, ); pub fn rb_iseq_reset_jit_func(iseq: *const rb_iseq_t); + pub fn rb_jit_get_page_size() -> u32; + pub fn rb_jit_reserve_addr_space(mem_size: u32) -> *mut u8; pub fn rb_jit_for_each_iseq(callback: rb_iseq_callback, data: *mut ::std::os::raw::c_void); + pub fn rb_jit_mark_writable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32) -> bool; + pub fn rb_jit_mark_executable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32); + pub fn rb_jit_mark_unused(mem_block: *mut ::std::os::raw::c_void, mem_size: u32) -> bool; + pub fn rb_jit_icache_invalidate( + start: *mut ::std::os::raw::c_void, + end: *mut ::std::os::raw::c_void, + ); } diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index 4ade541922641f..3726d8ec0e44d5 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -2968,17 +2968,8 @@ fn compute_bytecode_info(iseq: *const rb_iseq_t) -> BytecodeInfo { #[derive(Debug, PartialEq, Clone, Copy)] pub enum CallType { - Splat, BlockArg, - Kwarg, - KwSplat, Tailcall, - Super, - Zsuper, - OptSend, - KwSplatMut, - SplatMut, - Forwarding, } #[derive(Clone, Debug, PartialEq)] @@ -2996,17 +2987,8 @@ fn num_locals(iseq: *const rb_iseq_t) -> usize { /// If we can't handle the type of send (yet), bail out. fn unknown_call_type(flag: u32) -> Result<(), CallType> { - if (flag & VM_CALL_KW_SPLAT_MUT) != 0 { return Err(CallType::KwSplatMut); } - if (flag & VM_CALL_ARGS_SPLAT_MUT) != 0 { return Err(CallType::SplatMut); } - if (flag & VM_CALL_ARGS_SPLAT) != 0 { return Err(CallType::Splat); } - if (flag & VM_CALL_KW_SPLAT) != 0 { return Err(CallType::KwSplat); } if (flag & VM_CALL_ARGS_BLOCKARG) != 0 { return Err(CallType::BlockArg); } - if (flag & VM_CALL_KWARG) != 0 { return Err(CallType::Kwarg); } if (flag & VM_CALL_TAILCALL) != 0 { return Err(CallType::Tailcall); } - if (flag & VM_CALL_SUPER) != 0 { return Err(CallType::Super); } - if (flag & VM_CALL_ZSUPER) != 0 { return Err(CallType::Zsuper); } - if (flag & VM_CALL_OPT_SEND) != 0 { return Err(CallType::OptSend); } - if (flag & VM_CALL_FORWARDING) != 0 { return Err(CallType::Forwarding); } Ok(()) } @@ -5126,7 +5108,9 @@ mod tests { fn test@:2: bb0(v0:BasicObject, v1:BasicObject): v6:ArrayExact = ToArray v1 - SideExit UnhandledCallType(Splat) + v8:BasicObject = SendWithoutBlock v0, :foo, v6 + CheckInterrupts + Return v8 "); } @@ -5151,7 +5135,9 @@ mod tests { fn test@:2: bb0(v0:BasicObject, v1:BasicObject): v5:Fixnum[1] = Const Value(1) - SideExit UnhandledCallType(Kwarg) + v7:BasicObject = SendWithoutBlock v0, :foo, v5 + CheckInterrupts + Return v7 "); } @@ -5163,7 +5149,9 @@ mod tests { assert_snapshot!(hir_string("test"), @r" fn test@:2: bb0(v0:BasicObject, v1:BasicObject): - SideExit UnhandledCallType(KwSplat) + v6:BasicObject = SendWithoutBlock v0, :foo, v1 + CheckInterrupts + Return v6 "); } @@ -5252,7 +5240,9 @@ mod tests { v13:StaticSymbol[:b] = Const Value(VALUE(0x1008)) v14:Fixnum[1] = Const Value(1) v16:BasicObject = SendWithoutBlock v12, :core#hash_merge_ptr, v11, v13, v14 - SideExit UnhandledCallType(KwSplatMut) + v18:BasicObject = SendWithoutBlock v0, :foo, v16 + CheckInterrupts + Return v18 "); } @@ -5267,7 +5257,9 @@ mod tests { v6:ArrayExact = ToNewArray v1 v7:Fixnum[1] = Const Value(1) ArrayPush v6, v7 - SideExit UnhandledCallType(SplatMut) + v11:BasicObject = SendWithoutBlock v0, :foo, v6 + CheckInterrupts + Return v11 "); } @@ -7863,7 +7855,9 @@ mod opt_tests { fn test@:3: bb0(v0:BasicObject): v4:Fixnum[1] = Const Value(1) - SideExit UnhandledCallType(Kwarg) + v6:BasicObject = SendWithoutBlock v0, :foo, v4 + CheckInterrupts + Return v6 "); } @@ -7879,7 +7873,9 @@ mod opt_tests { fn test@:3: bb0(v0:BasicObject): v4:Fixnum[1] = Const Value(1) - SideExit UnhandledCallType(Kwarg) + v6:BasicObject = SendWithoutBlock v0, :foo, v4 + CheckInterrupts + Return v6 "); } diff --git a/zjit/src/options.rs b/zjit/src/options.rs index dbb6ee8ebbdf24..2a9b2e7d279055 100644 --- a/zjit/src/options.rs +++ b/zjit/src/options.rs @@ -35,6 +35,10 @@ pub struct Options { /// Note that the command line argument is expressed in MiB and not bytes. pub exec_mem_bytes: usize, + /// Hard limit of ZJIT's total memory usage. + /// Note that the command line argument is expressed in MiB and not bytes. + pub mem_bytes: usize, + /// Number of times YARV instructions should be profiled. pub num_profiles: u8, @@ -79,6 +83,7 @@ impl Default for Options { fn default() -> Self { Options { exec_mem_bytes: 64 * 1024 * 1024, + mem_bytes: 128 * 1024 * 1024, num_profiles: DEFAULT_NUM_PROFILES, stats: false, print_stats: false, @@ -100,9 +105,8 @@ impl Default for Options { /// Note that --help allows only 80 chars per line, including indentation, and it also puts the /// description in a separate line if the option name is too long. 80-char limit --> | (any character beyond this `|` column fails the test) pub const ZJIT_OPTIONS: &[(&str, &str)] = &[ - // TODO: Hide --zjit-exec-mem-size from ZJIT_OPTIONS once we add --zjit-mem-size (Shopify/ruby#686) - ("--zjit-exec-mem-size=num", - "Size of executable memory block in MiB (default: 64)."), + ("--zjit-mem-size=num", + "Max amount of memory that ZJIT can use (in MiB)."), ("--zjit-call-threshold=num", "Number of calls to trigger JIT (default: 2)."), ("--zjit-num-profiles=num", diff --git a/zjit/src/state.rs b/zjit/src/state.rs index 02bba3b7a3e8a8..0c657f450a25dc 100644 --- a/zjit/src/state.rs +++ b/zjit/src/state.rs @@ -60,7 +60,7 @@ impl ZJITState { use crate::options::*; let exec_mem_bytes: usize = get_option!(exec_mem_bytes); - let virt_block: *mut u8 = unsafe { rb_zjit_reserve_addr_space(64 * 1024 * 1024) }; + let virt_block: *mut u8 = unsafe { rb_jit_reserve_addr_space(64 * 1024 * 1024) }; // Memory protection syscalls need page-aligned addresses, so check it here. Assuming // `virt_block` is page-aligned, `second_half` should be page-aligned as long as the @@ -69,7 +69,7 @@ impl ZJITState { // // Basically, we don't support x86-64 2MiB and 1GiB pages. ARMv8 can do up to 64KiB // (2¹⁶ bytes) pages, which should be fine. 4KiB pages seem to be the most popular though. - let page_size = unsafe { rb_zjit_get_page_size() }; + let page_size = unsafe { rb_jit_get_page_size() }; assert_eq!( virt_block as usize % page_size as usize, 0, "Start of virtual address block should be page-aligned", @@ -85,7 +85,7 @@ impl ZJITState { page_size, NonNull::new(virt_block).unwrap(), exec_mem_bytes, - exec_mem_bytes, // TODO: change this to --zjit-mem-size (Shopify/ruby#686) + get_option!(mem_bytes) ); let mem_block = Rc::new(RefCell::new(mem_block)); diff --git a/zjit/src/stats.rs b/zjit/src/stats.rs index d1a6d584b9c020..8a7073dd977c94 100644 --- a/zjit/src/stats.rs +++ b/zjit/src/stats.rs @@ -1,6 +1,11 @@ //! Counters and associated methods for events when ZJIT is run. use std::time::Instant; +use std::sync::atomic::Ordering; + +#[cfg(feature = "stats_allocator")] +#[path = "../../jit/src/lib.rs"] +mod jit; use crate::{cruby::*, hir::ParseError, options::get_option, state::{zjit_enabled_p, ZJITState}}; @@ -109,17 +114,8 @@ make_counters! { } // unhanded_call_: Unhandled call types - unhandled_call_splat, unhandled_call_block_arg, - unhandled_call_kwarg, - unhandled_call_kw_splat, unhandled_call_tailcall, - unhandled_call_super, - unhandled_call_zsuper, - unhandled_call_optsend, - unhandled_call_kw_splat_mut, - unhandled_call_splat_mut, - unhandled_call_forwarding, // compile_error_: Compile error reasons compile_error_iseq_stack_too_large, @@ -137,6 +133,10 @@ make_counters! { // The number of times we do a dynamic dispatch from JIT code dynamic_send_count, + dynamic_send_type_send_without_block, + dynamic_send_type_send, + dynamic_send_type_invokeblock, + dynamic_send_type_invokesuper, } /// Increase a counter by a specified amount @@ -167,17 +167,8 @@ pub fn exit_counter_ptr_for_call_type(call_type: crate::hir::CallType) -> *mut u use crate::hir::CallType::*; use crate::stats::Counter::*; let counter = match call_type { - Splat => unhandled_call_splat, - BlockArg => unhandled_call_block_arg, - Kwarg => unhandled_call_kwarg, - KwSplat => unhandled_call_kw_splat, - Tailcall => unhandled_call_tailcall, - Super => unhandled_call_super, - Zsuper => unhandled_call_zsuper, - OptSend => unhandled_call_optsend, - KwSplatMut => unhandled_call_kw_splat_mut, - SplatMut => unhandled_call_splat_mut, - Forwarding => unhandled_call_forwarding, + BlockArg => unhandled_call_block_arg, + Tailcall => unhandled_call_tailcall, }; counter_ptr(counter) } @@ -353,5 +344,5 @@ pub fn with_time_stat(counter: Counter, func: F) -> R where F: FnOnce() -> /// The number of bytes ZJIT has allocated on the Rust heap. pub fn zjit_alloc_size() -> usize { - 0 // TODO: report the actual memory usage to support --zjit-mem-size (Shopify/ruby#686) + jit::GLOBAL_ALLOCATOR.alloc_size.load(Ordering::SeqCst) } diff --git a/zjit/src/virtualmem.rs b/zjit/src/virtualmem.rs index 42ce525fde7a6c..0d19858c8711f5 100644 --- a/zjit/src/virtualmem.rs +++ b/zjit/src/virtualmem.rs @@ -7,6 +7,9 @@ use std::ptr::NonNull; use crate::stats::zjit_alloc_size; +#[cfg(test)] +use crate::options::get_option; + #[cfg(not(test))] pub type VirtualMem = VirtualMemory; @@ -287,15 +290,15 @@ pub mod sys { impl super::Allocator for SystemAllocator { fn mark_writable(&mut self, ptr: *const u8, size: u32) -> bool { - unsafe { rb_zjit_mark_writable(ptr as VoidPtr, size) } + unsafe { rb_jit_mark_writable(ptr as VoidPtr, size) } } fn mark_executable(&mut self, ptr: *const u8, size: u32) { - unsafe { rb_zjit_mark_executable(ptr as VoidPtr, size) } + unsafe { rb_jit_mark_executable(ptr as VoidPtr, size) } } fn mark_unused(&mut self, ptr: *const u8, size: u32) -> bool { - unsafe { rb_zjit_mark_unused(ptr as VoidPtr, size) } + unsafe { rb_jit_mark_unused(ptr as VoidPtr, size) } } } } @@ -369,6 +372,12 @@ pub mod tests { // Fictional architecture where each page is 4 bytes long const PAGE_SIZE: usize = 4; fn new_dummy_virt_mem() -> VirtualMemory { + unsafe { + if crate::options::OPTIONS.is_none() { + crate::options::OPTIONS = Some(crate::options::Options::default()); + } + } + let mem_size = PAGE_SIZE * 10; let alloc = TestingAllocator::new(mem_size); let mem_start: *const u8 = alloc.mem_start(); @@ -378,7 +387,7 @@ pub mod tests { PAGE_SIZE.try_into().unwrap(), NonNull::new(mem_start as *mut u8).unwrap(), mem_size, - 128 * 1024 * 1024, + get_option!(mem_bytes), ) } diff --git a/zjit/zjit.mk b/zjit/zjit.mk index be989bdecd41c3..f0bf1b0da59fb2 100644 --- a/zjit/zjit.mk +++ b/zjit/zjit.mk @@ -9,6 +9,7 @@ ZJIT_SRC_FILES = $(wildcard \ $(top_srcdir)/zjit/src/*/*.rs \ $(top_srcdir)/zjit/src/*/*/*.rs \ $(top_srcdir)/zjit/src/*/*/*/*.rs \ + $(top_srcdir)/jit/src/lib.rs \ ) $(RUST_LIB): $(ZJIT_SRC_FILES)