From 292c9793abb20242b12973c5fb9db5c612e2a90a Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Tue, 1 Oct 2024 02:55:16 +0900 Subject: [PATCH 01/21] Remove no longer needed excludes files --- test/.excludes-parsey/TestBugReporter.rb | 1 - test/.excludes-parsey/TestRubyOptions.rb | 10 ---------- 2 files changed, 11 deletions(-) delete mode 100644 test/.excludes-parsey/TestBugReporter.rb delete mode 100644 test/.excludes-parsey/TestRubyOptions.rb diff --git a/test/.excludes-parsey/TestBugReporter.rb b/test/.excludes-parsey/TestBugReporter.rb deleted file mode 100644 index 72357760a67b8c..00000000000000 --- a/test/.excludes-parsey/TestBugReporter.rb +++ /dev/null @@ -1 +0,0 @@ -exclude(:test_bug_reporter_add, "unexpected +PRISM in the Ruby description") diff --git a/test/.excludes-parsey/TestRubyOptions.rb b/test/.excludes-parsey/TestRubyOptions.rb deleted file mode 100644 index f6ac8c744a1361..00000000000000 --- a/test/.excludes-parsey/TestRubyOptions.rb +++ /dev/null @@ -1,10 +0,0 @@ -exclude(:test_crash_report_executable_path, "unexpected +PRISM in the Ruby description") -exclude(:test_crash_report_script_path, "unexpected +PRISM in the Ruby description") -exclude(:test_crash_report_script, "unexpected +PRISM in the Ruby description") -exclude(:test_crash_report, "unexpected +PRISM in the Ruby description") -exclude(:test_rjit_disabled_version, "unexpected +PRISM in the Ruby description") -exclude(:test_segv_loaded_features, "unexpected +PRISM in the Ruby description") -exclude(:test_segv_setproctitle, "unexpected +PRISM in the Ruby description") -exclude(:test_segv_test, "unexpected +PRISM in the Ruby description") -exclude(:test_verbose, "unexpected +PRISM in the Ruby description") -exclude(:test_version, "unexpected +PRISM in the Ruby description") From 30507a4aed83ee0712fbe84875b1d9bec1f1dd70 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 30 Sep 2024 13:37:28 -0400 Subject: [PATCH 02/21] Move RUBY_INTERNAL_EVENT_FREEOBJ into GC implementation Instead of calling rb_gc_event_hook inside of rb_gc_obj_free, it should be up to the GC implementation to call the event. --- gc.c | 2 -- gc/default.c | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gc.c b/gc.c index ba7007da39da8c..f4d4e137ae41e3 100644 --- a/gc.c +++ b/gc.c @@ -1087,8 +1087,6 @@ rb_gc_obj_free(void *objspace, VALUE obj) { RB_DEBUG_COUNTER_INC(obj_free); - rb_gc_event_hook(obj, RUBY_INTERNAL_EVENT_FREEOBJ); - switch (BUILTIN_TYPE(obj)) { case T_NIL: case T_FIXNUM: diff --git a/gc/default.c b/gc/default.c index 4df9a8b9fd47e0..09acad161871b1 100644 --- a/gc/default.c +++ b/gc/default.c @@ -3565,6 +3565,8 @@ gc_sweep_plane(rb_objspace_t *objspace, rb_heap_t *heap, uintptr_t p, bits_t bit #undef CHECK #endif + rb_gc_event_hook(vp, RUBY_INTERNAL_EVENT_FREEOBJ); + bool has_object_id = FL_TEST(vp, FL_SEEN_OBJ_ID); if (rb_gc_obj_free(objspace, vp)) { if (has_object_id) { From af63b4f8b7a659ab78a75af97416c042ca357a3b Mon Sep 17 00:00:00 2001 From: whtsht <85547207+whtsht@users.noreply.github.com> Date: Tue, 1 Oct 2024 08:00:54 +0900 Subject: [PATCH 03/21] Return an Iterator Instead of a Vector in `addrs_to_pages` Method (#11725) * Returning an iterator instead of a vec * Avoid changing the meaning of end_page --------- Co-authored-by: Takashi Kokubun --- yjit/src/asm/mod.rs | 6 +++--- yjit/src/codegen.rs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/yjit/src/asm/mod.rs b/yjit/src/asm/mod.rs index bd56074b96e7f7..a113a41c739949 100644 --- a/yjit/src/asm/mod.rs +++ b/yjit/src/asm/mod.rs @@ -432,7 +432,7 @@ impl CodeBlock { } /// Convert an address range to memory page indexes against a num_pages()-sized array. - pub fn addrs_to_pages(&self, start_addr: CodePtr, end_addr: CodePtr) -> Vec { + pub fn addrs_to_pages(&self, start_addr: CodePtr, end_addr: CodePtr) -> impl Iterator { let mem_start = self.mem_block.borrow().start_ptr().raw_addr(self); let mem_end = self.mem_block.borrow().mapped_end_ptr().raw_addr(self); assert!(mem_start <= start_addr.raw_addr(self)); @@ -441,12 +441,12 @@ impl CodeBlock { // Ignore empty code ranges if start_addr == end_addr { - return vec![]; + return (0..0).into_iter(); } let start_page = (start_addr.raw_addr(self) - mem_start) / self.page_size; let end_page = (end_addr.raw_addr(self) - mem_start - 1) / self.page_size; - (start_page..=end_page).collect() // TODO: consider returning an iterator + (start_page..end_page + 1).into_iter() } /// Get a (possibly dangling) direct pointer to the current write position diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs index a105ab97916b97..d3aac992aa5c41 100644 --- a/yjit/src/codegen.rs +++ b/yjit/src/codegen.rs @@ -10563,7 +10563,7 @@ impl CodegenGlobals { let cfunc_exit_code = gen_full_cfunc_return(&mut ocb).unwrap(); let ocb_end_addr = ocb.unwrap().get_write_ptr(); - let ocb_pages = ocb.unwrap().addrs_to_pages(ocb_start_addr, ocb_end_addr); + let ocb_pages = ocb.unwrap().addrs_to_pages(ocb_start_addr, ocb_end_addr).collect(); // Mark all code memory as executable cb.mark_all_executable(); From 522614a3775a5eb3058aed9169b3504b4b9d9347 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Oct 2024 02:58:56 +0000 Subject: [PATCH 04/21] Bump github/codeql-action from 3.26.9 to 3.26.10 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3.26.9 to 3.26.10. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/461ef6c76dfe95d5c364de2f431ddbd31a417628...e2b3eafc8d227b0241d48be5f425d47c2d750a13) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 8 ++++---- .github/workflows/scorecards.yml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 604d38c162c3d0..f7f315d56f476a 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -77,15 +77,15 @@ jobs: run: sudo rm /usr/lib/ruby/vendor_ruby/rubygems/defaults/operating_system.rb - name: Initialize CodeQL - uses: github/codeql-action/init@461ef6c76dfe95d5c364de2f431ddbd31a417628 # v3.26.9 + uses: github/codeql-action/init@e2b3eafc8d227b0241d48be5f425d47c2d750a13 # v3.26.10 with: languages: ${{ matrix.language }} - name: Autobuild - uses: github/codeql-action/autobuild@461ef6c76dfe95d5c364de2f431ddbd31a417628 # v3.26.9 + uses: github/codeql-action/autobuild@e2b3eafc8d227b0241d48be5f425d47c2d750a13 # v3.26.10 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@461ef6c76dfe95d5c364de2f431ddbd31a417628 # v3.26.9 + uses: github/codeql-action/analyze@e2b3eafc8d227b0241d48be5f425d47c2d750a13 # v3.26.10 with: category: '/language:${{ matrix.language }}' upload: False @@ -115,7 +115,7 @@ jobs: continue-on-error: true - name: Upload SARIF - uses: github/codeql-action/upload-sarif@461ef6c76dfe95d5c364de2f431ddbd31a417628 # v3.26.9 + uses: github/codeql-action/upload-sarif@e2b3eafc8d227b0241d48be5f425d47c2d750a13 # v3.26.10 with: sarif_file: sarif-results/${{ matrix.language }}.sarif continue-on-error: true diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 60f6f50c8cbf40..6597fe3061ec2a 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -67,6 +67,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: 'Upload to code-scanning' - uses: github/codeql-action/upload-sarif@461ef6c76dfe95d5c364de2f431ddbd31a417628 # v3.26.9 + uses: github/codeql-action/upload-sarif@e2b3eafc8d227b0241d48be5f425d47c2d750a13 # v3.26.10 with: sarif_file: results.sarif From 4822220e6bab0991a9acc7fc7731d08b67fde8ad Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 1 Oct 2024 16:52:34 +0900 Subject: [PATCH 05/21] Pass MSPECOPT from compilers.yml --- .github/actions/compilers/action.yml | 7 +++++++ .github/actions/compilers/entrypoint.sh | 2 +- .github/workflows/compilers.yml | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/actions/compilers/action.yml b/.github/actions/compilers/action.yml index ecb8e4f485168b..aa23365e49e99c 100644 --- a/.github/actions/compilers/action.yml +++ b/.github/actions/compilers/action.yml @@ -60,6 +60,12 @@ inputs: description: >- Whether to run `make check` + mspecopt: + required: false + default: '' + description: >- + Additional options for mspec. + static_exts: required: false description: >- @@ -89,6 +95,7 @@ runs: --env INPUT_CPPFLAGS='${{ inputs.cppflags }}' --env INPUT_APPEND_CONFIGURE='${{ inputs.append_configure }}' --env INPUT_CHECK='${{ inputs.check }}' + --env INPUT_MSPECOPT='${{ inputs.mspecopt }}' --env INPUT_ENABLE_SHARED='${{ inputs.enable_shared }}' --env INPUT_STATIC_EXTS='${{ inputs.static_exts }}' 'ghcr.io/ruby/ruby-ci-image:${{ inputs.tag }}' diff --git a/.github/actions/compilers/entrypoint.sh b/.github/actions/compilers/entrypoint.sh index 0a11ef9cc11376..198ac0e1746da6 100755 --- a/.github/actions/compilers/entrypoint.sh +++ b/.github/actions/compilers/entrypoint.sh @@ -93,4 +93,4 @@ fi grouped make install grouped make test-tool grouped make test-all TESTS="-- $tests" -grouped env CHECK_LEAKS=true make test-spec +grouped env CHECK_LEAKS=true make test-spec MSPECOPT="$INPUT_MSPECOPT" diff --git a/.github/workflows/compilers.yml b/.github/workflows/compilers.yml index 692c15cc42f2be..a3b3322591210a 100644 --- a/.github/workflows/compilers.yml +++ b/.github/workflows/compilers.yml @@ -193,7 +193,7 @@ jobs: - { uses: './.github/actions/setup/directories', with: { srcdir: 'src', builddir: 'build', makeup: true } } - { uses: './.github/actions/compilers', name: '-O0', with: { optflags: '-O0 -march=x86-64 -mtune=generic' } } # - { uses: './.github/actions/compilers', name: '-O3', with: { optflags: '-O3 -march=x86-64 -mtune=generic', check: true } } - - { uses: './.github/actions/compilers', name: 'gmp', with: { append_configure: '--with-gmp', check: 'ruby/test_bignum.rb' } } + - { uses: './.github/actions/compilers', name: 'gmp', with: { append_configure: '--with-gmp', check: 'ruby/test_bignum.rb', mspecopt: "spec/ruby/core/integer" } } - { uses: './.github/actions/compilers', name: 'jemalloc', with: { append_configure: '--with-jemalloc' } } - { uses: './.github/actions/compilers', name: 'valgrind', with: { append_configure: '--with-valgrind' } } - { uses: './.github/actions/compilers', name: 'coroutine=ucontext', with: { append_configure: '--with-coroutine=ucontext' } } From 76111af632a0689cbe56787b0fb1ba4c8bbbe2e0 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 1 Oct 2024 17:08:27 +0900 Subject: [PATCH 06/21] expand relative path --- .github/workflows/compilers.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/compilers.yml b/.github/workflows/compilers.yml index a3b3322591210a..3fb3813d0cf792 100644 --- a/.github/workflows/compilers.yml +++ b/.github/workflows/compilers.yml @@ -193,7 +193,7 @@ jobs: - { uses: './.github/actions/setup/directories', with: { srcdir: 'src', builddir: 'build', makeup: true } } - { uses: './.github/actions/compilers', name: '-O0', with: { optflags: '-O0 -march=x86-64 -mtune=generic' } } # - { uses: './.github/actions/compilers', name: '-O3', with: { optflags: '-O3 -march=x86-64 -mtune=generic', check: true } } - - { uses: './.github/actions/compilers', name: 'gmp', with: { append_configure: '--with-gmp', check: 'ruby/test_bignum.rb', mspecopt: "spec/ruby/core/integer" } } + - { uses: './.github/actions/compilers', name: 'gmp', with: { append_configure: '--with-gmp', check: 'ruby/test_bignum.rb', mspecopt: "/github/workspace/src/spec/ruby/core/integer" } } - { uses: './.github/actions/compilers', name: 'jemalloc', with: { append_configure: '--with-jemalloc' } } - { uses: './.github/actions/compilers', name: 'valgrind', with: { append_configure: '--with-valgrind' } } - { uses: './.github/actions/compilers', name: 'coroutine=ucontext', with: { append_configure: '--with-coroutine=ucontext' } } From 136b30b414b3fcaddf0505d5a72fed8b08b2da3c Mon Sep 17 00:00:00 2001 From: rm155 Date: Fri, 30 Jul 2021 15:45:34 -0400 Subject: [PATCH 07/21] [ruby/singleton] Improve Ractor-compliance; Create PerRactorSingleton https://github.com/ruby/singleton/commit/ec4f66d9c1 --- lib/singleton.rb | 104 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 85 insertions(+), 19 deletions(-) diff --git a/lib/singleton.rb b/lib/singleton.rb index 6da939124ee064..577a7b8894df1e 100644 --- a/lib/singleton.rb +++ b/lib/singleton.rb @@ -93,21 +93,25 @@ # module Singleton VERSION = "0.2.0" + VERSION.freeze - # Raises a TypeError to prevent cloning. - def clone - raise TypeError, "can't clone instance of singleton #{self.class}" - end + module SingletonInstanceMethods + # Raises a TypeError to prevent cloning. + def clone + raise TypeError, "can't clone instance of singleton #{self.class}" + end - # Raises a TypeError to prevent duping. - def dup - raise TypeError, "can't dup instance of singleton #{self.class}" - end + # Raises a TypeError to prevent duping. + def dup + raise TypeError, "can't dup instance of singleton #{self.class}" + end - # By default, do not retain any state when marshalling. - def _dump(depth = -1) - '' + # By default, do not retain any state when marshalling. + def _dump(depth = -1) + '' + end end + include SingletonInstanceMethods module SingletonClassMethods # :nodoc: @@ -121,7 +125,7 @@ def _load(str) end def instance # :nodoc: - @singleton__instance__ || @singleton__mutex__.synchronize { @singleton__instance__ ||= new } + @singleton__instance__ || @singleton__mutex__.synchronize { @singleton__instance__ ||= set_instance(new) } end private @@ -130,22 +134,42 @@ def inherited(sub_klass) super Singleton.__init__(sub_klass) end + + def set_instance(val) + @singleton__instance__ = val + end + + def set_mutex(val) + @singleton__mutex__ = val + end end - class << Singleton # :nodoc: + def self.module_with_class_methods + SingletonClassMethods + end + + module SingletonClassProperties + + def self.included(c) + # extending an object with Singleton is a bad idea + c.undef_method :extend_object + end + + def self.extended(c) + # extending an object with Singleton is a bad idea + c.singleton_class.undef_method :extend_object + end + def __init__(klass) # :nodoc: klass.instance_eval { - @singleton__instance__ = nil - @singleton__mutex__ = Thread::Mutex.new + set_instance(nil) + set_mutex(Thread::Mutex.new) } klass end private - # extending an object with Singleton is a bad idea - undef_method :extend_object - def append_features(mod) # help out people counting on transitive mixins unless mod.instance_of?(Class) @@ -157,10 +181,11 @@ def append_features(mod) def included(klass) super klass.private_class_method :new, :allocate - klass.extend SingletonClassMethods + klass.extend module_with_class_methods Singleton.__init__(klass) end end + extend SingletonClassProperties ## # :singleton-method: _load @@ -170,3 +195,44 @@ def included(klass) # :singleton-method: instance # Returns the singleton instance. end + +module PerRactorSingleton + include Singleton::SingletonInstanceMethods + + module PerRactorSingletonClassMethods + include Singleton::SingletonClassMethods + def instance + set_mutex(Thread::Mutex.new) if Ractor.current[mutex_key].nil? + return Ractor.current[instance_key] if Ractor.current[instance_key] + Ractor.current[mutex_key].synchronize { + return Ractor.current[instance_key] if Ractor.current[instance_key] + set_instance(new()) + } + Ractor.current[instance_key] + end + + private + + def instance_key + :"__PerRactorSingleton_instance_with_class_id_#{object_id}__" + end + + def mutex_key + :"__PerRactorSingleton_mutex_with_class_id_#{object_id}__" + end + + def set_instance(val) + Ractor.current[instance_key] = val + end + + def set_mutex(val) + Ractor.current[mutex_key] = val + end + end + + def self.module_with_class_methods + PerRactorSingletonClassMethods + end + + extend Singleton::SingletonClassProperties +end From a6e96df573b8180568d7365a258dddc36a246fd1 Mon Sep 17 00:00:00 2001 From: rm155 Date: Sat, 31 Jul 2021 16:17:55 -0400 Subject: [PATCH 08/21] [ruby/singleton] Change PerRactorSingleton to RactorLocalSingleton https://github.com/ruby/singleton/commit/1216a86303 --- lib/singleton.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/singleton.rb b/lib/singleton.rb index 577a7b8894df1e..4a47190e2b579e 100644 --- a/lib/singleton.rb +++ b/lib/singleton.rb @@ -196,10 +196,10 @@ def included(klass) # Returns the singleton instance. end -module PerRactorSingleton +module RactorLocalSingleton include Singleton::SingletonInstanceMethods - module PerRactorSingletonClassMethods + module RactorLocalSingletonClassMethods include Singleton::SingletonClassMethods def instance set_mutex(Thread::Mutex.new) if Ractor.current[mutex_key].nil? @@ -214,11 +214,11 @@ def instance private def instance_key - :"__PerRactorSingleton_instance_with_class_id_#{object_id}__" + :"__RactorLocalSingleton_instance_with_class_id_#{object_id}__" end def mutex_key - :"__PerRactorSingleton_mutex_with_class_id_#{object_id}__" + :"__RactorLocalSingleton_mutex_with_class_id_#{object_id}__" end def set_instance(val) @@ -231,7 +231,7 @@ def set_mutex(val) end def self.module_with_class_methods - PerRactorSingletonClassMethods + RactorLocalSingletonClassMethods end extend Singleton::SingletonClassProperties From d0c1eef51174701889767e1fed584d2d56d9f9c9 Mon Sep 17 00:00:00 2001 From: rm155 Date: Wed, 18 Aug 2021 16:51:13 -0400 Subject: [PATCH 09/21] [ruby/singleton] Only use RactorLocalSingleton if Ractor is defined https://github.com/ruby/singleton/commit/f684d36a47 --- lib/singleton.rb | 60 +++++++++++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/lib/singleton.rb b/lib/singleton.rb index 4a47190e2b579e..924cbf1ae84879 100644 --- a/lib/singleton.rb +++ b/lib/singleton.rb @@ -196,43 +196,45 @@ def included(klass) # Returns the singleton instance. end -module RactorLocalSingleton - include Singleton::SingletonInstanceMethods - - module RactorLocalSingletonClassMethods - include Singleton::SingletonClassMethods - def instance - set_mutex(Thread::Mutex.new) if Ractor.current[mutex_key].nil? - return Ractor.current[instance_key] if Ractor.current[instance_key] - Ractor.current[mutex_key].synchronize { +if defined?(Ractor) + module RactorLocalSingleton + include Singleton::SingletonInstanceMethods + + module RactorLocalSingletonClassMethods + include Singleton::SingletonClassMethods + def instance + set_mutex(Thread::Mutex.new) if Ractor.current[mutex_key].nil? return Ractor.current[instance_key] if Ractor.current[instance_key] - set_instance(new()) - } - Ractor.current[instance_key] - end + Ractor.current[mutex_key].synchronize { + return Ractor.current[instance_key] if Ractor.current[instance_key] + set_instance(new()) + } + Ractor.current[instance_key] + end - private + private - def instance_key - :"__RactorLocalSingleton_instance_with_class_id_#{object_id}__" - end + def instance_key + :"__RactorLocalSingleton_instance_with_class_id_#{object_id}__" + end - def mutex_key - :"__RactorLocalSingleton_mutex_with_class_id_#{object_id}__" - end + def mutex_key + :"__RactorLocalSingleton_mutex_with_class_id_#{object_id}__" + end - def set_instance(val) - Ractor.current[instance_key] = val + def set_instance(val) + Ractor.current[instance_key] = val + end + + def set_mutex(val) + Ractor.current[mutex_key] = val + end end - def set_mutex(val) - Ractor.current[mutex_key] = val + def self.module_with_class_methods + RactorLocalSingletonClassMethods end - end - def self.module_with_class_methods - RactorLocalSingletonClassMethods + extend Singleton::SingletonClassProperties end - - extend Singleton::SingletonClassProperties end From 2f53c9fc62e48d31101ab3072085a1ace336c938 Mon Sep 17 00:00:00 2001 From: rm155 Date: Thu, 16 Sep 2021 22:42:45 -0400 Subject: [PATCH 10/21] [ruby/singleton] Clean VERSION freezing https://github.com/ruby/singleton/commit/f31334a736 --- lib/singleton.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/singleton.rb b/lib/singleton.rb index 924cbf1ae84879..0120f059576f1f 100644 --- a/lib/singleton.rb +++ b/lib/singleton.rb @@ -1,4 +1,4 @@ -# frozen_string_literal: false +# frozen_string_literal: true # The Singleton module implements the Singleton pattern. # @@ -93,7 +93,6 @@ # module Singleton VERSION = "0.2.0" - VERSION.freeze module SingletonInstanceMethods # Raises a TypeError to prevent cloning. From 0f00544984f0b2b7fefc7d9039b206f9415fd371 Mon Sep 17 00:00:00 2001 From: rm155 Date: Mon, 8 Nov 2021 13:59:38 -0500 Subject: [PATCH 11/21] [ruby/singleton] Make compatible with Ruby 2.4 https://github.com/ruby/singleton/commit/0a77bb492d --- lib/singleton.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/singleton.rb b/lib/singleton.rb index 0120f059576f1f..afe669e06d7e50 100644 --- a/lib/singleton.rb +++ b/lib/singleton.rb @@ -156,7 +156,7 @@ def self.included(c) def self.extended(c) # extending an object with Singleton is a bad idea - c.singleton_class.undef_method :extend_object + c.singleton_class.send(:undef_method, :extend_object) end def __init__(klass) # :nodoc: From d109c7c620a9f426b9aa60f9ff3a227fd9794933 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 1 Oct 2024 18:32:40 +0900 Subject: [PATCH 12/21] Temporary disabled RactorLocalSingleton from constant leak check. --- spec/ruby/default.mspec | 1 + 1 file changed, 1 insertion(+) diff --git a/spec/ruby/default.mspec b/spec/ruby/default.mspec index 1e8f8893aa1f9c..2ebd455fd0d987 100644 --- a/spec/ruby/default.mspec +++ b/spec/ruby/default.mspec @@ -47,5 +47,6 @@ class MSpecScript /\wSpecs?$/, /^CS_CONST/, /^CSL_CONST/, + /^RactorLocalSingleton/, ] end From 3ebc85e240c64849e8645ae9b6242abbd1a2b63a Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 1 Oct 2024 18:41:38 +0900 Subject: [PATCH 13/21] Define RactorLocalSingleton on .mspec.constants --- spec/ruby/.mspec.constants | 1 + spec/ruby/default.mspec | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/ruby/.mspec.constants b/spec/ruby/.mspec.constants index 6e09a44362df22..4da36337152b0e 100644 --- a/spec/ruby/.mspec.constants +++ b/spec/ruby/.mspec.constants @@ -146,6 +146,7 @@ Prime Private ProcFromMethod Psych +RactorLocalSingleton REXML RUBY_SIGNALS RbReadline diff --git a/spec/ruby/default.mspec b/spec/ruby/default.mspec index 2ebd455fd0d987..1e8f8893aa1f9c 100644 --- a/spec/ruby/default.mspec +++ b/spec/ruby/default.mspec @@ -47,6 +47,5 @@ class MSpecScript /\wSpecs?$/, /^CS_CONST/, /^CSL_CONST/, - /^RactorLocalSingleton/, ] end From 35e124832e29b65c84d4e0e4e434616859f9bdf5 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Mon, 30 Sep 2024 22:49:09 +0900 Subject: [PATCH 14/21] [Bug #20755] Frozen string should not be writable via IO::Buffer --- io_buffer.c | 3 ++- test/ruby/test_io_buffer.rb | 25 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/io_buffer.c b/io_buffer.c index 2e789abe0faf7d..cc76f12ecc070a 100644 --- a/io_buffer.c +++ b/io_buffer.c @@ -843,7 +843,8 @@ rb_io_buffer_get_bytes(VALUE self, void **base, size_t *size) static inline void io_buffer_get_bytes_for_writing(struct rb_io_buffer *buffer, void **base, size_t *size) { - if (buffer->flags & RB_IO_BUFFER_READONLY) { + if (buffer->flags & RB_IO_BUFFER_READONLY || + (!NIL_P(buffer->source) && OBJ_FROZEN(buffer->source))) { rb_raise(rb_eIOBufferAccessError, "Buffer is not writable!"); } diff --git a/test/ruby/test_io_buffer.rb b/test/ruby/test_io_buffer.rb index 7087a2b957d734..98cf45d0c3da07 100644 --- a/test/ruby/test_io_buffer.rb +++ b/test/ruby/test_io_buffer.rb @@ -248,6 +248,31 @@ def test_slice_readonly assert_equal "Hello World", hello end + def test_transfer + hello = %w"Hello World".join(" ") + buffer = IO::Buffer.for(hello) + transferred = buffer.transfer + assert_equal "Hello World", transferred.get_string + assert_predicate buffer, :null? + assert_raise IO::Buffer::AccessError do + transferred.set_string("Goodbye") + end + assert_equal "Hello World", hello + end + + def test_transfer_in_block + hello = %w"Hello World".join(" ") + buffer = IO::Buffer.for(hello, &:transfer) + assert_equal "Hello World", buffer.get_string + buffer.set_string("Ciao!") + assert_equal "Ciao! World", hello + hello.freeze + assert_raise IO::Buffer::AccessError do + buffer.set_string("Hola") + end + assert_equal "Ciao! World", hello + end + def test_locked buffer = IO::Buffer.new(128, IO::Buffer::INTERNAL|IO::Buffer::LOCKED) From 86ae409467e3674c142f012ee3b2288b0305d87c Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Tue, 1 Oct 2024 01:03:40 +0900 Subject: [PATCH 15/21] [Bug #20764] Refactor argument forwarding in lambda Reject argument forwarding in lambda: - without parentheses - after optional argument(s) --- parse.y | 24 ++++++++++++++---------- test/ruby/test_syntax.rb | 4 ++++ 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/parse.y b/parse.y index 1104697b19ab95..63471895c8ed0b 100644 --- a/parse.y +++ b/parse.y @@ -5202,7 +5202,6 @@ lambda : tLAMBDA[lpar] { token_info_push(p, "->", &@1); $$ = dyna_push(p); - p->lex.lpar_beg = p->lex.paren_nest; }[dyna] max_numparam numparam it_id allow_exits f_larglist[args] @@ -5235,7 +5234,7 @@ lambda : tLAMBDA[lpar] f_larglist : '(' f_args opt_bv_decl ')' { p->ctxt.in_argdef = 0; - $$ = $2; + $$ = $f_args; p->max_numparam = ORDINAL_PARAM; /*% ripper: paren!($:2) %*/ } @@ -5244,7 +5243,7 @@ f_larglist : '(' f_args opt_bv_decl ')' p->ctxt.in_argdef = 0; if (!args_info_empty_p(&$1->nd_ainfo)) p->max_numparam = ORDINAL_PARAM; - $$ = $1; + $$ = $f_args; } ; @@ -6430,8 +6429,16 @@ args_tail : f_kwarg(f_kw) ',' f_kwrest opt_f_block_arg } | args_forward { - add_forwarding_args(p); - $$ = new_args_tail(p, 0, $1, arg_FWD_BLOCK, &@1); + ID fwd = $args_forward; + if (lambda_beginning_p() || + (p->lex.lpar_beg >= 0 && p->lex.lpar_beg+1 == p->lex.paren_nest)) { + yyerror0("unexpected ... in lambda argument"); + fwd = 0; + } + else { + add_forwarding_args(p); + } + $$ = new_args_tail(p, 0, fwd, arg_FWD_BLOCK, &@1); $$->nd_ainfo.forwarding = 1; /*% ripper: [Qnil, $:1, Qnil] %*/ } @@ -11049,6 +11056,7 @@ parser_yylex(struct parser_params *p) if (c == '>') { SET_LEX_STATE(EXPR_ENDFN); yylval.num = p->lex.lpar_beg; + p->lex.lpar_beg = p->lex.paren_nest; return tLAMBDA; } if (IS_BEG() || (IS_SPCARG(c) && arg_ambiguous(p, '-'))) { @@ -11068,17 +11076,13 @@ parser_yylex(struct parser_params *p) SET_LEX_STATE(EXPR_BEG); if ((c = nextc(p)) == '.') { if ((c = nextc(p)) == '.') { - if (p->ctxt.in_argdef) { + if (p->ctxt.in_argdef || IS_LABEL_POSSIBLE() || lambda_beginning_p()) { SET_LEX_STATE(EXPR_ENDARG); return tBDOT3; } if (p->lex.paren_nest == 0 && looking_at_eol_p(p)) { rb_warn0("... at EOL, should be parenthesized?"); } - else if (p->lex.lpar_beg >= 0 && p->lex.lpar_beg+1 == p->lex.paren_nest) { - if (IS_lex_state_for(last_state, EXPR_LABEL)) - return tDOT3; - } return is_beg ? tBDOT3 : tDOT3; } pushback(p, c); diff --git a/test/ruby/test_syntax.rb b/test/ruby/test_syntax.rb index 08b4a7ee1c3e4d..16bb914e3f452b 100644 --- a/test/ruby/test_syntax.rb +++ b/test/ruby/test_syntax.rb @@ -1980,6 +1980,10 @@ def test_argument_forwarding assert_syntax_error('iter {|...|}', /unexpected/) assert_syntax_error('->... {}', unexpected) assert_syntax_error('->(...) {}', unexpected) + assert_syntax_error('->a,... {}', unexpected) + assert_syntax_error('->(a,...) {}', unexpected) + assert_syntax_error('->a=1,... {}', unexpected) + assert_syntax_error('->(a=1,...) {}', unexpected) assert_syntax_error('def foo(x, y, z) bar(...); end', /unexpected/) assert_syntax_error('def foo(x, y, z) super(...); end', /unexpected/) assert_syntax_error('def foo(...) yield(...); end', /unexpected/) From 3932d8a87ad1a9468bb8954dfe117f9bfb54505d Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 30 Sep 2024 10:14:51 -0400 Subject: [PATCH 16/21] Replace heap_eden_total_slots with objspace_available_slots --- gc/default.c | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/gc/default.c b/gc/default.c index 09acad161871b1..01021b72776ee4 100644 --- a/gc/default.c +++ b/gc/default.c @@ -939,16 +939,6 @@ heap_eden_total_pages(rb_objspace_t *objspace) return count; } -static inline size_t -heap_eden_total_slots(rb_objspace_t *objspace) -{ - size_t count = 0; - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - count += SIZE_POOL_EDEN_HEAP(&size_pools[i])->total_slots; - } - return count; -} - static inline size_t total_allocated_objects(rb_objspace_t *objspace) { @@ -5461,7 +5451,7 @@ gc_marks_finish(rb_objspace_t *objspace) { const unsigned long r_mul = objspace->live_ractor_cache_count > 8 ? 8 : objspace->live_ractor_cache_count; // upto 8 - size_t total_slots = heap_eden_total_slots(objspace); + size_t total_slots = objspace_available_slots(objspace); size_t sweep_slots = total_slots - objspace->marked_slots; /* will be swept slots */ size_t max_free_slots = (size_t)(total_slots * gc_params.heap_free_slots_max_ratio); size_t min_free_slots = (size_t)(total_slots * gc_params.heap_free_slots_min_ratio); @@ -5471,7 +5461,7 @@ gc_marks_finish(rb_objspace_t *objspace) int full_marking = is_full_marking(objspace); - GC_ASSERT(heap_eden_total_slots(objspace) >= objspace->marked_slots); + GC_ASSERT(objspace_available_slots(objspace) >= objspace->marked_slots); /* Setup freeable slots. */ size_t total_init_slots = 0; @@ -5525,7 +5515,7 @@ gc_marks_finish(rb_objspace_t *objspace) gc_report(1, objspace, "gc_marks_finish (marks %"PRIdSIZE" objects, " "old %"PRIdSIZE" objects, total %"PRIdSIZE" slots, " "sweep %"PRIdSIZE" slots, allocatable %"PRIdSIZE" slots, next GC: %s)\n", - objspace->marked_slots, objspace->rgengc.old_objects, heap_eden_total_slots(objspace), sweep_slots, objspace->heap_pages.allocatable_slots, + objspace->marked_slots, objspace->rgengc.old_objects, objspace_available_slots(objspace), sweep_slots, objspace->heap_pages.allocatable_slots, gc_needs_major_flags ? "major" : "minor"); } @@ -6861,7 +6851,7 @@ rb_gc_impl_prepare_heap(void *objspace_ptr) { rb_objspace_t *objspace = objspace_ptr; - size_t orig_total_slots = heap_eden_total_slots(objspace); + size_t orig_total_slots = objspace_available_slots(objspace); size_t orig_allocatable_slots = objspace->heap_pages.allocatable_slots; rb_gc_impl_each_objects(objspace, gc_set_candidate_object_i, objspace_ptr); @@ -6877,7 +6867,7 @@ rb_gc_impl_prepare_heap(void *objspace_ptr) GC_ASSERT(objspace->empty_pages_count == 0); objspace->heap_pages.allocatable_slots = orig_allocatable_slots; - size_t total_slots = heap_eden_total_slots(objspace); + size_t total_slots = objspace_available_slots(objspace); if (orig_total_slots > total_slots) { objspace->heap_pages.allocatable_slots += orig_total_slots - total_slots; } From 75568d23e90148490208e3ac5b719406b3e9f75f Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 1 Oct 2024 08:42:07 -0400 Subject: [PATCH 17/21] [ruby/prism] Disallow &. after endless range https://github.com/ruby/prism/commit/498dd922d4 --- prism/prism.c | 2 +- test/prism/errors/amperand_dot_after_endless_range.txt | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 test/prism/errors/amperand_dot_after_endless_range.txt diff --git a/prism/prism.c b/prism/prism.c index 3c6863ab1bfa82..4bbcf0ea9ddaf4 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -21657,7 +21657,7 @@ parse_expression(pm_parser_t *parser, pm_binding_power_t binding_power, bool acc // 1.. * 2 // if (PM_NODE_TYPE_P(node, PM_RANGE_NODE) && ((pm_range_node_t *) node)->right == NULL) { - if (match3(parser, PM_TOKEN_UAMPERSAND, PM_TOKEN_USTAR, PM_TOKEN_DOT)) { + if (match4(parser, PM_TOKEN_UAMPERSAND, PM_TOKEN_USTAR, PM_TOKEN_DOT, PM_TOKEN_AMPERSAND_DOT)) { PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_NON_ASSOCIATIVE_OPERATOR, pm_token_type_human(parser->current.type), pm_token_type_human(parser->previous.type)); break; } diff --git a/test/prism/errors/amperand_dot_after_endless_range.txt b/test/prism/errors/amperand_dot_after_endless_range.txt new file mode 100644 index 00000000000000..ab8c8ccc4d4d10 --- /dev/null +++ b/test/prism/errors/amperand_dot_after_endless_range.txt @@ -0,0 +1,3 @@ +0 if true...&.abs + ^~ unexpected '&.'; ... is a non-associative operator + From d86b4041b5eca43eea581e4af874e3fd46faaaa2 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 1 Oct 2024 09:45:09 -0400 Subject: [PATCH 18/21] [ruby/prism] Disallow empty labels when not allowed https://github.com/ruby/prism/commit/399ee28dc1 --- prism/prism.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/prism/prism.c b/prism/prism.c index 4bbcf0ea9ddaf4..a99fb712e35297 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -16545,6 +16545,8 @@ parse_strings(pm_parser_t *parser, pm_node_t *current, bool accepts_label, uint1 pm_string_shared_init(&symbol->unescaped, content.start, content.end); node = (pm_node_t *) symbol; + + if (!label_allowed) pm_parser_err_node(parser, node, PM_ERR_UNEXPECTED_LABEL); } else if (!lex_interpolation) { // If we don't accept interpolation then we expect the string to // start with a single string content node. From 169a5ee99e1e8742e6821f8b846d39e90c3c9ea8 Mon Sep 17 00:00:00 2001 From: ydah Date: Tue, 1 Oct 2024 21:25:09 +0900 Subject: [PATCH 19/21] Use user defined inline rules `user_or_keyword_variable` --- parse.y | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/parse.y b/parse.y index 63471895c8ed0b..e3391d64eed975 100644 --- a/parse.y +++ b/parse.y @@ -2925,6 +2925,11 @@ rb_parser_ary_free(rb_parser_t *p, rb_parser_ary_t *ary) | tCONSTANT ; +%rule %inline user_or_keyword_variable + : user_variable + | keyword_variable + ; + /* * parameterizing rules */ @@ -3657,12 +3662,7 @@ mlhs_post : mlhs_item } ; -mlhs_node : user_variable - { - /*% ripper: var_field!($:1) %*/ - $$ = assignable(p, $1, 0, &@$); - } - | keyword_variable +mlhs_node : user_or_keyword_variable { /*% ripper: var_field!($:1) %*/ $$ = assignable(p, $1, 0, &@$); @@ -3707,12 +3707,7 @@ mlhs_node : user_variable } ; -lhs : user_variable - { - /*% ripper: var_field!($:1) %*/ - $$ = assignable(p, $1, 0, &@$); - } - | keyword_variable +lhs : user_or_keyword_variable { /*% ripper: var_field!($:1) %*/ $$ = assignable(p, $1, 0, &@$); @@ -6336,12 +6331,7 @@ var_ref : user_variable } ; -var_lhs : user_variable - { - /*% ripper: var_field!($:1) %*/ - $$ = assignable(p, $1, 0, &@$); - } - | keyword_variable +var_lhs : user_or_keyword_variable { /*% ripper: var_field!($:1) %*/ $$ = assignable(p, $1, 0, &@$); From ec230ac6432ea89f1ee53d82a62337d4883dc83a Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 1 Oct 2024 10:22:07 -0400 Subject: [PATCH 20/21] Resync to latest Prism --- prism/config.yml | 1 - prism/prism.c | 42 +++++++++++++++++-- prism/templates/src/diagnostic.c.erb | 1 - test/prism/errors/command_call_in.txt | 6 +-- test/prism/errors/infix_after_label.txt | 6 +++ .../errors/trailing_comma_after_block.txt | 3 ++ 6 files changed, 49 insertions(+), 10 deletions(-) create mode 100644 test/prism/errors/infix_after_label.txt create mode 100644 test/prism/errors/trailing_comma_after_block.txt diff --git a/prism/config.yml b/prism/config.yml index b5de4d2b4afced..972f1e8e69a52f 100644 --- a/prism/config.yml +++ b/prism/config.yml @@ -15,7 +15,6 @@ errors: - ARGUMENT_FORMAL_GLOBAL - ARGUMENT_FORMAL_IVAR - ARGUMENT_FORWARDING_UNBOUND - - ARGUMENT_IN - ARGUMENT_NO_FORWARDING_AMPERSAND - ARGUMENT_NO_FORWARDING_ELLIPSES - ARGUMENT_NO_FORWARDING_STAR diff --git a/prism/prism.c b/prism/prism.c index a99fb712e35297..2e98702d9b5cd4 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -14271,9 +14271,6 @@ parse_arguments(pm_parser_t *parser, pm_arguments_t *arguments, bool accepts_for pm_static_literals_free(&hash_keys); parsed_bare_hash = true; - } else if (accept1(parser, PM_TOKEN_KEYWORD_IN)) { - // TODO: Could we solve this with binding powers instead? - pm_parser_err_current(parser, PM_ERR_ARGUMENT_IN); } parse_arguments_append(parser, arguments, argument); @@ -14786,7 +14783,7 @@ parse_parameters( } default: if (parser->previous.type == PM_TOKEN_COMMA) { - if (allows_trailing_comma) { + if (allows_trailing_comma && order >= PM_PARAMETERS_ORDER_NAMED) { // If we get here, then we have a trailing comma in a // block parameter list. pm_node_t *param = (pm_node_t *) pm_implicit_rest_node_create(parser, &parser->previous); @@ -21575,6 +21572,19 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t #undef PM_PARSE_PATTERN_TOP #undef PM_PARSE_PATTERN_MULTI +/** + * Determine if a given call node looks like a "command", which means it has + * arguments but does not have parentheses. + */ +static inline bool +pm_call_node_command_p(const pm_call_node_t *node) { + return ( + (node->opening_loc.start == NULL) && + (node->block == NULL || PM_NODE_TYPE_P(node->block, PM_BLOCK_ARGUMENT_NODE)) && + (node->arguments != NULL || node->block != NULL) + ); +} + /** * Parse an expression at the given point of the parser using the given binding * power to parse subsequent chains. If this function finds a syntax error, it @@ -21609,6 +21619,23 @@ parse_expression(pm_parser_t *parser, pm_binding_power_t binding_power, bool acc return node; } break; + case PM_CALL_NODE: + // If we have a call node, then we need to check if it looks like a + // method call without parentheses that contains arguments. If it + // does, then it has different rules for parsing infix operators, + // namely that it only accepts composition (and/or) and modifiers + // (if/unless/etc.). + if ((pm_binding_powers[parser->current.type].left > PM_BINDING_POWER_COMPOSITION) && pm_call_node_command_p((pm_call_node_t *) node)) { + return node; + } + break; + case PM_SYMBOL_NODE: + // If we have a symbol node that is being parsed as a label, then we + // need to immediately return, because there should never be an + // infix operator following this node. + if (pm_symbol_node_label_p(node)) { + return node; + } default: break; } @@ -21624,6 +21651,13 @@ parse_expression(pm_parser_t *parser, pm_binding_power_t binding_power, bool acc node = parse_expression_infix(parser, node, binding_power, current_binding_powers.right, accepts_command_call, (uint16_t) (depth + 1)); switch (PM_NODE_TYPE(node)) { + case PM_MULTI_WRITE_NODE: + // Multi-write nodes are statements, and cannot be followed by + // operators except modifiers. + if (pm_binding_powers[parser->current.type].left > PM_BINDING_POWER_MODIFIER) { + return node; + } + break; case PM_CLASS_VARIABLE_WRITE_NODE: case PM_CONSTANT_PATH_WRITE_NODE: case PM_CONSTANT_WRITE_NODE: diff --git a/prism/templates/src/diagnostic.c.erb b/prism/templates/src/diagnostic.c.erb index b2381db57fab2c..d2b7b4f691ec9a 100644 --- a/prism/templates/src/diagnostic.c.erb +++ b/prism/templates/src/diagnostic.c.erb @@ -100,7 +100,6 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_ARGUMENT_FORMAL_GLOBAL] = { "invalid formal argument; formal argument cannot be a global variable", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_FORMAL_IVAR] = { "invalid formal argument; formal argument cannot be an instance variable", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_FORWARDING_UNBOUND] = { "unexpected `...` in an non-parenthesized call", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_ARGUMENT_IN] = { "unexpected `in` keyword in arguments", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_NO_FORWARDING_AMPERSAND] = { "unexpected `&`; no anonymous block parameter", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES] = { "unexpected ... when the parent method is not forwarding", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_NO_FORWARDING_STAR] = { "unexpected `*`; no anonymous rest parameter", PM_ERROR_LEVEL_SYNTAX }, diff --git a/test/prism/errors/command_call_in.txt b/test/prism/errors/command_call_in.txt index a4357028c67768..2fdcf0973897b7 100644 --- a/test/prism/errors/command_call_in.txt +++ b/test/prism/errors/command_call_in.txt @@ -1,7 +1,5 @@ foo 1 in a - ^ unexpected `in` keyword in arguments - ^ unexpected local variable or method, expecting end-of-input + ^~ unexpected 'in', expecting end-of-input + ^~ unexpected 'in', ignoring it a = foo 2 in b - ^ unexpected `in` keyword in arguments - ^ unexpected local variable or method, expecting end-of-input diff --git a/test/prism/errors/infix_after_label.txt b/test/prism/errors/infix_after_label.txt new file mode 100644 index 00000000000000..c3bcfaecebd928 --- /dev/null +++ b/test/prism/errors/infix_after_label.txt @@ -0,0 +1,6 @@ +{ 'a':.upcase => 1 } + ^ unexpected '.'; expected a value in the hash literal + ^ expected a `}` to close the hash literal + ^ unexpected '}', expecting end-of-input + ^ unexpected '}', ignoring it + diff --git a/test/prism/errors/trailing_comma_after_block.txt b/test/prism/errors/trailing_comma_after_block.txt new file mode 100644 index 00000000000000..d25db0efbeca3b --- /dev/null +++ b/test/prism/errors/trailing_comma_after_block.txt @@ -0,0 +1,3 @@ +p{|&,|} + ^ unexpected `,` in parameters + From e320da60976f6818c8667afb98fe88142c3073d2 Mon Sep 17 00:00:00 2001 From: tomoya ishida Date: Wed, 2 Oct 2024 02:01:31 +0900 Subject: [PATCH 21/21] [ruby/reline] Fix Reline crash with invalid encoding history (https://github.com/ruby/reline/pull/751) https://github.com/ruby/reline/commit/e9d4b37e34 --- lib/reline/history.rb | 6 +++--- lib/reline/line_editor.rb | 2 +- lib/reline/unicode.rb | 16 ++++++++++++++++ test/reline/test_history.rb | 9 +++++++++ test/reline/test_unicode.rb | 28 ++++++++++++++++++++++++++++ 5 files changed, 57 insertions(+), 4 deletions(-) diff --git a/lib/reline/history.rb b/lib/reline/history.rb index 3f3b65fea64aeb..47c68ba7745e10 100644 --- a/lib/reline/history.rb +++ b/lib/reline/history.rb @@ -19,7 +19,7 @@ def [](index) def []=(index, val) index = check_index(index) - super(index, String.new(val, encoding: Reline.encoding_system_needs)) + super(index, Reline::Unicode.safe_encode(val, Reline.encoding_system_needs)) end def concat(*val) @@ -45,7 +45,7 @@ def push(*val) end end super(*(val.map{ |v| - String.new(v, encoding: Reline.encoding_system_needs) + Reline::Unicode.safe_encode(v, Reline.encoding_system_needs) })) end @@ -56,7 +56,7 @@ def <<(val) if @config.history_size.positive? shift if size + 1 > @config.history_size end - super(String.new(val, encoding: Reline.encoding_system_needs)) + super(Reline::Unicode.safe_encode(val, Reline.encoding_system_needs)) end private def check_index(index) diff --git a/lib/reline/line_editor.rb b/lib/reline/line_editor.rb index c71a5f79eef9c5..56dc235c030362 100644 --- a/lib/reline/line_editor.rb +++ b/lib/reline/line_editor.rb @@ -1325,7 +1325,7 @@ def insert_multiline_text(text) save_old_buffer pre = @buffer_of_lines[@line_index].byteslice(0, @byte_pointer) post = @buffer_of_lines[@line_index].byteslice(@byte_pointer..) - lines = (pre + text.gsub(/\r\n?/, "\n") + post).split("\n", -1) + lines = (pre + Reline::Unicode.safe_encode(text, @encoding).gsub(/\r\n?/, "\n") + post).split("\n", -1) lines << '' if lines.empty? @buffer_of_lines[@line_index, 1] = lines @line_index += lines.size - 1 diff --git a/lib/reline/unicode.rb b/lib/reline/unicode.rb index ef239d5e9ee947..0ec815aeea6f3d 100644 --- a/lib/reline/unicode.rb +++ b/lib/reline/unicode.rb @@ -54,6 +54,22 @@ def self.escape_for_print(str) }.join end + def self.safe_encode(str, encoding) + # Reline only supports utf-8 convertible string. + converted = str.encode(encoding, invalid: :replace, undef: :replace) + return converted if str.encoding == Encoding::UTF_8 || converted.encoding == Encoding::UTF_8 || converted.ascii_only? + + # This code is essentially doing the same thing as + # `str.encode(utf8, **replace_options).encode(encoding, **replace_options)` + # but also avoids unneccesary irreversible encoding conversion. + converted.gsub(/\X/) do |c| + c.encode(Encoding::UTF_8) + c + rescue Encoding::UndefinedConversionError + '?' + end + end + require 'reline/unicode/east_asian_width' def self.get_mbchar_width(mbchar) diff --git a/test/reline/test_history.rb b/test/reline/test_history.rb index ddf8fb14726b3b..ea902b065394ce 100644 --- a/test/reline/test_history.rb +++ b/test/reline/test_history.rb @@ -266,6 +266,15 @@ def test_history_size_negative_unlimited assert_equal 5, history.size end + def test_history_encoding_conversion + history = history_new + text1 = String.new("a\u{65535}b\xFFc", encoding: Encoding::UTF_8) + text2 = String.new("d\xFFe", encoding: Encoding::Shift_JIS) + history.push(text1.dup, text2.dup) + expected = [text1, text2].map { |s| s.encode(Reline.encoding_system_needs, invalid: :replace, undef: :replace) } + assert_equal(expected, history.to_a) + end + private def history_new(history_size: 10) diff --git a/test/reline/test_unicode.rb b/test/reline/test_unicode.rb index deba4d4681173c..688d25e2387780 100644 --- a/test/reline/test_unicode.rb +++ b/test/reline/test_unicode.rb @@ -89,4 +89,32 @@ def test_take_mbchar_range assert_equal ["\e[31mc\1ABC\2d\e[0mef", 2, 4], Reline::Unicode.take_mbchar_range("\e[31mabc\1ABC\2d\e[0mefghi", 2, 4) assert_equal ["\e[41m \e[42mい\e[43m ", 1, 4], Reline::Unicode.take_mbchar_range("\e[41mあ\e[42mい\e[43mう", 1, 4, padding: true) end + + def test_encoding_conversion + texts = [ + String.new("invalid\xFFutf8", encoding: 'utf-8'), + String.new("invalid\xFFsjis", encoding: 'sjis'), + "utf8#{33111.chr('sjis')}convertible", + "utf8#{33222.chr('sjis')}inconvertible", + "sjis->utf8->sjis#{60777.chr('sjis')}irreversible" + ] + utf8_texts = [ + 'invalid�utf8', + 'invalid�sjis', + 'utf8仝convertible', + 'utf8�inconvertible', + 'sjis->utf8->sjis劦irreversible' + ] + sjis_texts = [ + 'invalid?utf8', + 'invalid?sjis', + "utf8#{33111.chr('sjis')}convertible", + 'utf8?inconvertible', + "sjis->utf8->sjis#{60777.chr('sjis')}irreversible" + ] + assert_equal(utf8_texts, texts.map { |s| Reline::Unicode.safe_encode(s, 'utf-8') }) + assert_equal(utf8_texts, texts.map { |s| Reline::Unicode.safe_encode(s, Encoding::UTF_8) }) + assert_equal(sjis_texts, texts.map { |s| Reline::Unicode.safe_encode(s, 'sjis') }) + assert_equal(sjis_texts, texts.map { |s| Reline::Unicode.safe_encode(s, Encoding::Windows_31J) }) + end end