diff --git a/.github/actions/compilers/action.yml b/.github/actions/compilers/action.yml index ecb8e4f485168b..aa23365e49e99c 100644 --- a/.github/actions/compilers/action.yml +++ b/.github/actions/compilers/action.yml @@ -60,6 +60,12 @@ inputs: description: >- Whether to run `make check` + mspecopt: + required: false + default: '' + description: >- + Additional options for mspec. + static_exts: required: false description: >- @@ -89,6 +95,7 @@ runs: --env INPUT_CPPFLAGS='${{ inputs.cppflags }}' --env INPUT_APPEND_CONFIGURE='${{ inputs.append_configure }}' --env INPUT_CHECK='${{ inputs.check }}' + --env INPUT_MSPECOPT='${{ inputs.mspecopt }}' --env INPUT_ENABLE_SHARED='${{ inputs.enable_shared }}' --env INPUT_STATIC_EXTS='${{ inputs.static_exts }}' 'ghcr.io/ruby/ruby-ci-image:${{ inputs.tag }}' diff --git a/.github/actions/compilers/entrypoint.sh b/.github/actions/compilers/entrypoint.sh index 0a11ef9cc11376..198ac0e1746da6 100755 --- a/.github/actions/compilers/entrypoint.sh +++ b/.github/actions/compilers/entrypoint.sh @@ -93,4 +93,4 @@ fi grouped make install grouped make test-tool grouped make test-all TESTS="-- $tests" -grouped env CHECK_LEAKS=true make test-spec +grouped env CHECK_LEAKS=true make test-spec MSPECOPT="$INPUT_MSPECOPT" diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 604d38c162c3d0..f7f315d56f476a 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -77,15 +77,15 @@ jobs: run: sudo rm /usr/lib/ruby/vendor_ruby/rubygems/defaults/operating_system.rb - name: Initialize CodeQL - uses: github/codeql-action/init@461ef6c76dfe95d5c364de2f431ddbd31a417628 # v3.26.9 + uses: github/codeql-action/init@e2b3eafc8d227b0241d48be5f425d47c2d750a13 # v3.26.10 with: languages: ${{ matrix.language }} - name: Autobuild - uses: github/codeql-action/autobuild@461ef6c76dfe95d5c364de2f431ddbd31a417628 # v3.26.9 + uses: github/codeql-action/autobuild@e2b3eafc8d227b0241d48be5f425d47c2d750a13 # v3.26.10 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@461ef6c76dfe95d5c364de2f431ddbd31a417628 # v3.26.9 + uses: github/codeql-action/analyze@e2b3eafc8d227b0241d48be5f425d47c2d750a13 # v3.26.10 with: category: '/language:${{ matrix.language }}' upload: False @@ -115,7 +115,7 @@ jobs: continue-on-error: true - name: Upload SARIF - uses: github/codeql-action/upload-sarif@461ef6c76dfe95d5c364de2f431ddbd31a417628 # v3.26.9 + uses: github/codeql-action/upload-sarif@e2b3eafc8d227b0241d48be5f425d47c2d750a13 # v3.26.10 with: sarif_file: sarif-results/${{ matrix.language }}.sarif continue-on-error: true diff --git a/.github/workflows/compilers.yml b/.github/workflows/compilers.yml index 692c15cc42f2be..3fb3813d0cf792 100644 --- a/.github/workflows/compilers.yml +++ b/.github/workflows/compilers.yml @@ -193,7 +193,7 @@ jobs: - { uses: './.github/actions/setup/directories', with: { srcdir: 'src', builddir: 'build', makeup: true } } - { uses: './.github/actions/compilers', name: '-O0', with: { optflags: '-O0 -march=x86-64 -mtune=generic' } } # - { uses: './.github/actions/compilers', name: '-O3', with: { optflags: '-O3 -march=x86-64 -mtune=generic', check: true } } - - { uses: './.github/actions/compilers', name: 'gmp', with: { append_configure: '--with-gmp', check: 'ruby/test_bignum.rb' } } + - { uses: './.github/actions/compilers', name: 'gmp', with: { append_configure: '--with-gmp', check: 'ruby/test_bignum.rb', mspecopt: "/github/workspace/src/spec/ruby/core/integer" } } - { uses: './.github/actions/compilers', name: 'jemalloc', with: { append_configure: '--with-jemalloc' } } - { uses: './.github/actions/compilers', name: 'valgrind', with: { append_configure: '--with-valgrind' } } - { uses: './.github/actions/compilers', name: 'coroutine=ucontext', with: { append_configure: '--with-coroutine=ucontext' } } diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 60f6f50c8cbf40..6597fe3061ec2a 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -67,6 +67,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: 'Upload to code-scanning' - uses: github/codeql-action/upload-sarif@461ef6c76dfe95d5c364de2f431ddbd31a417628 # v3.26.9 + uses: github/codeql-action/upload-sarif@e2b3eafc8d227b0241d48be5f425d47c2d750a13 # v3.26.10 with: sarif_file: results.sarif diff --git a/gc.c b/gc.c index fc5e18877d9a79..46618d82f0c112 100644 --- a/gc.c +++ b/gc.c @@ -1143,8 +1143,6 @@ rb_gc_obj_free(void *objspace, VALUE obj) { RB_DEBUG_COUNTER_INC(obj_free); - rb_gc_event_hook(obj, RUBY_INTERNAL_EVENT_FREEOBJ); - switch (BUILTIN_TYPE(obj)) { case T_NIL: case T_FIXNUM: diff --git a/gc/default.c b/gc/default.c index 5641a67c6c477a..49df7d0b5ab90c 100644 --- a/gc/default.c +++ b/gc/default.c @@ -939,16 +939,6 @@ heap_eden_total_pages(rb_objspace_t *objspace) return count; } -static inline size_t -heap_eden_total_slots(rb_objspace_t *objspace) -{ - size_t count = 0; - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - count += SIZE_POOL_EDEN_HEAP(&size_pools[i])->total_slots; - } - return count; -} - static inline size_t total_allocated_objects(rb_objspace_t *objspace) { @@ -3565,6 +3555,8 @@ gc_sweep_plane(rb_objspace_t *objspace, rb_heap_t *heap, uintptr_t p, bits_t bit #undef CHECK #endif + rb_gc_event_hook(vp, RUBY_INTERNAL_EVENT_FREEOBJ); + bool has_object_id = FL_TEST(vp, FL_SEEN_OBJ_ID); if (rb_gc_obj_free(objspace, vp)) { if (has_object_id) { @@ -5460,7 +5452,7 @@ gc_marks_finish(rb_objspace_t *objspace) { const unsigned long r_mul = objspace->live_ractor_cache_count > 8 ? 8 : objspace->live_ractor_cache_count; // upto 8 - size_t total_slots = heap_eden_total_slots(objspace); + size_t total_slots = objspace_available_slots(objspace); size_t sweep_slots = total_slots - objspace->marked_slots; /* will be swept slots */ size_t max_free_slots = (size_t)(total_slots * gc_params.heap_free_slots_max_ratio); size_t min_free_slots = (size_t)(total_slots * gc_params.heap_free_slots_min_ratio); @@ -5470,7 +5462,7 @@ gc_marks_finish(rb_objspace_t *objspace) int full_marking = is_full_marking(objspace); - GC_ASSERT(heap_eden_total_slots(objspace) >= objspace->marked_slots); + GC_ASSERT(objspace_available_slots(objspace) >= objspace->marked_slots); /* Setup freeable slots. */ size_t total_init_slots = 0; @@ -5524,7 +5516,7 @@ gc_marks_finish(rb_objspace_t *objspace) gc_report(1, objspace, "gc_marks_finish (marks %"PRIdSIZE" objects, " "old %"PRIdSIZE" objects, total %"PRIdSIZE" slots, " "sweep %"PRIdSIZE" slots, allocatable %"PRIdSIZE" slots, next GC: %s)\n", - objspace->marked_slots, objspace->rgengc.old_objects, heap_eden_total_slots(objspace), sweep_slots, objspace->heap_pages.allocatable_slots, + objspace->marked_slots, objspace->rgengc.old_objects, objspace_available_slots(objspace), sweep_slots, objspace->heap_pages.allocatable_slots, gc_needs_major_flags ? "major" : "minor"); } @@ -6860,7 +6852,7 @@ rb_gc_impl_prepare_heap(void *objspace_ptr) { rb_objspace_t *objspace = objspace_ptr; - size_t orig_total_slots = heap_eden_total_slots(objspace); + size_t orig_total_slots = objspace_available_slots(objspace); size_t orig_allocatable_slots = objspace->heap_pages.allocatable_slots; rb_gc_impl_each_objects(objspace, gc_set_candidate_object_i, objspace_ptr); @@ -6876,7 +6868,7 @@ rb_gc_impl_prepare_heap(void *objspace_ptr) GC_ASSERT(objspace->empty_pages_count == 0); objspace->heap_pages.allocatable_slots = orig_allocatable_slots; - size_t total_slots = heap_eden_total_slots(objspace); + size_t total_slots = objspace_available_slots(objspace); if (orig_total_slots > total_slots) { objspace->heap_pages.allocatable_slots += orig_total_slots - total_slots; } diff --git a/io_buffer.c b/io_buffer.c index 2e789abe0faf7d..cc76f12ecc070a 100644 --- a/io_buffer.c +++ b/io_buffer.c @@ -843,7 +843,8 @@ rb_io_buffer_get_bytes(VALUE self, void **base, size_t *size) static inline void io_buffer_get_bytes_for_writing(struct rb_io_buffer *buffer, void **base, size_t *size) { - if (buffer->flags & RB_IO_BUFFER_READONLY) { + if (buffer->flags & RB_IO_BUFFER_READONLY || + (!NIL_P(buffer->source) && OBJ_FROZEN(buffer->source))) { rb_raise(rb_eIOBufferAccessError, "Buffer is not writable!"); } diff --git a/lib/reline/history.rb b/lib/reline/history.rb index 3f3b65fea64aeb..47c68ba7745e10 100644 --- a/lib/reline/history.rb +++ b/lib/reline/history.rb @@ -19,7 +19,7 @@ def [](index) def []=(index, val) index = check_index(index) - super(index, String.new(val, encoding: Reline.encoding_system_needs)) + super(index, Reline::Unicode.safe_encode(val, Reline.encoding_system_needs)) end def concat(*val) @@ -45,7 +45,7 @@ def push(*val) end end super(*(val.map{ |v| - String.new(v, encoding: Reline.encoding_system_needs) + Reline::Unicode.safe_encode(v, Reline.encoding_system_needs) })) end @@ -56,7 +56,7 @@ def <<(val) if @config.history_size.positive? shift if size + 1 > @config.history_size end - super(String.new(val, encoding: Reline.encoding_system_needs)) + super(Reline::Unicode.safe_encode(val, Reline.encoding_system_needs)) end private def check_index(index) diff --git a/lib/reline/line_editor.rb b/lib/reline/line_editor.rb index c71a5f79eef9c5..56dc235c030362 100644 --- a/lib/reline/line_editor.rb +++ b/lib/reline/line_editor.rb @@ -1325,7 +1325,7 @@ def insert_multiline_text(text) save_old_buffer pre = @buffer_of_lines[@line_index].byteslice(0, @byte_pointer) post = @buffer_of_lines[@line_index].byteslice(@byte_pointer..) - lines = (pre + text.gsub(/\r\n?/, "\n") + post).split("\n", -1) + lines = (pre + Reline::Unicode.safe_encode(text, @encoding).gsub(/\r\n?/, "\n") + post).split("\n", -1) lines << '' if lines.empty? @buffer_of_lines[@line_index, 1] = lines @line_index += lines.size - 1 diff --git a/lib/reline/unicode.rb b/lib/reline/unicode.rb index ef239d5e9ee947..0ec815aeea6f3d 100644 --- a/lib/reline/unicode.rb +++ b/lib/reline/unicode.rb @@ -54,6 +54,22 @@ def self.escape_for_print(str) }.join end + def self.safe_encode(str, encoding) + # Reline only supports utf-8 convertible string. + converted = str.encode(encoding, invalid: :replace, undef: :replace) + return converted if str.encoding == Encoding::UTF_8 || converted.encoding == Encoding::UTF_8 || converted.ascii_only? + + # This code is essentially doing the same thing as + # `str.encode(utf8, **replace_options).encode(encoding, **replace_options)` + # but also avoids unneccesary irreversible encoding conversion. + converted.gsub(/\X/) do |c| + c.encode(Encoding::UTF_8) + c + rescue Encoding::UndefinedConversionError + '?' + end + end + require 'reline/unicode/east_asian_width' def self.get_mbchar_width(mbchar) diff --git a/lib/singleton.rb b/lib/singleton.rb index 6da939124ee064..afe669e06d7e50 100644 --- a/lib/singleton.rb +++ b/lib/singleton.rb @@ -1,4 +1,4 @@ -# frozen_string_literal: false +# frozen_string_literal: true # The Singleton module implements the Singleton pattern. # @@ -94,20 +94,23 @@ module Singleton VERSION = "0.2.0" - # Raises a TypeError to prevent cloning. - def clone - raise TypeError, "can't clone instance of singleton #{self.class}" - end + module SingletonInstanceMethods + # Raises a TypeError to prevent cloning. + def clone + raise TypeError, "can't clone instance of singleton #{self.class}" + end - # Raises a TypeError to prevent duping. - def dup - raise TypeError, "can't dup instance of singleton #{self.class}" - end + # Raises a TypeError to prevent duping. + def dup + raise TypeError, "can't dup instance of singleton #{self.class}" + end - # By default, do not retain any state when marshalling. - def _dump(depth = -1) - '' + # By default, do not retain any state when marshalling. + def _dump(depth = -1) + '' + end end + include SingletonInstanceMethods module SingletonClassMethods # :nodoc: @@ -121,7 +124,7 @@ def _load(str) end def instance # :nodoc: - @singleton__instance__ || @singleton__mutex__.synchronize { @singleton__instance__ ||= new } + @singleton__instance__ || @singleton__mutex__.synchronize { @singleton__instance__ ||= set_instance(new) } end private @@ -130,22 +133,42 @@ def inherited(sub_klass) super Singleton.__init__(sub_klass) end + + def set_instance(val) + @singleton__instance__ = val + end + + def set_mutex(val) + @singleton__mutex__ = val + end end - class << Singleton # :nodoc: + def self.module_with_class_methods + SingletonClassMethods + end + + module SingletonClassProperties + + def self.included(c) + # extending an object with Singleton is a bad idea + c.undef_method :extend_object + end + + def self.extended(c) + # extending an object with Singleton is a bad idea + c.singleton_class.send(:undef_method, :extend_object) + end + def __init__(klass) # :nodoc: klass.instance_eval { - @singleton__instance__ = nil - @singleton__mutex__ = Thread::Mutex.new + set_instance(nil) + set_mutex(Thread::Mutex.new) } klass end private - # extending an object with Singleton is a bad idea - undef_method :extend_object - def append_features(mod) # help out people counting on transitive mixins unless mod.instance_of?(Class) @@ -157,10 +180,11 @@ def append_features(mod) def included(klass) super klass.private_class_method :new, :allocate - klass.extend SingletonClassMethods + klass.extend module_with_class_methods Singleton.__init__(klass) end end + extend SingletonClassProperties ## # :singleton-method: _load @@ -170,3 +194,46 @@ def included(klass) # :singleton-method: instance # Returns the singleton instance. end + +if defined?(Ractor) + module RactorLocalSingleton + include Singleton::SingletonInstanceMethods + + module RactorLocalSingletonClassMethods + include Singleton::SingletonClassMethods + def instance + set_mutex(Thread::Mutex.new) if Ractor.current[mutex_key].nil? + return Ractor.current[instance_key] if Ractor.current[instance_key] + Ractor.current[mutex_key].synchronize { + return Ractor.current[instance_key] if Ractor.current[instance_key] + set_instance(new()) + } + Ractor.current[instance_key] + end + + private + + def instance_key + :"__RactorLocalSingleton_instance_with_class_id_#{object_id}__" + end + + def mutex_key + :"__RactorLocalSingleton_mutex_with_class_id_#{object_id}__" + end + + def set_instance(val) + Ractor.current[instance_key] = val + end + + def set_mutex(val) + Ractor.current[mutex_key] = val + end + end + + def self.module_with_class_methods + RactorLocalSingletonClassMethods + end + + extend Singleton::SingletonClassProperties + end +end diff --git a/parse.y b/parse.y index 1104697b19ab95..e3391d64eed975 100644 --- a/parse.y +++ b/parse.y @@ -2925,6 +2925,11 @@ rb_parser_ary_free(rb_parser_t *p, rb_parser_ary_t *ary) | tCONSTANT ; +%rule %inline user_or_keyword_variable + : user_variable + | keyword_variable + ; + /* * parameterizing rules */ @@ -3657,12 +3662,7 @@ mlhs_post : mlhs_item } ; -mlhs_node : user_variable - { - /*% ripper: var_field!($:1) %*/ - $$ = assignable(p, $1, 0, &@$); - } - | keyword_variable +mlhs_node : user_or_keyword_variable { /*% ripper: var_field!($:1) %*/ $$ = assignable(p, $1, 0, &@$); @@ -3707,12 +3707,7 @@ mlhs_node : user_variable } ; -lhs : user_variable - { - /*% ripper: var_field!($:1) %*/ - $$ = assignable(p, $1, 0, &@$); - } - | keyword_variable +lhs : user_or_keyword_variable { /*% ripper: var_field!($:1) %*/ $$ = assignable(p, $1, 0, &@$); @@ -5202,7 +5197,6 @@ lambda : tLAMBDA[lpar] { token_info_push(p, "->", &@1); $$ = dyna_push(p); - p->lex.lpar_beg = p->lex.paren_nest; }[dyna] max_numparam numparam it_id allow_exits f_larglist[args] @@ -5235,7 +5229,7 @@ lambda : tLAMBDA[lpar] f_larglist : '(' f_args opt_bv_decl ')' { p->ctxt.in_argdef = 0; - $$ = $2; + $$ = $f_args; p->max_numparam = ORDINAL_PARAM; /*% ripper: paren!($:2) %*/ } @@ -5244,7 +5238,7 @@ f_larglist : '(' f_args opt_bv_decl ')' p->ctxt.in_argdef = 0; if (!args_info_empty_p(&$1->nd_ainfo)) p->max_numparam = ORDINAL_PARAM; - $$ = $1; + $$ = $f_args; } ; @@ -6337,12 +6331,7 @@ var_ref : user_variable } ; -var_lhs : user_variable - { - /*% ripper: var_field!($:1) %*/ - $$ = assignable(p, $1, 0, &@$); - } - | keyword_variable +var_lhs : user_or_keyword_variable { /*% ripper: var_field!($:1) %*/ $$ = assignable(p, $1, 0, &@$); @@ -6430,8 +6419,16 @@ args_tail : f_kwarg(f_kw) ',' f_kwrest opt_f_block_arg } | args_forward { - add_forwarding_args(p); - $$ = new_args_tail(p, 0, $1, arg_FWD_BLOCK, &@1); + ID fwd = $args_forward; + if (lambda_beginning_p() || + (p->lex.lpar_beg >= 0 && p->lex.lpar_beg+1 == p->lex.paren_nest)) { + yyerror0("unexpected ... in lambda argument"); + fwd = 0; + } + else { + add_forwarding_args(p); + } + $$ = new_args_tail(p, 0, fwd, arg_FWD_BLOCK, &@1); $$->nd_ainfo.forwarding = 1; /*% ripper: [Qnil, $:1, Qnil] %*/ } @@ -11049,6 +11046,7 @@ parser_yylex(struct parser_params *p) if (c == '>') { SET_LEX_STATE(EXPR_ENDFN); yylval.num = p->lex.lpar_beg; + p->lex.lpar_beg = p->lex.paren_nest; return tLAMBDA; } if (IS_BEG() || (IS_SPCARG(c) && arg_ambiguous(p, '-'))) { @@ -11068,17 +11066,13 @@ parser_yylex(struct parser_params *p) SET_LEX_STATE(EXPR_BEG); if ((c = nextc(p)) == '.') { if ((c = nextc(p)) == '.') { - if (p->ctxt.in_argdef) { + if (p->ctxt.in_argdef || IS_LABEL_POSSIBLE() || lambda_beginning_p()) { SET_LEX_STATE(EXPR_ENDARG); return tBDOT3; } if (p->lex.paren_nest == 0 && looking_at_eol_p(p)) { rb_warn0("... at EOL, should be parenthesized?"); } - else if (p->lex.lpar_beg >= 0 && p->lex.lpar_beg+1 == p->lex.paren_nest) { - if (IS_lex_state_for(last_state, EXPR_LABEL)) - return tDOT3; - } return is_beg ? tBDOT3 : tDOT3; } pushback(p, c); diff --git a/prism/config.yml b/prism/config.yml index b5de4d2b4afced..972f1e8e69a52f 100644 --- a/prism/config.yml +++ b/prism/config.yml @@ -15,7 +15,6 @@ errors: - ARGUMENT_FORMAL_GLOBAL - ARGUMENT_FORMAL_IVAR - ARGUMENT_FORWARDING_UNBOUND - - ARGUMENT_IN - ARGUMENT_NO_FORWARDING_AMPERSAND - ARGUMENT_NO_FORWARDING_ELLIPSES - ARGUMENT_NO_FORWARDING_STAR diff --git a/prism/prism.c b/prism/prism.c index 3c6863ab1bfa82..2e98702d9b5cd4 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -14271,9 +14271,6 @@ parse_arguments(pm_parser_t *parser, pm_arguments_t *arguments, bool accepts_for pm_static_literals_free(&hash_keys); parsed_bare_hash = true; - } else if (accept1(parser, PM_TOKEN_KEYWORD_IN)) { - // TODO: Could we solve this with binding powers instead? - pm_parser_err_current(parser, PM_ERR_ARGUMENT_IN); } parse_arguments_append(parser, arguments, argument); @@ -14786,7 +14783,7 @@ parse_parameters( } default: if (parser->previous.type == PM_TOKEN_COMMA) { - if (allows_trailing_comma) { + if (allows_trailing_comma && order >= PM_PARAMETERS_ORDER_NAMED) { // If we get here, then we have a trailing comma in a // block parameter list. pm_node_t *param = (pm_node_t *) pm_implicit_rest_node_create(parser, &parser->previous); @@ -16545,6 +16542,8 @@ parse_strings(pm_parser_t *parser, pm_node_t *current, bool accepts_label, uint1 pm_string_shared_init(&symbol->unescaped, content.start, content.end); node = (pm_node_t *) symbol; + + if (!label_allowed) pm_parser_err_node(parser, node, PM_ERR_UNEXPECTED_LABEL); } else if (!lex_interpolation) { // If we don't accept interpolation then we expect the string to // start with a single string content node. @@ -21573,6 +21572,19 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t #undef PM_PARSE_PATTERN_TOP #undef PM_PARSE_PATTERN_MULTI +/** + * Determine if a given call node looks like a "command", which means it has + * arguments but does not have parentheses. + */ +static inline bool +pm_call_node_command_p(const pm_call_node_t *node) { + return ( + (node->opening_loc.start == NULL) && + (node->block == NULL || PM_NODE_TYPE_P(node->block, PM_BLOCK_ARGUMENT_NODE)) && + (node->arguments != NULL || node->block != NULL) + ); +} + /** * Parse an expression at the given point of the parser using the given binding * power to parse subsequent chains. If this function finds a syntax error, it @@ -21607,6 +21619,23 @@ parse_expression(pm_parser_t *parser, pm_binding_power_t binding_power, bool acc return node; } break; + case PM_CALL_NODE: + // If we have a call node, then we need to check if it looks like a + // method call without parentheses that contains arguments. If it + // does, then it has different rules for parsing infix operators, + // namely that it only accepts composition (and/or) and modifiers + // (if/unless/etc.). + if ((pm_binding_powers[parser->current.type].left > PM_BINDING_POWER_COMPOSITION) && pm_call_node_command_p((pm_call_node_t *) node)) { + return node; + } + break; + case PM_SYMBOL_NODE: + // If we have a symbol node that is being parsed as a label, then we + // need to immediately return, because there should never be an + // infix operator following this node. + if (pm_symbol_node_label_p(node)) { + return node; + } default: break; } @@ -21622,6 +21651,13 @@ parse_expression(pm_parser_t *parser, pm_binding_power_t binding_power, bool acc node = parse_expression_infix(parser, node, binding_power, current_binding_powers.right, accepts_command_call, (uint16_t) (depth + 1)); switch (PM_NODE_TYPE(node)) { + case PM_MULTI_WRITE_NODE: + // Multi-write nodes are statements, and cannot be followed by + // operators except modifiers. + if (pm_binding_powers[parser->current.type].left > PM_BINDING_POWER_MODIFIER) { + return node; + } + break; case PM_CLASS_VARIABLE_WRITE_NODE: case PM_CONSTANT_PATH_WRITE_NODE: case PM_CONSTANT_WRITE_NODE: @@ -21657,7 +21693,7 @@ parse_expression(pm_parser_t *parser, pm_binding_power_t binding_power, bool acc // 1.. * 2 // if (PM_NODE_TYPE_P(node, PM_RANGE_NODE) && ((pm_range_node_t *) node)->right == NULL) { - if (match3(parser, PM_TOKEN_UAMPERSAND, PM_TOKEN_USTAR, PM_TOKEN_DOT)) { + if (match4(parser, PM_TOKEN_UAMPERSAND, PM_TOKEN_USTAR, PM_TOKEN_DOT, PM_TOKEN_AMPERSAND_DOT)) { PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_NON_ASSOCIATIVE_OPERATOR, pm_token_type_human(parser->current.type), pm_token_type_human(parser->previous.type)); break; } diff --git a/prism/templates/src/diagnostic.c.erb b/prism/templates/src/diagnostic.c.erb index b2381db57fab2c..d2b7b4f691ec9a 100644 --- a/prism/templates/src/diagnostic.c.erb +++ b/prism/templates/src/diagnostic.c.erb @@ -100,7 +100,6 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_ARGUMENT_FORMAL_GLOBAL] = { "invalid formal argument; formal argument cannot be a global variable", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_FORMAL_IVAR] = { "invalid formal argument; formal argument cannot be an instance variable", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_FORWARDING_UNBOUND] = { "unexpected `...` in an non-parenthesized call", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_ARGUMENT_IN] = { "unexpected `in` keyword in arguments", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_NO_FORWARDING_AMPERSAND] = { "unexpected `&`; no anonymous block parameter", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES] = { "unexpected ... when the parent method is not forwarding", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_NO_FORWARDING_STAR] = { "unexpected `*`; no anonymous rest parameter", PM_ERROR_LEVEL_SYNTAX }, diff --git a/spec/ruby/.mspec.constants b/spec/ruby/.mspec.constants index 6e09a44362df22..4da36337152b0e 100644 --- a/spec/ruby/.mspec.constants +++ b/spec/ruby/.mspec.constants @@ -146,6 +146,7 @@ Prime Private ProcFromMethod Psych +RactorLocalSingleton REXML RUBY_SIGNALS RbReadline diff --git a/test/.excludes-parsey/TestBugReporter.rb b/test/.excludes-parsey/TestBugReporter.rb deleted file mode 100644 index 72357760a67b8c..00000000000000 --- a/test/.excludes-parsey/TestBugReporter.rb +++ /dev/null @@ -1 +0,0 @@ -exclude(:test_bug_reporter_add, "unexpected +PRISM in the Ruby description") diff --git a/test/.excludes-parsey/TestRubyOptions.rb b/test/.excludes-parsey/TestRubyOptions.rb deleted file mode 100644 index f6ac8c744a1361..00000000000000 --- a/test/.excludes-parsey/TestRubyOptions.rb +++ /dev/null @@ -1,10 +0,0 @@ -exclude(:test_crash_report_executable_path, "unexpected +PRISM in the Ruby description") -exclude(:test_crash_report_script_path, "unexpected +PRISM in the Ruby description") -exclude(:test_crash_report_script, "unexpected +PRISM in the Ruby description") -exclude(:test_crash_report, "unexpected +PRISM in the Ruby description") -exclude(:test_rjit_disabled_version, "unexpected +PRISM in the Ruby description") -exclude(:test_segv_loaded_features, "unexpected +PRISM in the Ruby description") -exclude(:test_segv_setproctitle, "unexpected +PRISM in the Ruby description") -exclude(:test_segv_test, "unexpected +PRISM in the Ruby description") -exclude(:test_verbose, "unexpected +PRISM in the Ruby description") -exclude(:test_version, "unexpected +PRISM in the Ruby description") diff --git a/test/prism/errors/amperand_dot_after_endless_range.txt b/test/prism/errors/amperand_dot_after_endless_range.txt new file mode 100644 index 00000000000000..ab8c8ccc4d4d10 --- /dev/null +++ b/test/prism/errors/amperand_dot_after_endless_range.txt @@ -0,0 +1,3 @@ +0 if true...&.abs + ^~ unexpected '&.'; ... is a non-associative operator + diff --git a/test/prism/errors/command_call_in.txt b/test/prism/errors/command_call_in.txt index a4357028c67768..2fdcf0973897b7 100644 --- a/test/prism/errors/command_call_in.txt +++ b/test/prism/errors/command_call_in.txt @@ -1,7 +1,5 @@ foo 1 in a - ^ unexpected `in` keyword in arguments - ^ unexpected local variable or method, expecting end-of-input + ^~ unexpected 'in', expecting end-of-input + ^~ unexpected 'in', ignoring it a = foo 2 in b - ^ unexpected `in` keyword in arguments - ^ unexpected local variable or method, expecting end-of-input diff --git a/test/prism/errors/infix_after_label.txt b/test/prism/errors/infix_after_label.txt new file mode 100644 index 00000000000000..c3bcfaecebd928 --- /dev/null +++ b/test/prism/errors/infix_after_label.txt @@ -0,0 +1,6 @@ +{ 'a':.upcase => 1 } + ^ unexpected '.'; expected a value in the hash literal + ^ expected a `}` to close the hash literal + ^ unexpected '}', expecting end-of-input + ^ unexpected '}', ignoring it + diff --git a/test/prism/errors/trailing_comma_after_block.txt b/test/prism/errors/trailing_comma_after_block.txt new file mode 100644 index 00000000000000..d25db0efbeca3b --- /dev/null +++ b/test/prism/errors/trailing_comma_after_block.txt @@ -0,0 +1,3 @@ +p{|&,|} + ^ unexpected `,` in parameters + diff --git a/test/reline/test_history.rb b/test/reline/test_history.rb index ddf8fb14726b3b..ea902b065394ce 100644 --- a/test/reline/test_history.rb +++ b/test/reline/test_history.rb @@ -266,6 +266,15 @@ def test_history_size_negative_unlimited assert_equal 5, history.size end + def test_history_encoding_conversion + history = history_new + text1 = String.new("a\u{65535}b\xFFc", encoding: Encoding::UTF_8) + text2 = String.new("d\xFFe", encoding: Encoding::Shift_JIS) + history.push(text1.dup, text2.dup) + expected = [text1, text2].map { |s| s.encode(Reline.encoding_system_needs, invalid: :replace, undef: :replace) } + assert_equal(expected, history.to_a) + end + private def history_new(history_size: 10) diff --git a/test/reline/test_unicode.rb b/test/reline/test_unicode.rb index deba4d4681173c..688d25e2387780 100644 --- a/test/reline/test_unicode.rb +++ b/test/reline/test_unicode.rb @@ -89,4 +89,32 @@ def test_take_mbchar_range assert_equal ["\e[31mc\1ABC\2d\e[0mef", 2, 4], Reline::Unicode.take_mbchar_range("\e[31mabc\1ABC\2d\e[0mefghi", 2, 4) assert_equal ["\e[41m \e[42mい\e[43m ", 1, 4], Reline::Unicode.take_mbchar_range("\e[41mあ\e[42mい\e[43mう", 1, 4, padding: true) end + + def test_encoding_conversion + texts = [ + String.new("invalid\xFFutf8", encoding: 'utf-8'), + String.new("invalid\xFFsjis", encoding: 'sjis'), + "utf8#{33111.chr('sjis')}convertible", + "utf8#{33222.chr('sjis')}inconvertible", + "sjis->utf8->sjis#{60777.chr('sjis')}irreversible" + ] + utf8_texts = [ + 'invalid�utf8', + 'invalid�sjis', + 'utf8仝convertible', + 'utf8�inconvertible', + 'sjis->utf8->sjis劦irreversible' + ] + sjis_texts = [ + 'invalid?utf8', + 'invalid?sjis', + "utf8#{33111.chr('sjis')}convertible", + 'utf8?inconvertible', + "sjis->utf8->sjis#{60777.chr('sjis')}irreversible" + ] + assert_equal(utf8_texts, texts.map { |s| Reline::Unicode.safe_encode(s, 'utf-8') }) + assert_equal(utf8_texts, texts.map { |s| Reline::Unicode.safe_encode(s, Encoding::UTF_8) }) + assert_equal(sjis_texts, texts.map { |s| Reline::Unicode.safe_encode(s, 'sjis') }) + assert_equal(sjis_texts, texts.map { |s| Reline::Unicode.safe_encode(s, Encoding::Windows_31J) }) + end end diff --git a/test/ruby/test_io_buffer.rb b/test/ruby/test_io_buffer.rb index 7087a2b957d734..98cf45d0c3da07 100644 --- a/test/ruby/test_io_buffer.rb +++ b/test/ruby/test_io_buffer.rb @@ -248,6 +248,31 @@ def test_slice_readonly assert_equal "Hello World", hello end + def test_transfer + hello = %w"Hello World".join(" ") + buffer = IO::Buffer.for(hello) + transferred = buffer.transfer + assert_equal "Hello World", transferred.get_string + assert_predicate buffer, :null? + assert_raise IO::Buffer::AccessError do + transferred.set_string("Goodbye") + end + assert_equal "Hello World", hello + end + + def test_transfer_in_block + hello = %w"Hello World".join(" ") + buffer = IO::Buffer.for(hello, &:transfer) + assert_equal "Hello World", buffer.get_string + buffer.set_string("Ciao!") + assert_equal "Ciao! World", hello + hello.freeze + assert_raise IO::Buffer::AccessError do + buffer.set_string("Hola") + end + assert_equal "Ciao! World", hello + end + def test_locked buffer = IO::Buffer.new(128, IO::Buffer::INTERNAL|IO::Buffer::LOCKED) diff --git a/test/ruby/test_syntax.rb b/test/ruby/test_syntax.rb index 08b4a7ee1c3e4d..16bb914e3f452b 100644 --- a/test/ruby/test_syntax.rb +++ b/test/ruby/test_syntax.rb @@ -1980,6 +1980,10 @@ def test_argument_forwarding assert_syntax_error('iter {|...|}', /unexpected/) assert_syntax_error('->... {}', unexpected) assert_syntax_error('->(...) {}', unexpected) + assert_syntax_error('->a,... {}', unexpected) + assert_syntax_error('->(a,...) {}', unexpected) + assert_syntax_error('->a=1,... {}', unexpected) + assert_syntax_error('->(a=1,...) {}', unexpected) assert_syntax_error('def foo(x, y, z) bar(...); end', /unexpected/) assert_syntax_error('def foo(x, y, z) super(...); end', /unexpected/) assert_syntax_error('def foo(...) yield(...); end', /unexpected/) diff --git a/yjit/src/asm/mod.rs b/yjit/src/asm/mod.rs index bd56074b96e7f7..a113a41c739949 100644 --- a/yjit/src/asm/mod.rs +++ b/yjit/src/asm/mod.rs @@ -432,7 +432,7 @@ impl CodeBlock { } /// Convert an address range to memory page indexes against a num_pages()-sized array. - pub fn addrs_to_pages(&self, start_addr: CodePtr, end_addr: CodePtr) -> Vec { + pub fn addrs_to_pages(&self, start_addr: CodePtr, end_addr: CodePtr) -> impl Iterator { let mem_start = self.mem_block.borrow().start_ptr().raw_addr(self); let mem_end = self.mem_block.borrow().mapped_end_ptr().raw_addr(self); assert!(mem_start <= start_addr.raw_addr(self)); @@ -441,12 +441,12 @@ impl CodeBlock { // Ignore empty code ranges if start_addr == end_addr { - return vec![]; + return (0..0).into_iter(); } let start_page = (start_addr.raw_addr(self) - mem_start) / self.page_size; let end_page = (end_addr.raw_addr(self) - mem_start - 1) / self.page_size; - (start_page..=end_page).collect() // TODO: consider returning an iterator + (start_page..end_page + 1).into_iter() } /// Get a (possibly dangling) direct pointer to the current write position diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs index a105ab97916b97..d3aac992aa5c41 100644 --- a/yjit/src/codegen.rs +++ b/yjit/src/codegen.rs @@ -10563,7 +10563,7 @@ impl CodegenGlobals { let cfunc_exit_code = gen_full_cfunc_return(&mut ocb).unwrap(); let ocb_end_addr = ocb.unwrap().get_write_ptr(); - let ocb_pages = ocb.unwrap().addrs_to_pages(ocb_start_addr, ocb_end_addr); + let ocb_pages = ocb.unwrap().addrs_to_pages(ocb_start_addr, ocb_end_addr).collect(); // Mark all code memory as executable cb.mark_all_executable();