From 5fdabb4c1455fbf9952799ecd35b426d65c35644 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Tue, 19 Dec 2023 22:42:42 +0800 Subject: [PATCH 1/2] Add V-TAB and FF to whitespace. --- CHANGELOG.md | 1 + src/tokenizer.rs | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fac126fda..c52f1e9ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ Potentially breaking changes * `ImmutableString` now derefs to `&str` instead of `&SmartString`. Normally this should not be a breaking change. * Traits implemented by `ImmutableString` are cleaned up. Normally this should not be a breaking change. * `EvalContext::new`, `FloatWrapper` and `ConditionalExpr` are now gated under `internals`. +* Previously, Rhai follows [Unicode's definition for _whitespace_](https://en.wikipedia.org/wiki/Template:Whitespace_(Unicode)), which allows many exotic whitespace characters in scripts. Starting from this version, whitespace is strictly defined as the set of six ASCII characters (TAB, SPACE, CR, LF, V-TAB and FF). All other Unicode whitespace characters (not inside strings) are not considered whitespace by Rhai. If a script used to contain non-ASCII whitespace characters, it now fails to parse with a syntax error. Deprecated API's ---------------- diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 86cb274cf..4a72f1d9f 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -2218,8 +2218,8 @@ fn get_next_token_inner( // \n ('\n', ..) => pos.new_line(), - // Whitespace - (' ' | '\t' | '\r', ..) => (), + // Whitespace - follows JavaScript's SPACE, TAB, CR, V-TAB, FF + (' ' | '\t' | '\r' | '\x0b' | '\x0c', ..) => (), _ => { return ( From 374a7e287db87023fc9a8bdfa4bf64c4151fbd71 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Wed, 20 Dec 2023 12:12:08 +0800 Subject: [PATCH 2/2] Fix fuzzing script trimming. --- fuzz/fuzz_targets/scripting.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fuzz/fuzz_targets/scripting.rs b/fuzz/fuzz_targets/scripting.rs index 32b439a9e..2473f46de 100644 --- a/fuzz/fuzz_targets/scripting.rs +++ b/fuzz/fuzz_targets/scripting.rs @@ -24,7 +24,7 @@ fuzz_target!(|ctx: Ctx| { engine.set_optimization_level(ctx.optimization_level); // Limit the length of scripts. - let script = &ctx.script[..(ctx.script.len().min(32 * 1020))]; + let script = ctx.script.chars().take(32 * 1024).collect::(); // We need fuzzing to be fast, so we'll stop executing after 1s. let start = Instant::now(); @@ -32,5 +32,5 @@ fuzz_target!(|ctx: Ctx| { let engine = engine; - _ = engine.run(script); + _ = engine.run(&script); });