From befb78de4af65d5a557376d2d1fbbd647fa46153 Mon Sep 17 00:00:00 2001 From: David Edey Date: Thu, 2 Jan 2025 20:24:15 +0000 Subject: [PATCH] WIP: Add `while` and `assign` commands --- CHANGELOG.md | 13 ++--- src/command.rs | 75 +++++++++++++++++++------- src/commands/control_flow_commands.rs | 56 +++++++++++++++++++ src/commands/expression_commands.rs | 64 +++++++++++++++++++++- src/commands/mod.rs | 2 + src/commands/token_commands.rs | 4 +- src/internal_prelude.rs | 16 +++--- src/interpreter.rs | 78 ++++++++++++++++++++++----- tests/control_flow.rs | 17 +++++- tests/{evaluate.rs => expressions.rs} | 13 +++++ 10 files changed, 290 insertions(+), 48 deletions(-) rename tests/{evaluate.rs => expressions.rs} (90%) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa5c02f..49059a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,24 +9,25 @@ * `[!increment! ...]` * Control flow commands: * `[!if! COND { ... }]` and `[!if! COND { ... } !else! { ... }]` + * `[!while! cond {}]` * Token-stream utility commands: * `[!empty!]` * `[!is_empty! #stream]` * `[!length! #stream]` which gives the number of token trees in the token stream. * `[!group! ...]` which wraps the tokens in a transparent group. Useful with `!for!`. + * Disallow `[!let! #x =]` and require `[!let! #x = [!empty!]]` (give a good error message). ### To come -* Use `[!let! #x = 12]` instead of `[!set! ..]`. -* Disallow `[!let! #x =]` and require `[!let! #x = [!empty!]]` (give a good error message). -* `[!while! cond {}]` -* `[!for! #x in [#y] {}]`... and make it so that whether commands or variable substitutions get replaced by groups depends on the interpreter context (likely only expressions should use groups) +* ? Use `[!let! #x = 12]` instead of `[!set! ..]`. +* `[!for! #x in [#y] {}]` * `[!range! 0..5]` * `[!error! "message" token stream for span]` * Remove `[!increment! ...]` and replace with `[!assign! #x += 1]` +* Reconfiguring iteration limit * Support `!else if!` in `!if!` -* Token stream manipulation... and make it performant - * Extend token stream +* Token stream manipulation... and make it performant (maybe by storing either TokenStream for extension; or `ParseStream` for consumption) + * Extend token stream `[!extend! #x += ...]` * Consume from start of token stream * Support string & char literals (for comparisons & casts) in expressions * Add more tests diff --git a/src/command.rs b/src/command.rs index 88b40e2..23ca201 100644 --- a/src/command.rs +++ b/src/command.rs @@ -15,6 +15,7 @@ macro_rules! define_commands { } ) => { #[allow(clippy::enum_variant_names)] + #[derive(Clone, Copy)] pub(crate) enum $enum_name { $( $command, @@ -50,6 +51,7 @@ macro_rules! define_commands { } pub(crate) use define_commands; +#[derive(Clone)] pub(crate) struct CommandInvocation { command_kind: CommandKind, command: Command, @@ -79,6 +81,7 @@ impl HasSpanRange for CommandInvocation { } } +#[derive(Clone)] pub(crate) struct Variable { marker: Punct, // # variable_name: Ident, @@ -92,28 +95,63 @@ impl Variable { } } - pub(crate) fn variable_name(&self) -> &Ident { - &self.variable_name + pub(crate) fn variable_name(&self) -> String { + self.variable_name.to_string() } - pub(crate) fn execute_substitution( + pub(crate) fn set<'i>( &self, - interpreter: &mut Interpreter, - ) -> Result { - let Variable { variable_name, .. } = self; - match interpreter.get_variable(&variable_name.to_string()) { - Some(variable_value) => Ok(variable_value.clone()), - None => { - self.span_range().err( - format!( - "The variable {} wasn't set.\nIf this wasn't intended to be a variable, work around this with [!raw! {}]", - self, - self, - ), - ) - } + interpreter: &'i mut Interpreter, + value: TokenStream, + ) { + interpreter.set_variable(self.variable_name(), value); + } + + pub(crate) fn read_substitution<'i>( + &self, + interpreter: &'i Interpreter, + ) -> Result<&'i TokenStream> { + self.read_or_else( + interpreter, + || format!( + "The variable {} wasn't set.\nIf this wasn't intended to be a variable, work around this with [!raw! {}]", + self, + self, + ) + ) + } + + pub(crate) fn read_required<'i>( + &self, + interpreter: &'i Interpreter, + ) -> Result<&'i TokenStream> { + self.read_or_else( + interpreter, + || format!( + "The variable {} wasn't set.", + self, + ) + ) + } + + pub(crate) fn read_or_else<'i>( + &self, + interpreter: &'i Interpreter, + create_error: impl FnOnce() -> String, + ) -> Result<&'i TokenStream> { + match self.read_option(interpreter) { + Some(token_stream) => Ok(token_stream), + None => self.span_range().err(create_error()), } } + + fn read_option<'i>( + &self, + interpreter: &'i Interpreter, + ) -> Option<&'i TokenStream> { + let Variable { variable_name, .. } = self; + interpreter.get_variable(&variable_name.to_string()) + } } impl HasSpanRange for Variable { @@ -128,6 +166,7 @@ impl core::fmt::Display for Variable { } } +#[derive(Clone)] pub(crate) struct Command { command_ident: Ident, command_span: Span, @@ -156,7 +195,7 @@ impl Command { self.command_span.error(message) } - pub(crate) fn err(&self, message: impl core::fmt::Display) -> Result { + pub(crate) fn err(&self, message: impl core::fmt::Display) -> Result { Err(self.error(message)) } diff --git a/src/commands/control_flow_commands.rs b/src/commands/control_flow_commands.rs index 1437cc1..92e510a 100644 --- a/src/commands/control_flow_commands.rs +++ b/src/commands/control_flow_commands.rs @@ -59,3 +59,59 @@ fn parse_if_statement(tokens: &mut Tokens) -> Option { false_code, }) } + +pub(crate) struct WhileCommand; + +impl CommandDefinition for WhileCommand { + const COMMAND_NAME: &'static str = "while"; + + fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { + let parsed = match parse_while_statement(command.argument_tokens()) { + Some(parsed) => parsed, + None => { + return command.err("Expected [!while! (condition) { code }]"); + } + }; + + let mut output = TokenStream::new(); + let mut iteration_count = 0; + loop { + let interpreted_condition = + interpreter.interpret_item(parsed.condition.clone(), SubstitutionMode::expression())?; + let evaluated_condition = evaluate_expression( + interpreted_condition, + ExpressionParsingMode::BeforeCurlyBraces, + )? + .expect_bool("An if condition must evaluate to a boolean")? + .value(); + + iteration_count += 1; + if !evaluated_condition { + break; + } + interpreter.config().check_iteration_count(&command, iteration_count)?; + interpreter.interpret_token_stream_into( + parsed.code.clone(), + SubstitutionMode::token_stream(), + &mut output, + )?; + } + + Ok(output) + } +} + +struct WhileStatement { + condition: NextItem, + code: TokenStream, +} + +fn parse_while_statement(tokens: &mut Tokens) -> Option { + let condition = tokens.next_item().ok()??; + let code = tokens.next_as_kinded_group(Delimiter::Brace)?.stream(); + tokens.check_end()?; + Some(WhileStatement { + condition, + code, + }) +} \ No newline at end of file diff --git a/src/commands/expression_commands.rs b/src/commands/expression_commands.rs index 27b3211..94a1d1a 100644 --- a/src/commands/expression_commands.rs +++ b/src/commands/expression_commands.rs @@ -12,6 +12,66 @@ impl CommandDefinition for EvaluateCommand { } } +pub(crate) struct AssignCommand; + +impl CommandDefinition for AssignCommand { + const COMMAND_NAME: &'static str = "assign"; + + fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { + let AssignStatementStart { + variable, + operator, + } = AssignStatementStart::parse(command.argument_tokens()) + .ok_or_else(|| command.error("Expected [!assign! #variable += ...] for + or some other operator supported in an expression"))?; + + let mut expression_tokens = TokenStream::new(); + expression_tokens.push_new_group( + // TODO: Replace with `variable.read_into(tokens, substitution_mode)` + // TODO: Replace most methods on interpeter with e.g. + // command.interpret_into, next_item.interpet_into, etc. + // And also create an Expression struct + // TODO: Fix Expression to not need different parsing modes, + // and to be parsed from the full token stream or until braces { .. } + // or as a single item + variable.read_required(interpreter)?.clone(), + Delimiter::None, + variable.span_range(), + ); + expression_tokens.push_token_tree(operator.into()); + expression_tokens.push_new_group( + command.interpret_remaining_arguments(interpreter, SubstitutionMode::expression())?, + Delimiter::None, + command.span_range(), + ); + + let output = evaluate_expression(expression_tokens, ExpressionParsingMode::Standard)?.into_token_stream(); + variable.set(interpreter, output); + + Ok(TokenStream::new()) + } +} + +struct AssignStatementStart { + variable: Variable, + operator: Punct, +} + +impl AssignStatementStart { + fn parse(tokens: &mut Tokens) -> Option { + let variable = tokens.next_item_as_variable("").ok()?; + let operator = tokens.next_as_punct()?; + match operator.as_char() { + '+' | '-' | '*' | '/' | '%' | '&' | '|' | '^' => {} + _ => return None, + } + tokens.next_as_punct_matching('=')?; + Some(AssignStatementStart { + variable, + operator, + }) + } +} + pub(crate) struct IncrementCommand; impl CommandDefinition for IncrementCommand { @@ -23,9 +83,9 @@ impl CommandDefinition for IncrementCommand { .argument_tokens() .next_item_as_variable(error_message)?; command.argument_tokens().assert_end(error_message)?; - let variable_contents = variable.execute_substitution(interpreter)?; + let variable_contents = variable.read_substitution(interpreter)?; let evaluated_integer = - evaluate_expression(variable_contents, ExpressionParsingMode::Standard)? + evaluate_expression(variable_contents.clone(), ExpressionParsingMode::Standard)? .expect_integer(&format!("Expected {variable} to evaluate to an integer"))?; interpreter.set_variable( variable.variable_name().to_string(), diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 943474f..ade9f90 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -43,10 +43,12 @@ define_commands! { // Expression Commands EvaluateCommand, + AssignCommand, IncrementCommand, // Control flow commands IfCommand, + WhileCommand, // Token Commands EmptyCommand, diff --git a/src/commands/token_commands.rs b/src/commands/token_commands.rs index 6e6908b..76559ca 100644 --- a/src/commands/token_commands.rs +++ b/src/commands/token_commands.rs @@ -49,9 +49,9 @@ impl CommandDefinition for GroupCommand { fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { let mut output = TokenStream::new(); output.push_new_group( - command.span_range(), - Delimiter::None, command.interpret_remaining_arguments(interpreter, SubstitutionMode::token_stream())?, + Delimiter::None, + command.span_range(), ); Ok(output) } diff --git a/src/internal_prelude.rs b/src/internal_prelude.rs index 62a8b1f..88ab062 100644 --- a/src/internal_prelude.rs +++ b/src/internal_prelude.rs @@ -13,21 +13,26 @@ pub(crate) use crate::string_conversion::*; pub(crate) trait TokenTreeExt: Sized { fn bool(value: bool, span: Span) -> Self; + fn group(tokens: TokenStream, delimeter: Delimiter, span: Span) -> Self; } impl TokenTreeExt for TokenTree { fn bool(value: bool, span: Span) -> Self { TokenTree::Ident(Ident::new(&value.to_string(), span)) } + + fn group(inner_tokens: TokenStream, delimeter: Delimiter, span: Span) -> Self { + TokenTree::Group(Group::new(delimeter, inner_tokens).with_span(span)) + } } pub(crate) trait TokenStreamExt: Sized { fn push_token_tree(&mut self, token_tree: TokenTree); fn push_new_group( &mut self, - span_range: SpanRange, - delimiter: Delimiter, inner_tokens: TokenStream, + delimiter: Delimiter, + span_range: SpanRange, ); } @@ -38,12 +43,11 @@ impl TokenStreamExt for TokenStream { fn push_new_group( &mut self, - span_range: SpanRange, - delimiter: Delimiter, inner_tokens: TokenStream, + delimiter: Delimiter, + span_range: SpanRange, ) { - let group = Group::new(delimiter, inner_tokens).with_span(span_range.span()); - self.push_token_tree(TokenTree::Group(group)); + self.push_token_tree(TokenTree::group(inner_tokens, delimiter, span_range.span())); } } diff --git a/src/interpreter.rs b/src/interpreter.rs index b07ad20..0f6f688 100644 --- a/src/interpreter.rs +++ b/src/interpreter.rs @@ -8,12 +8,14 @@ pub(crate) fn interpret(token_stream: TokenStream) -> Result { } pub(crate) struct Interpreter { + config: InterpreterConfig, variables: HashMap, } impl Interpreter { pub(crate) fn new() -> Self { Self { + config: Default::default(), variables: Default::default(), } } @@ -26,6 +28,10 @@ impl Interpreter { self.variables.get(name) } + pub(crate) fn config(&self) -> &InterpreterConfig { + &self.config + } + pub(crate) fn interpret_token_stream( &mut self, token_stream: TokenStream, @@ -34,13 +40,22 @@ impl Interpreter { self.interpret_tokens(&mut Tokens::new(token_stream), substitution_mode) } + pub(crate) fn interpret_token_stream_into( + &mut self, + token_stream: TokenStream, + substitution_mode: SubstitutionMode, + output: &mut TokenStream, + ) -> Result<()> { + self.interpret_tokens_into(&mut Tokens::new(token_stream), substitution_mode, output) + } + pub(crate) fn interpret_item( &mut self, item: NextItem, substitution_mode: SubstitutionMode, ) -> Result { let mut expanded = TokenStream::new(); - self.interpret_next_item(item, substitution_mode, &mut expanded)?; + self.interpret_next_item_into(item, substitution_mode, &mut expanded)?; Ok(expanded) } @@ -50,27 +65,32 @@ impl Interpreter { substitution_mode: SubstitutionMode, ) -> Result { let mut expanded = TokenStream::new(); + self.interpret_tokens_into(source_tokens, substitution_mode, &mut expanded)?; + Ok(expanded) + } + + pub(crate) fn interpret_tokens_into( + &mut self, + source_tokens: &mut Tokens, + substitution_mode: SubstitutionMode, + output: &mut TokenStream, + ) -> Result<()> { loop { match source_tokens.next_item()? { Some(next_item) => { - self.interpret_next_item(next_item, substitution_mode, &mut expanded)? + self.interpret_next_item_into(next_item, substitution_mode, output)? } - None => return Ok(expanded), + None => return Ok(()), } } } - fn interpret_next_item( + fn interpret_next_item_into( &mut self, next_item: NextItem, substitution_mode: SubstitutionMode, output: &mut TokenStream, ) -> Result<()> { - // We wrap command/variable substitutions in a transparent group so that they - // can be treated as a single item in other commands. - // e.g. if #x = 1 + 1, then [!math! #x * #x] should be 4. - // Note that such groups are ignored in the macro output, due to this - // issue in rustc: https://github.com/rust-lang/rust/issues/67062 match next_item { NextItem::Leaf(token_tree) => { output.push_token_tree(token_tree); @@ -78,16 +98,16 @@ impl Interpreter { NextItem::Group(group) => { // If it's a group, run interpret on its contents recursively. output.push_new_group( - group.span_range(), - group.delimiter(), self.interpret_tokens(&mut Tokens::new(group.stream()), substitution_mode)?, + group.delimiter(), + group.span_range(), ); } NextItem::Variable(variable) => { substitution_mode.apply( output, variable.span_range(), - variable.execute_substitution(self)?, + variable.read_substitution(self)?.clone(), ); } NextItem::CommandInvocation(command_invocation) => { @@ -102,6 +122,29 @@ impl Interpreter { } } +pub(crate) struct InterpreterConfig { + iteration_limit: Option, +} + +impl Default for InterpreterConfig { + fn default() -> Self { + Self { + iteration_limit: Some(10000), + } + } +} + +impl InterpreterConfig { + pub(crate) fn check_iteration_count(&self, command: &Command, count: usize) -> Result<()> { + if let Some(limit) = self.iteration_limit { + if count > limit { + return command.err(format!("Iteration limit of {} exceeded", limit)); + } + } + Ok(()) + } +} + /// How to output `#variables` and `[!commands!]` into the output stream #[derive(Clone, Copy)] pub(crate) struct SubstitutionMode(SubstitutionModeInternal); @@ -139,12 +182,13 @@ impl SubstitutionMode { match self.0 { SubstitutionModeInternal::Extend => tokens.extend(substitution), SubstitutionModeInternal::Group(delimiter) => { - tokens.push_new_group(span_range, delimiter, substitution) + tokens.push_new_group(substitution, delimiter, span_range) } } } } +#[derive(Clone)] pub(crate) struct Tokens(iter::Peekable<::IntoIter>); impl Tokens { @@ -167,6 +211,13 @@ impl Tokens { } } + pub(crate) fn next_as_punct(&mut self) -> Option { + match self.next() { + Some(TokenTree::Punct(punct)) => Some(punct), + _ => None, + } + } + pub(crate) fn next_as_punct_matching(&mut self, char: char) -> Option { match self.next() { Some(TokenTree::Punct(punct)) if punct.as_char() == char => Some(punct), @@ -242,6 +293,7 @@ impl Tokens { } } +#[derive(Clone)] pub(crate) enum NextItem { CommandInvocation(CommandInvocation), Variable(Variable), diff --git a/tests/control_flow.rs b/tests/control_flow.rs index 6e657cd..2221b28 100644 --- a/tests/control_flow.rs +++ b/tests/control_flow.rs @@ -1,3 +1,5 @@ +#![allow(clippy::identity_op)] // https://github.com/rust-lang/rust-clippy/issues/13924 + use preinterpret::preinterpret; macro_rules! assert_preinterpret_eq { @@ -7,7 +9,6 @@ macro_rules! assert_preinterpret_eq { } #[test] -#[allow(clippy::identity_op)] // https://github.com/rust-lang/rust-clippy/issues/13924 fn test_if() { assert_preinterpret_eq!([!if! (1 == 2) { "YES" } !else! { "NO" }], "NO"); assert_preinterpret_eq!({ @@ -28,3 +29,17 @@ fn test_if() { [!if! false { + 1 }] }, 0); } + +#[test] +fn test_while() { + assert_preinterpret_eq!({ + [!set! #x = 0] + [!while! (#x < 5) { [!increment! #x] }] + #x + }, 5); +} + +// TODO: Check compilation error for: +// assert_preinterpret_eq!({ +// [!while! true {}] +// }, 5); \ No newline at end of file diff --git a/tests/evaluate.rs b/tests/expressions.rs similarity index 90% rename from tests/evaluate.rs rename to tests/expressions.rs index 4ecc32e..9f04f25 100644 --- a/tests/evaluate.rs +++ b/tests/expressions.rs @@ -53,5 +53,18 @@ fn increment_works() { ); } +#[test] +fn assign_works() { + assert_preinterpret_eq!( + { + [!set! #x = 8 + 2] // 10 + [!assign! #x /= 1 + 1] // 5 + [!assign! #x += 2 + #x] // 12 + #x + }, + 12 + ); +} + // TODO - Add failing tests for these: // assert_preinterpret_eq!([!evaluate! !!(!!({true}))], true);