zero-copy done, input stream changed accordingly.

src/atn.rs

@@ -4,14 +4,14 @@ use std::rc::Rc;
 use crate::atn_state::ATNState;
 use crate::atn_state::ATNStateRef;
 use crate::atn_type::ATNType;
-use crate::common_token_factory::{CommonTokenFactory, TokenFactory};
 use crate::dfa::ScopeExt;
 use crate::interval_set::IntervalSet;
 use crate::lexer_action::LexerAction;
 use crate::ll1_analyzer::LL1Analyzer;
 use crate::parser_rule_context::{ParserRuleContext, ParserRuleContextType};
 use crate::rule_context::RuleContext;
 use crate::token::{TOKEN_EOF, TOKEN_EPSILON};
+use crate::token_factory::{CommonTokenFactory, TokenFactory};
 use crate::transition::RuleTransition;
 
 pub const INVALID_ALT: isize = 0;

src/common_token_stream.rs

@@ -1,10 +1,10 @@
 use std::borrow::Borrow;
 use std::ops::Deref;
 
-use crate::common_token_factory::TokenFactory;
 use crate::errors::ANTLRError;
 use crate::int_stream::{EOF, IntStream, IterWrapper};
 use crate::token::{OwningToken, Token, TOKEN_DEFAULT_CHANNEL, TOKEN_INVALID_TYPE};
+use crate::token_factory::TokenFactory;
 use crate::token_source::TokenSource;
 use crate::token_stream::{TokenStream, UnbufferedTokenStream};
 

src/error_strategy.rs

@@ -8,13 +8,13 @@ use std::rc::Rc;
 use crate::atn_simulator::IATNSimulator;
 use crate::atn_state::*;
 use crate::char_stream::CharStream;
-use crate::common_token_factory::TokenFactory;
 use crate::dfa::ScopeExt;
 use crate::errors::{ANTLRError, FailedPredicateError, InputMisMatchError, NoViableAltError, RecognitionError};
 use crate::interval_set::IntervalSet;
 use crate::parser::Parser;
 use crate::parser_rule_context::{ParserRuleContext, ParserRuleContextType};
 use crate::token::{OwningToken, Token, TOKEN_DEFAULT_CHANNEL, TOKEN_EOF, TOKEN_EPSILON, TOKEN_INVALID_TYPE};
+use crate::token_factory::TokenFactory;
 use crate::transition::RuleTransition;
 use crate::utils::escape_whitespaces;
 

src/input_stream.rs

@@ -5,7 +5,7 @@ use std::iter::FromIterator;
 use std::marker::PhantomData;
 use std::result;
 use std::slice::from_raw_parts;
-use std::str::Chars;
+use std::str::{CharIndices, Chars};
 
 use crate::char_stream::CharStream;
 use crate::errors::ANTLRError;
@@ -15,10 +15,9 @@ use crate::token::Token;
 
 pub struct InputStream<'a, T: 'a + From<&'a str>> {
     name: String,
-    data_raw: Vec<&'a str>,
+    data_raw: &'a str,
     index: isize,
-    iter: Chars<'a>,
-    data: Vec<isize>,
+    data: Vec<(u32, u32)>,
     phantom: PhantomData<fn() -> T>,
 }
 
@@ -52,12 +51,11 @@ impl<'a> CharStream<'_> for InputStream<'a, String> {
 
 impl<'a, T: 'a + From<&'a str>> InputStream<'a, T> {
     fn new_inner(data_raw: &'a str) -> Self {
-        let data = data_raw.chars().map(|ch| ch as isize).collect();
+        let data = data_raw.char_indices().map(|(i, ch)| (i as u32, ch as u32)).collect();
         Self {
             name: "<empty>".to_string(),
-            data_raw: vec![data_raw],
+            data_raw,
             index: 0,
-            iter: data_raw.chars(),
             data,
             phantom: Default::default(),
         }
@@ -76,11 +74,13 @@ impl<'a, T: 'a + From<&'a str>> InputStream<'a, T> {
     /// Returns text from underlying string for start..=stop range
     #[inline]
     fn text(&self, start: isize, stop: isize) -> &'a str {
-        if ((stop + 1) as usize) < self.data_raw.len() {
-            let stop_offset = self.data_raw[(stop + 1) as usize].as_ptr() as usize - self.data_raw[start as usize].as_ptr() as usize;
-            &self.data_raw[start as usize][..stop_offset]
+        let stop = (stop + 1) as usize;
+        let start_offset = self.data[start as usize].0 as usize;
+        if stop < self.data.len() {
+            let stop_offset = self.data[stop].0 as usize;
+            &self.data_raw[start_offset..stop_offset]
         } else {
-            self.data_raw[start as usize]
+            &self.data_raw[start_offset..]
         }
     }
 }
@@ -92,10 +92,6 @@ impl<'a, T: 'a + From<&'a str>> IntStream for InputStream<'a, T> {
             return Err(ANTLRError::IllegalStateError("cannot consume EOF".into()));
         }
         self.index += 1;
-        if self.data_raw.len() >= self.index as usize {
-            self.iter.next();
-            self.data_raw.push(self.iter.as_str())
-        }
         Ok(())
     }
 
@@ -114,7 +110,7 @@ impl<'a, T: 'a + From<&'a str>> IntStream for InputStream<'a, T> {
         if (self.index + offset - 1) >= self.size() {
             return crate::int_stream::EOF;
         }
-        return self.data[(self.index + offset - 1) as usize] as isize;
+        return self.data[(self.index + offset - 1) as usize].1 as isize;
     }
 
     #[inline]

src/lexer.rs

@@ -5,14 +5,14 @@ use std::ops::DerefMut;
 use std::rc::Rc;
 
 use crate::char_stream::CharStream;
-use crate::common_token_factory::{CommonTokenFactory, TokenAware, TokenFactory};
 use crate::error_listener::{ConsoleErrorListener, ErrorListener};
 use crate::errors::ANTLRError;
 use crate::int_stream::IntStream;
 use crate::lexer_atn_simulator::{ILexerATNSimulator, LexerATNSimulator};
 use crate::parser_rule_context::ParserRuleContext;
 use crate::recognizer::{Actions, Recognizer};
 use crate::token::{Token, TOKEN_INVALID_TYPE};
+use crate::token_factory::{CommonTokenFactory, TokenAware, TokenFactory};
 use crate::token_source::TokenSource;
 
 pub trait Lexer<'input>: TokenSource<'input> + Recognizer<'input> {

src/lib.rs

@@ -52,7 +52,7 @@ pub use parser::{BaseParser, ListenerId, Parser};
 pub use prediction_context::PredictionContextCache;
 
 mod ll1_analyzer;
-pub mod common_token_factory;
+pub mod token_factory;
 pub mod recognizer;
 pub mod int_stream;
 mod lexer_action;
@@ -108,5 +108,21 @@ pub mod atn_type;
 pub mod rule_context;
 pub mod vocabulary;
 
+#[doc(hidden)]
+#[macro_export]
+macro_rules! type_id {
+    ($struct: tt) => {
+        unsafe impl antlr_rust::rule_context::Tid for $struct<'_> {
+            fn self_id(&self) -> TypeId{
+                core::any::TypeId::of::<$struct<'static>>()
+            }
+            fn id() -> TypeId where Self:Sized{
+                core::any::TypeId::of::<$struct<'static>>()
+            }
+
+        }
+    }
+}
+
 //#[cfg(test)]
 // tests are either integration tests in "tests" foulder or unit tests in some modules

src/ll1_analyzer.rs

@@ -7,12 +7,12 @@ use bit_set::BitSet;
 use crate::atn::ATN;
 use crate::atn_config::ATNConfig;
 use crate::atn_state::{ATNState, ATNStateType};
-use crate::common_token_factory::TokenFactory;
 use crate::interval_set::IntervalSet;
 use crate::parser_rule_context::ParserRuleContext;
 use crate::prediction_context::EMPTY_PREDICTION_CONTEXT;
 use crate::prediction_context::PredictionContext;
 use crate::token::{TOKEN_EOF, TOKEN_EPSILON, TOKEN_INVALID_TYPE, TOKEN_MIN_USER_TOKEN_TYPE};
+use crate::token_factory::TokenFactory;
 use crate::transition::{RuleTransition, TransitionType};
 use crate::transition::TransitionType::TRANSITION_NOTSET;
 

src/parser.rs

@@ -10,7 +10,6 @@ use std::sync::Arc;
 
 use crate::atn::ATN;
 use crate::atn_simulator::IATNSimulator;
-use crate::common_token_factory::{CommonTokenFactory, TokenAware, TokenFactory};
 use crate::error_listener::{ConsoleErrorListener, ErrorListener, ProxyErrorListener};
 use crate::error_strategy::ErrorStrategy;
 use crate::errors::ANTLRError;
@@ -21,6 +20,7 @@ use crate::parser_rule_context::{BaseParserRuleContext, ParserRuleContext, Parse
 use crate::recognizer::{Actions, Recognizer};
 use crate::rule_context::RuleContext;
 use crate::token::{OwningToken, Token, TOKEN_EOF};
+use crate::token_factory::{CommonTokenFactory, TokenAware, TokenFactory};
 use crate::token_source::TokenSource;
 use crate::token_stream::TokenStream;
 use crate::tree::{ErrorNode, ErrorNodeCtx, ParseTreeListener, TerminalNode, TerminalNodeCtx};
@@ -57,6 +57,17 @@ pub trait Parser<'input>: Recognizer<'input> {
     fn get_rule_invocation_stack(&self) -> Vec<String>;
 }
 
+trait Context<'input> {
+    type TF: TokenFactory<'input> + 'input;
+    type Listener: ParseTreeListener<'input, Self::TF>;
+    type Node: ParserRuleContext<'input, TF=Self::TF>;
+    type Visitor;
+}
+
+trait Listenable<'input, T: ParseTreeListener<'input>> {
+    fn enter(&self, listener: &mut T);
+}
+
 /// ### Main underlying Parser struct
 ///
 /// It is a member of generated parser struct, so
@@ -479,6 +490,7 @@ impl<'input, I, T, Ext> BaseParser<'input, Ext, I, T>
 //        println!("{}",prev.get_start().unwrap());
         localctx.set_start(Some(prev.start_mut().clone()));
         self.ctx = Some(localctx);
+        1
 
         if self.build_parse_trees {
             self.ctx.as_ref().unwrap().add_child(prev);

src/parser_atn_simulator.rs

@@ -17,7 +17,6 @@ use crate::atn_config_set::ATNConfigSet;
 use crate::atn_simulator::{BaseATNSimulator, IATNSimulator};
 use crate::atn_state::{ATNDecisionState, ATNState, ATNSTATE_BLOCK_END, ATNStateRef, ATNStateType};
 use crate::atn_state::ATNStateType::RuleStopState;
-use crate::common_token_factory::CommonTokenFactory;
 use crate::dfa::{DFA, ScopeExt};
 use crate::dfa_state::{DFAState, DFAStateRef, PredPrediction};
 use crate::errors::{ANTLRError, NoViableAltError};
@@ -31,6 +30,7 @@ use crate::prediction_mode::{all_subsets_conflict, all_subsets_equal, get_alts,
 use crate::rule_context::RuleContext;
 use crate::semantic_context::SemanticContext;
 use crate::token::{Token, TOKEN_EOF, TOKEN_EPSILON};
+use crate::token_factory::CommonTokenFactory;
 use crate::token_stream::TokenStream;
 use crate::transition::{ActionTransition, EpsilonTransition, PrecedencePredicateTransition, PredicateTransition, RuleTransition, Transition, TransitionType};
 

src/parser_rule_context.rs

@@ -7,15 +7,14 @@ use std::marker::PhantomData;
 use std::ops::{Deref, DerefMut};
 use std::rc::Rc;
 
-use crate::common_token_factory::{CommonTokenFactory, TokenFactory};
 use crate::errors::ANTLRError;
 use crate::interval_set::Interval;
-use crate::rule_context::{BaseRuleContext, CustomRuleContext, EmptyCustomRuleContext, RuleContext};
+use crate::rule_context::{BaseRuleContext, CustomRuleContext, EmptyCustomRuleContext, RuleContext, Tid};
 use crate::token::{OwningToken, Token};
+use crate::token_factory::{CommonTokenFactory, TokenFactory};
 use crate::tree::{ErrorNode, ErrorNodeCtx, ParseTree, TerminalNode, TerminalNodeCtx, Tree};
 
-pub trait ParserRuleContext<'input>:
-ParseTree<'input> + RuleContext<'input> + Debug
+pub trait ParserRuleContext<'input>: ParseTree<'input> + RuleContext<'input> + Debug
 {
     fn set_exception(&self, e: ANTLRError);
 
@@ -48,7 +47,7 @@ ParseTree<'input> + RuleContext<'input> + Debug
 
     fn child_of_type<T: ParserRuleContext<'input, TF=Self::TF> + 'input>(&self, pos: usize) -> Option<Rc<T>> where Self: Sized {
         let result = self.get_children().iter()
-            .filter(|&it| it.get_rule_index() == T::type_rule_index())
+            .filter(|&it| it.self_id() == T::id())
             .nth(pos)
             .cloned();
 
@@ -60,15 +59,15 @@ ParseTree<'input> + RuleContext<'input> + Debug
         self.get_children()
             .iter()
             // not fully sound until `non_static_type_id` is implemented
-            .filter(|&it| it.get_rule_index() == T::type_rule_index())
+            .filter(|&it| it.self_id() == T::id())
             .map(|it| cast_rc::<T>(it.clone()))
             .collect()
     }
 
     fn get_token(&self, ttype: isize, pos: usize) -> Option<Rc<TerminalNode<'input, Self::TF>>> {
         self.get_children()
             .iter()
-            .filter(|&it| it.get_rule_index() == TerminalNode::<'input, Self::TF>::type_rule_index())
+            .filter(|&it| it.self_id() == TerminalNode::<'input, Self::TF>::id())
             .map(|it| cast_rc::<TerminalNode<'input, Self::TF>>(it.clone()))
             .filter(|it| it.symbol.borrow().get_token_type() == ttype)
             .nth(pos)
@@ -77,7 +76,7 @@ ParseTree<'input> + RuleContext<'input> + Debug
     fn get_tokens(&self, ttype: isize) -> Vec<Rc<TerminalNode<'input, Self::TF>>> {
         self.get_children()
             .iter()
-            .filter(|&it| it.get_rule_index() == TerminalNode::<'input, Self::TF>::type_rule_index())
+            .filter(|&it| it.self_id() == TerminalNode::<'input, Self::TF>::id())
             .map(|it| cast_rc::<TerminalNode<'input, Self::TF>>(it.clone()))
             .filter(|it| it.symbol.borrow().get_token_type() == ttype)
             .collect()
@@ -212,6 +211,16 @@ impl<'input, Ctx: CustomRuleContext<'input>> CustomRuleContext<'input> for BaseP
     fn get_rule_index(&self) -> usize { self.base.ext.get_rule_index() }
 }
 
+unsafe impl<'input, Ctx: CustomRuleContext<'input>> Tid for BaseParserRuleContext<'input, Ctx> {
+    fn self_id(&self) -> TypeId {
+        self.base.ext.self_id()
+    }
+
+    fn id() -> TypeId where Self: Sized {
+        Ctx::id()
+    }
+}
+
 impl<'input, Ctx: CustomRuleContext<'input>> Deref for BaseParserRuleContext<'input, Ctx> {
     type Target = Ctx;
 
@@ -363,8 +372,8 @@ impl<'input, Ctx: CustomRuleContext<'input> + 'input> BaseParserRuleContext<'inp
     pub fn new_parser_ctx(parent_ctx: Option<ParserRuleContextType<'input, Ctx::TF>>, invoking_state: isize, ext: Ctx) -> Self {
         Self {
             base: BaseRuleContext::new_ctx(parent_ctx, invoking_state, ext),
-            start: RefCell::new(Ctx::TF::create_invalid().clone()),
-            stop: RefCell::new(Ctx::TF::create_invalid().clone()),
+            start: RefCell::new(Ctx::TF::create_invalid()),
+            stop: RefCell::new(Ctx::TF::create_invalid()),
             exception: None,
             children: RefCell::new(vec![]),
         }
@@ -394,7 +403,7 @@ impl<'input, Ctx: CustomRuleContext<'input> + 'input> BaseParserRuleContext<'inp
 #[doc(hidden)]
 pub trait DerefSeal: Deref {}
 
-impl<'input, T: DerefSeal<Target=I> + 'input + Debug, I: ParserRuleContext<'input> + 'input + ?Sized> ParserRuleContext<'input> for T {
+impl<'input, T: DerefSeal<Target=I> + 'input + Debug + Tid, I: ParserRuleContext<'input> + 'input + ?Sized> ParserRuleContext<'input> for T {
     fn set_exception(&self, e: ANTLRError) { self.deref().set_exception(e) }
 
     fn set_start(&self, t: Option<<Self::TF as TokenFactory<'input>>::Tok>) { self.deref().set_start(t) }
@@ -424,7 +433,7 @@ impl<'input, T: DerefSeal<Target=I> + 'input + Debug, I: ParserRuleContext<'inpu
     fn upcast(&self) -> &dyn ParserRuleContext<'input, TF=Self::TF> { self.deref().upcast() }
 }
 
-impl<'input, T: DerefSeal<Target=I> + 'input + Debug, I: ParserRuleContext<'input> + 'input + ?Sized> RuleContext<'input> for T {
+impl<'input, T: DerefSeal<Target=I> + 'input + Debug + Tid, I: ParserRuleContext<'input> + 'input + ?Sized> RuleContext<'input> for T {
     fn get_invoking_state(&self) -> isize { self.deref().get_invoking_state() }
 
     fn set_invoking_state(&self, t: isize) { self.deref().set_invoking_state(t) }
@@ -436,13 +445,13 @@ impl<'input, T: DerefSeal<Target=I> + 'input + Debug, I: ParserRuleContext<'inpu
     fn set_parent(&self, parent: &Option<ParserRuleContextType<'input, Self::TF>>) { self.deref().set_parent(parent) }
 }
 
-impl<'input, T: DerefSeal<Target=I> + 'input + Debug, I: ParserRuleContext<'input> + 'input + ?Sized> ParseTree<'input> for T {
+impl<'input, T: DerefSeal<Target=I> + 'input + Debug + Tid, I: ParserRuleContext<'input> + 'input + ?Sized> ParseTree<'input> for T {
     fn get_source_interval(&self) -> Interval { self.deref().get_source_interval() }
 
     fn get_text(&self) -> String { self.deref().get_text() }
 }
 
-impl<'input, T: DerefSeal<Target=I> + 'input + Debug, I: ParserRuleContext<'input> + 'input + ?Sized> Tree<'input> for T {
+impl<'input, T: DerefSeal<Target=I> + 'input + Debug + Tid, I: ParserRuleContext<'input> + 'input + ?Sized +> Tree<'input> for T {
     fn get_parent(&self) -> Option<ParserRuleContextType<'input, Self::TF>> { self.deref().get_parent() }
 
     fn has_parent(&self) -> bool { self.deref().has_parent() }
@@ -458,11 +467,13 @@ impl<'input, T: DerefSeal<Target=I> + 'input + Debug, I: ParserRuleContext<'inpu
     fn get_children_full(&self) -> &RefCell<Vec<ParserRuleContextType<'input, Self::TF>>> { self.deref().get_children_full() }
 }
 
-impl<'input, T: DerefSeal<Target=I> + 'input + Debug, I: ParserRuleContext<'input> + 'input + ?Sized> CustomRuleContext<'input> for T {
+impl<'input, T: DerefSeal<Target=I> + 'input + Debug + Tid, I: ParserRuleContext<'input> + 'input + ?Sized> CustomRuleContext<'input> for T {
     type TF = I::TF;
 
     fn get_rule_index(&self) -> usize { self.deref().get_rule_index() }
 
+    // fn type_rule_index() -> usize where Self: Sized { unimplemented!() }
+
     fn get_alt_number(&self) -> isize { self.deref().get_alt_number() }
 
     fn set_alt_number(&self, _alt_number: isize) { self.deref().set_alt_number(_alt_number) }

src/prediction_context.rs

@@ -9,11 +9,11 @@ use std::sync::{Arc, RwLock};
 use murmur3::murmur3_32::MurmurHasher;
 
 use crate::atn::ATN;
-use crate::common_token_factory::TokenFactory;
 use crate::dfa::ScopeExt;
 use crate::parser_atn_simulator::MergeCache;
 use crate::parser_rule_context::{empty_ctx, ParserRuleContext};
 use crate::prediction_context::PredictionContext::{Array, Singleton};
+use crate::token_factory::TokenFactory;
 use crate::transition::RuleTransition;
 
 pub const PREDICTION_CONTEXT_EMPTY_RETURN_STATE: isize = 0x7FFFFFFF;

src/recognizer.rs

@@ -1,6 +1,6 @@
 use crate::atn::ATN;
-use crate::common_token_factory::{TokenAware, TokenFactory};
 use crate::parser_rule_context::ParserRuleContext;
+use crate::token_factory::{TokenAware, TokenFactory};
 use crate::vocabulary::Vocabulary;
 
 /// Major version of this runtime.