token stat util

src/lexer/token.rs

@@ -1,6 +1,6 @@
 use std::fmt::{Display, Formatter};
 
-#[derive(Debug, PartialEq, Clone)]
+#[derive(Debug, PartialEq, Clone, Eq, Hash)]
 pub enum TokenKind {
     /// Two consecutive [TokenKind::Eol]
     Terminator,

utlis/src/commands/mod.rs

@@ -0,0 +1,5 @@
+mod random;
+mod token_stat;
+
+pub use random::random;
+pub use token_stat::token_stat;

utlis/src/commands/random.rs

@@ -0,0 +1,76 @@
+use rand::{rngs::ThreadRng, Rng};
+use yamd::lexer::TokenKind;
+
+pub fn random(length: usize, max_literal_len: usize) {
+    let mut rng = rand::thread_rng();
+    let t = TokenKind::Eol;
+    match t {
+        TokenKind::Terminator => 0,
+        TokenKind::Eol => 1,
+        TokenKind::LeftCurlyBrace => 2,
+        TokenKind::RightCurlyBrace => 3,
+        TokenKind::CollapsibleStart => 4,
+        TokenKind::CollapsibleEnd => 5,
+        TokenKind::Tilde => 6,
+        TokenKind::Star => 7,
+        TokenKind::Space => 8,
+        TokenKind::Minus => 9,
+        TokenKind::Hash => 10,
+        TokenKind::GreaterThan => 11,
+        TokenKind::Bang => 12,
+        TokenKind::Backtick => 13,
+        TokenKind::Plus => 14,
+        TokenKind::LeftSquareBracket => 15,
+        TokenKind::RightSquareBracket => 16,
+        TokenKind::LeftParenthesis => 17,
+        TokenKind::RightParenthesis => 18,
+        TokenKind::Underscore => 19,
+        TokenKind::Pipe => 20,
+        TokenKind::Literal => 21,
+    };
+
+    let mut out = String::with_capacity(length + 10);
+    while out.len() < length {
+        let token = match rng.gen_range(0..=21) {
+            0 => "\n\n",
+            1 => "\n",
+            2 => "{",
+            3 => "}",
+            4 => "{%",
+            5 => "%}",
+            6 => "~",
+            7 => "*",
+            8 => " ",
+            9 => "-",
+            10 => "#",
+            11 => ">",
+            12 => "!",
+            13 => "`",
+            14 => "+",
+            15 => "[",
+            16 => "]",
+            17 => "(",
+            18 => ")",
+            19 => "_",
+            20 => "|",
+            _ => &random_string(rng.gen_range(3..max_literal_len), &mut rng),
+        };
+        out.push_str(token);
+    }
+    println!("{out}")
+}
+
+fn random_char(rng: &mut ThreadRng) -> char {
+    match char::from_u32(rng.gen()) {
+        Some(c) => c,
+        None => random_char(rng),
+    }
+}
+
+fn random_string(len: usize, rng: &mut ThreadRng) -> String {
+    let mut str = String::with_capacity(len);
+    for _ in 0..len {
+        str.push(random_char(rng));
+    }
+    str
+}

utlis/src/commands/token_stat.rs

@@ -0,0 +1,33 @@
+use std::{collections::HashMap, fs, path::PathBuf};
+
+use yamd::lexer::{Lexer, TokenKind};
+
+pub fn token_stat(path: PathBuf) {
+    let str = fs::read_to_string(&path).expect("Should have been able to read the file");
+    let l = Lexer::new(str.as_ref());
+    let stat = l.fold(HashMap::<TokenKind, usize>::new(), |mut acc, t| {
+        acc.entry(t.kind)
+            .and_modify(|counter| *counter += 1)
+            .or_insert(1);
+        acc
+    });
+
+    let total: usize = stat.values().sum();
+
+    let mut table: Vec<(String, usize)> = stat
+        .iter()
+        .map(|(kind, count)| (format!("{:?}", kind), *count))
+        .collect();
+
+    println!("\nToken statistics for: {:?}\n", path);
+    println!("Total token count: {total}\n");
+
+    table.sort_by(|(_, left), (_, right)| right.cmp(left));
+    table
+        .iter()
+        .for_each(|(kind, count)| println!("{}:{}{}", kind, tab(kind.as_ref()), count));
+}
+
+fn tab(str: &str) -> String {
+    " ".repeat(20_usize.checked_sub(str.len()).unwrap_or(1))
+}

utlis/src/main.rs

@@ -1,11 +1,17 @@
-use std::char;
+mod commands;
+
+use std::path::PathBuf;
 
 use clap::{Parser, Subcommand};
-use rand::{rngs::ThreadRng, Rng};
-use yamd::lexer::TokenKind;
+use commands::{random, token_stat};
 
 #[derive(Subcommand)]
 enum Commands {
+    /// Token statistics for a file
+    TokenStat {
+        /// Path to a yamd file
+        path: PathBuf,
+    },
     /// Generate random tokens
     Random {
         /// length of a sequence in bytes
@@ -31,79 +37,6 @@ fn main() {
             length,
             max_literal_len,
         } => random(length, max_literal_len),
+        Commands::TokenStat { path } => token_stat(path),
     }
 }
-
-fn random(length: usize, max_literal_len: usize) {
-    let mut rng = rand::thread_rng();
-    let t = TokenKind::Eol;
-    match t {
-        TokenKind::Terminator => 0,
-        TokenKind::Eol => 1,
-        TokenKind::LeftCurlyBrace => 2,
-        TokenKind::RightCurlyBrace => 3,
-        TokenKind::CollapsibleStart => 4,
-        TokenKind::CollapsibleEnd => 5,
-        TokenKind::Tilde => 6,
-        TokenKind::Star => 7,
-        TokenKind::Space => 8,
-        TokenKind::Minus => 9,
-        TokenKind::Hash => 10,
-        TokenKind::GreaterThan => 11,
-        TokenKind::Bang => 12,
-        TokenKind::Backtick => 13,
-        TokenKind::Plus => 14,
-        TokenKind::LeftSquareBracket => 15,
-        TokenKind::RightSquareBracket => 16,
-        TokenKind::LeftParenthesis => 17,
-        TokenKind::RightParenthesis => 18,
-        TokenKind::Underscore => 19,
-        TokenKind::Pipe => 20,
-        TokenKind::Literal => 21,
-    };
-
-    let mut out = String::with_capacity(length + 10);
-    while out.len() < length {
-        let token = match rng.gen_range(0..=21) {
-            0 => "\n\n",
-            1 => "\n",
-            2 => "{",
-            3 => "}",
-            4 => "{%",
-            5 => "%}",
-            6 => "~",
-            7 => "*",
-            8 => " ",
-            9 => "-",
-            10 => "#",
-            11 => ">",
-            12 => "!",
-            13 => "`",
-            14 => "+",
-            15 => "[",
-            16 => "]",
-            17 => "(",
-            18 => ")",
-            19 => "_",
-            20 => "|",
-            _ => &random_string(rng.gen_range(3..max_literal_len), &mut rng),
-        };
-        out.push_str(token);
-    }
-    println!("{out}")
-}
-
-fn random_char(rng: &mut ThreadRng) -> char {
-    match char::from_u32(rng.gen()) {
-        Some(c) => c,
-        None => random_char(rng),
-    }
-}
-
-fn random_string(len: usize, rng: &mut ThreadRng) -> String {
-    let mut str = String::with_capacity(len);
-    for _ in 0..len {
-        str.push(random_char(rng));
-    }
-    str
-}