feat(whiskers): support alternative template file encodings

currently supports whatever encodings the encoding_rs_io crate[1] supports

there are test cases for UTF-8, UTF-8 with BOM, UTF-16 LE, and UTF-16 BE.

[1] https://github.com/BurntSushi/encoding_rs_io

whiskers/Cargo.toml

@@ -36,17 +36,18 @@ anyhow = "1.0"
 base64 = "0.22"
 catppuccin = { version = "2.2", features = ["serde", "css-colors"] }
 clap = { version = "4.5", features = ["derive"] }
-clap-stdin = "0.4.0"
+clap-stdin = "0.4"
 css-colors = "1.0"
+encoding_rs_io = "0.1"
 indexmap = { version = "2.2", features = ["serde"] }
 itertools = "0.12"
 lzma-rust = "0.1"
 rmp-serde = "1.2"
-semver = { version = "1.0.22", features = ["serde"] }
+semver = { version = "1.0", features = ["serde"] }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 serde_yaml = "0.9"
-tempfile = "3.10.1"
+tempfile = "3.10"
 tera = { version = "1.19", features = ["preserve_order"] }
 thiserror = "1.0"
 

whiskers/src/main.rs

@@ -1,14 +1,15 @@
 use std::{
     collections::{hash_map::Entry, HashMap},
     env,
-    io::Write as _,
+    io::{Read, Write as _},
     path::{Path, PathBuf},
     process,
 };
 
 use anyhow::{anyhow, Context as _};
 use catppuccin::FlavorName;
 use clap::Parser as _;
+use encoding_rs_io::DecodeReaderBytes;
 use itertools::Itertools;
 use whiskers::{
     cli::{Args, OutputFormat},
@@ -81,12 +82,18 @@ fn main() -> anyhow::Result<()> {
         .expect("args.template is guaranteed by clap to be set");
     let template_from_stdin = matches!(template.source, clap_stdin::Source::Stdin);
     let template_name = template_name(&template);
-    let doc = frontmatter::parse(
-        &template
-            .contents()
-            .context("Template contents could not be read")?,
-    )
-    .context("Frontmatter is invalid")?;
+
+    let mut decoder = DecodeReaderBytes::new(
+        template
+            .into_reader()
+            .context("Failed to open template file")?,
+    );
+    let mut template = String::new();
+    decoder
+        .read_to_string(&mut template)
+        .context("Template could not be read")?;
+
+    let doc = frontmatter::parse(&template).context("Frontmatter is invalid")?;
     let mut template_opts =
         TemplateOptions::from_frontmatter(&doc.frontmatter, args.flavor.map(Into::into))
             .context("Could not get template options from frontmatter")?;

whiskers/tests/cli.rs

@@ -36,6 +36,38 @@ mod happy_path {
             "catppuccin-macchiato-yellow-no-italics.ini",
         ));
     }
+
+    /// Test that the CLI can render a UTF-8 template file
+    #[test]
+    fn test_utf8() {
+        let mut cmd = Command::cargo_bin("whiskers").expect("binary exists");
+        let assert = cmd.args(["tests/fixtures/encodings/utf8.tera"]).assert();
+        assert.success().stdout("it worked!");
+    }
+
+    /// Test that the CLI can render a UTF-8 with BOM template file
+    #[test]
+    fn test_utf8_bom() {
+        let mut cmd = Command::cargo_bin("whiskers").expect("binary exists");
+        let assert = cmd.args(["tests/fixtures/encodings/utf8bom.tera"]).assert();
+        assert.success().stdout("it worked!");
+    }
+
+    /// Test that the CLI can render a UTF-16 BE template file
+    #[test]
+    fn test_utf16be() {
+        let mut cmd = Command::cargo_bin("whiskers").expect("binary exists");
+        let assert = cmd.args(["tests/fixtures/encodings/utf16be.tera"]).assert();
+        assert.success().stdout("it worked!");
+    }
+
+    /// Test that the CLI can render a UTF-16 LE template file
+    #[test]
+    fn test_utf16le() {
+        let mut cmd = Command::cargo_bin("whiskers").expect("binary exists");
+        let assert = cmd.args(["tests/fixtures/encodings/utf16le.tera"]).assert();
+        assert.success().stdout("it worked!");
+    }
 }
 
 #[cfg(test)]
@@ -47,9 +79,9 @@ mod sad_path {
     fn nonexistent_template_file() {
         let mut cmd = Command::cargo_bin("whiskers").expect("binary exists");
         cmd.arg("test/file/doesnt/exist");
-        cmd.assert().failure().stderr(predicate::str::contains(
-            "Template contents could not be read",
-        ));
+        cmd.assert()
+            .failure()
+            .stderr(predicate::str::contains("Failed to open template file"));
     }
 
     #[test]

whiskers/tests/encodings.rs

@@ -0,0 +1,36 @@
+//! tests that ensure the special encoding fixtures are left untouched
+#[test]
+fn utf8() {
+    let bytes = &include_bytes!("fixtures/encodings/utf8.tera")[..3];
+    assert_eq!(
+        bytes, b"---",
+        "fixtures/encodings/utf8.tera needs to be re-encoded to UTF-8"
+    );
+}
+
+#[test]
+fn utf8bom() {
+    let bytes = &include_bytes!("fixtures/encodings/utf8bom.tera")[..6];
+    assert_eq!(
+        bytes, b"\xEF\xBB\xBF---",
+        "fixtures/encodings/utf8bom.tera needs to be re-encoded to UTF-8 with BOM"
+    );
+}
+
+#[test]
+fn utf16be() {
+    let bytes = &include_bytes!("fixtures/encodings/utf16be.tera")[..2];
+    assert_eq!(
+        bytes, b"\xFE\xFF",
+        "fixtures/encodings/utf16be.tera needs to be re-encoded to UTF-16 BE"
+    );
+}
+
+#[test]
+fn utf16le() {
+    let bytes = &include_bytes!("fixtures/encodings/utf16le.tera")[..2];
+    assert_eq!(
+        bytes, b"\xFF\xFE",
+        "fixtures/encodings/utf16le.tera needs to be re-encoded to UTF-16 LE"
+    );
+}

whiskers/tests/fixtures/encodings/README.md

@@ -0,0 +1,5 @@
+The fixtures in this directory are encoded in various formats to test the encoding detection and decoding capabilities of Whiskers.
+
+Some text editors like to normalize the encoding of files when saving them. Please be careful not to change them unintentionally.
+
+There are tests in `tests/encodings.rs` that ensure these fixtures are not unintentionally changed.

whiskers/tests/fixtures/encodings/utf8.tera

@@ -0,0 +1,5 @@
+---
+whiskers:
+    version: "2.0"
+---
+it worked!

whiskers/tests/fixtures/encodings/utf8bom.tera

@@ -0,0 +1,5 @@
+---
+whiskers:
+    version: "2.0"
+---
+it worked!