fix(duvet): remove BOM markers (#132)

awslabs · Nov 5, 2024 · 517a0e7 · 517a0e7
1 parent 5ef7827
commit 517a0e7
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 3 deletions.
diff --git a/duvet/src/specification/ietf/snapshots.tar.gz b/duvet/src/specification/ietf/snapshots.tar.gz
diff --git a/duvet/src/specification/ietf/tokenizer.rs b/duvet/src/specification/ietf/tokenizer.rs
@@ -128,7 +128,12 @@ macro_rules! expect_contents {
 
 /// Transforms file contents into lines
 fn lines(contents: &SourceFile) -> impl Iterator<Item = Token> + '_ {
-    contents.lines().enumerate().map(move |(line, value)| {
+    contents.lines().enumerate().map(move |(line, mut value)| {
+        // look for Byte Order Mark and filter it out
+        if line == 0 {
+            value = value.trim_start_matches("\u{feff}");
+        }
+
         // line numbers start at 1
         let line = line + 1;
         let value = contents.substr(value).unwrap();