Add support for dynamo_cpp_guards_str, and some helpers for finding u…

…nrecognized fields (#49)

src/cli.rs

@@ -30,6 +30,9 @@ pub struct Cli {
     /// Some custom HTML to append to the top of report
     #[arg(long, default_value = "")]
     custom_header_html: String,
+    /// Be more chatty
+    #[arg(short, long)]
+    verbose: bool,
 }
 
 fn main() -> anyhow::Result<()> {
@@ -53,6 +56,7 @@ fn main() -> anyhow::Result<()> {
         strict_compile_id: cli.strict_compile_id,
         custom_parsers: Vec::new(),
         custom_header_html: cli.custom_header_html,
+        verbose: cli.verbose,
     };
 
     let output = parse_path(&path, config)?;

src/lib.rs

@@ -1,5 +1,5 @@
 use anyhow::{anyhow, bail};
-use fxhash::FxHashMap;
+use fxhash::{FxHashMap, FxHashSet};
 use md5::{Digest, Md5};
 use std::ffi::{OsStr, OsString};
 
@@ -26,6 +26,7 @@ pub struct ParseConfig {
     pub strict_compile_id: bool,
     pub custom_parsers: Vec<Box<dyn crate::parsers::StructuredLogParser>>,
     pub custom_header_html: String,
+    pub verbose: bool,
 }
 
 impl Default for ParseConfig {
@@ -35,6 +36,7 @@ impl Default for ParseConfig {
             strict_compile_id: false,
             custom_parsers: Vec::default(),
             custom_header_html: String::default(),
+            verbose: false,
         }
     }
 }
@@ -191,6 +193,8 @@ pub fn parse_path(path: &PathBuf, config: ParseConfig) -> anyhow::Result<ParseOu
         TEMPLATE_AOT_AUTOGRAD_BACKWARD_COMPILATION_METRICS,
     )?;
 
+    let mut unknown_fields: FxHashSet<String> = FxHashSet::default();
+
     let mut output_count = 0;
 
     let mut breaks = RestartsAndFailuresContext {
@@ -246,6 +250,15 @@ pub fn parse_path(path: &PathBuf, config: ParseConfig) -> anyhow::Result<ParseOu
             }
         };
 
+        stats.unknown += e._other.len() as u64;
+
+        for k in e._other.keys() {
+            unknown_fields.insert(k.clone());
+            if config.verbose {
+                multi.suspend(|| eprintln!("Unknown field {}", k))
+            }
+        }
+
         if let Some((s, i)) = e.str {
             let mut intern_table = INTERN_TABLE.lock().unwrap();
             intern_table.insert(i, s);
@@ -425,6 +438,10 @@ pub fn parse_path(path: &PathBuf, config: ParseConfig) -> anyhow::Result<ParseOu
     spinner.finish();
 
     eprintln!("{:?}", stats);
+    eprintln!(
+        "Unknown fields: {:?} (consider updating tlparse to render these)",
+        unknown_fields
+    );
 
     let has_unknown_compile_id = directory.contains_key(&None);
 

src/parsers.rs

@@ -497,6 +497,9 @@ pub fn default_parsers<'t>(tt: &'t TinyTemplate<'t>) -> Vec<Box<dyn StructuredLo
         Box::new(SentinelFileParser::new("inductor_post_grad_graph", |e| {
             e.inductor_post_grad_graph.as_ref()
         })),
+        Box::new(SentinelFileParser::new("dynamo_cpp_guards_str", |e| {
+            e.dynamo_cpp_guards_str.as_ref()
+        })),
         Box::new(GraphDumpParser),
         Box::new(DynamoOutputGraphParser),
         Box::new(DynamoGuardParser { tt }),

src/types.rs

@@ -3,6 +3,7 @@ use fxhash::{FxHashMap, FxHasher};
 use html_escape::encode_text;
 use indexmap::IndexMap;
 use regex::Regex;
+use serde_json::Value;
 
 use std::fmt::{self, Display, Write};
 use std::path::PathBuf;
@@ -144,6 +145,7 @@ pub struct Stats {
     pub fail_payload_md5: u64,
     pub fail_dynamo_guards_json: u64,
     pub fail_parser: u64,
+    pub unknown: u64,
 }
 
 #[derive(Debug, Hash, Eq, PartialEq, Deserialize, Serialize, Clone)]
@@ -375,6 +377,7 @@ pub struct Envelope {
     pub aot_backward_graph: Option<EmptyMetadata>,
     pub aot_joint_graph: Option<EmptyMetadata>,
     pub inductor_post_grad_graph: Option<EmptyMetadata>,
+    pub dynamo_cpp_guards_str: Option<EmptyMetadata>,
     pub inductor_output_code: Option<InductorOutputCodeMetadata>,
     pub compilation_metrics: Option<CompilationMetricsMetadata>,
     pub aot_autograd_backward_compilation_metrics:
@@ -383,6 +386,8 @@ pub struct Envelope {
     pub link: Option<LinkMetadata>,
     pub symbolic_shape_specialization: Option<SymbolicShapeSpecializationMetadata>,
     pub artifact: Option<ArtifactMetadata>,
+    #[serde(flatten)]
+    pub _other: FxHashMap<String, Value>,
 }
 
 #[derive(Debug, Deserialize, Serialize)]