From d2bb2c4627dd2f23267c70e651548e454baf9ec1 Mon Sep 17 00:00:00 2001 From: Cameron Bytheway Date: Mon, 21 Oct 2024 10:29:15 -0600 Subject: [PATCH] feat: add config file support --- duvet.toml | 4 + duvet/Cargo.toml | 2 +- duvet/src/annotation.rs | 84 ++++-- duvet/src/arguments.rs | 56 ++++ duvet/src/comment.rs | 59 +++++ duvet/src/comment/parser.rs | 242 ++++++++++++++++++ ...comment__tests__content_without_meta.snap} | 5 +- ..._comment__tests__meta_without_content.snap | 10 + ...et__comment__tests__missing_new_line.snap} | 19 +- ...duvet__comment__tests__type_citation.snap} | 19 +- ...uvet__comment__tests__type_exception.snap} | 17 +- .../duvet__comment__tests__type_test.snap} | 17 +- .../duvet__comment__tests__type_todo.snap} | 17 +- ...vet__comment__tokenizer__tests__basic.snap | 28 ++ ...comment__tokenizer__tests__configured.snap | 15 ++ ...ent__tokenizer__tests__duplicate_meta.snap | 16 ++ ...vet__comment__tokenizer__tests__empty.snap | 5 + ...mment__tokenizer__tests__only_unnamed.snap | 14 + duvet/src/{pattern => comment}/tests.rs | 19 +- duvet/src/comment/tokenizer.rs | 183 +++++++++++++ duvet/src/config.rs | 61 +++++ duvet/src/config/v1.rs | 113 ++++++++ duvet/src/extract.rs | 9 +- duvet/src/lib.rs | 36 +-- duvet/src/manifest.rs | 109 ++++++++ duvet/src/parser.rs | 139 ---------- duvet/src/pattern.rs | 211 --------------- ..._pattern__tests__meta_without_content.snap | 7 - duvet/src/project.rs | 120 +++++---- duvet/src/report/ci.rs | 2 +- duvet/src/report/json.rs | 10 +- duvet/src/report/lcov.rs | 2 +- duvet/src/report/mod.rs | 45 ++-- duvet/src/report/stats.rs | 2 +- duvet/src/report/status.rs | 2 +- duvet/src/source.rs | 203 ++++++++------- duvet/src/specification/mod.rs | 4 +- duvet/src/target.rs | 65 ++--- duvet/src/tests.rs | 23 +- 39 files changed, 1279 insertions(+), 715 deletions(-) create mode 100644 duvet.toml create mode 100644 duvet/src/arguments.rs create mode 100644 duvet/src/comment.rs create mode 100644 duvet/src/comment/parser.rs rename duvet/src/{pattern/snapshots/duvet__pattern__tests__content_without_meta.snap => comment/snapshots/duvet__comment__tests__content_without_meta.snap} (70%) create mode 100644 duvet/src/comment/snapshots/duvet__comment__tests__meta_without_content.snap rename duvet/src/{pattern/snapshots/duvet__pattern__tests__missing_new_line.snap => comment/snapshots/duvet__comment__tests__missing_new_line.snap} (51%) rename duvet/src/{pattern/snapshots/duvet__pattern__tests__type_citation.snap => comment/snapshots/duvet__comment__tests__type_citation.snap} (51%) rename duvet/src/{pattern/snapshots/duvet__pattern__tests__type_exception.snap => comment/snapshots/duvet__comment__tests__type_exception.snap} (52%) rename duvet/src/{pattern/snapshots/duvet__pattern__tests__type_test.snap => comment/snapshots/duvet__comment__tests__type_test.snap} (53%) rename duvet/src/{pattern/snapshots/duvet__pattern__tests__type_todo.snap => comment/snapshots/duvet__comment__tests__type_todo.snap} (50%) create mode 100644 duvet/src/comment/snapshots/duvet__comment__tokenizer__tests__basic.snap create mode 100644 duvet/src/comment/snapshots/duvet__comment__tokenizer__tests__configured.snap create mode 100644 duvet/src/comment/snapshots/duvet__comment__tokenizer__tests__duplicate_meta.snap create mode 100644 duvet/src/comment/snapshots/duvet__comment__tokenizer__tests__empty.snap create mode 100644 duvet/src/comment/snapshots/duvet__comment__tokenizer__tests__only_unnamed.snap rename duvet/src/{pattern => comment}/tests.rs (72%) create mode 100644 duvet/src/comment/tokenizer.rs create mode 100644 duvet/src/config.rs create mode 100644 duvet/src/config/v1.rs create mode 100644 duvet/src/manifest.rs delete mode 100644 duvet/src/parser.rs delete mode 100644 duvet/src/pattern.rs delete mode 100644 duvet/src/pattern/snapshots/duvet__pattern__tests__meta_without_content.snap diff --git a/duvet.toml b/duvet.toml new file mode 100644 index 00000000..3bc8eca5 --- /dev/null +++ b/duvet.toml @@ -0,0 +1,4 @@ +version = "1" + +[source] +pattern = "src/**/*.rs" diff --git a/duvet/Cargo.toml b/duvet/Cargo.toml index b0400cfc..8d1cbb1c 100644 --- a/duvet/Cargo.toml +++ b/duvet/Cargo.toml @@ -30,7 +30,7 @@ slug = { version = "0.1" } tokio = { version = "1", features = ["macros", "rt-multi-thread"] } tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } -toml = "0.5" +toml = "0.8" triple_accel = "0.4" url = "2" v_jsonescape = "0.7" diff --git a/duvet/src/annotation.rs b/duvet/src/annotation.rs index a936621a..47110a9c 100644 --- a/duvet/src/annotation.rs +++ b/duvet/src/annotation.rs @@ -8,10 +8,11 @@ use crate::{ }; use anyhow::anyhow; use core::{fmt, ops::Range, str::FromStr}; -use serde::Serialize; +use duvet_core::{diagnostic::IntoDiagnostic, path::Path, query, Result}; +use serde::{Deserialize, Serialize}; use std::{ collections::{BTreeSet, HashMap}, - path::{Path, PathBuf}, + sync::Arc, }; pub type AnnotationSet = BTreeSet; @@ -44,19 +45,52 @@ impl AnnotationSetExt for AnnotationSet { } } +#[query] +pub async fn query() -> Result> { + let mut errors = vec![]; + + // TODO use try_join after fixing `duvet_core::Query` concurrency issues + // let (sources, requirements) = + // try_join!(crate::manifest::sources(), crate::manifest::requirements())?; + let sources = crate::manifest::sources().await?; + let requirements = crate::manifest::requirements().await?; + + let mut tasks = tokio::task::JoinSet::new(); + + for source in sources.iter().chain(requirements.iter()) { + let source = source.clone(); + tasks.spawn(async move { source.annotations().await }); + } + + let mut annotations = AnnotationSet::default(); + while let Some(res) = tasks.join_next().await { + match res.into_diagnostic() { + Ok((local_annotations, local_errors)) => { + annotations.extend(local_annotations); + errors.extend(local_errors.iter().cloned()); + } + Err(err) => { + errors.push(err); + } + } + } + + if !errors.is_empty() { + Err(errors.into()) + } else { + Ok(Arc::new(annotations)) + } +} + #[derive(Debug, PartialEq, PartialOrd, Eq, Ord, Hash)] pub struct Annotation { - pub source: PathBuf, + pub source: Path, pub anno_line: u32, pub anno_column: u32, - pub item_line: u32, - pub item_column: u32, - pub path: String, pub anno: AnnotationType, pub target: String, pub quote: String, pub comment: String, - pub manifest_dir: PathBuf, pub level: AnnotationLevel, pub format: Format, pub tracking_issue: String, @@ -65,6 +99,11 @@ pub struct Annotation { } impl Annotation { + pub fn relative_source(&self) -> &std::path::Path { + let cwd = duvet_core::env::current_dir().unwrap(); + self.source.strip_prefix(&cwd).unwrap_or(&self.source) + } + pub fn target(&self) -> Result { Target::from_annotation(self) } @@ -82,7 +121,7 @@ impl Annotation { true => target_path.into(), // A file path needs to match false => String::from( - self.resolve_file(Path::new(target_path)) + self.resolve_file(target_path.into()) .unwrap() .to_str() .unwrap(), @@ -102,20 +141,18 @@ impl Annotation { }) } - pub fn resolve_file(&self, file: &Path) -> Result { + pub fn resolve_file(&self, file: Path) -> Result { // If we have the right path, just return it if file.is_file() { - return Ok(file.to_path_buf()); + return Ok(file); } - let mut manifest_dir = self.manifest_dir.clone(); - loop { - if manifest_dir.join(file).is_file() { - return Ok(manifest_dir.join(file)); - } + let mut manifest_dir = self.source.as_ref(); + while let Some(dir) = manifest_dir.parent() { + manifest_dir = dir; - if !manifest_dir.pop() { - break; + if manifest_dir.join(&file).is_file() { + return Ok(manifest_dir.join(file).into()); } } @@ -129,9 +166,9 @@ impl Annotation { #[derive(Clone, Copy, Debug, PartialEq, PartialOrd, Eq, Ord, Hash)] pub enum AnnotationType { + Implementation, Spec, Test, - Citation, Exception, Todo, Implication, @@ -139,16 +176,16 @@ pub enum AnnotationType { impl Default for AnnotationType { fn default() -> Self { - Self::Citation + Self::Implementation } } impl fmt::Display for AnnotationType { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { f.write_str(match self { + Self::Implementation => "IMPLEMENTATION", Self::Spec => "SPEC", Self::Test => "TEST", - Self::Citation => "CITATION", Self::Exception => "EXCEPTION", Self::Todo => "TODO", Self::Implication => "IMPLICATION", @@ -163,7 +200,9 @@ impl FromStr for AnnotationType { match v { "SPEC" | "spec" => Ok(Self::Spec), "TEST" | "test" => Ok(Self::Test), - "CITATION" | "citation" => Ok(Self::Citation), + "CITATION" | "citation" | "IMPLEMENTATION" | "implementation" => { + Ok(Self::Implementation) + } "EXCEPTION" | "exception" => Ok(Self::Exception), "TODO" | "todo" => Ok(Self::Todo), "IMPLICATION" | "implication" => Ok(Self::Implication), @@ -173,7 +212,8 @@ impl FromStr for AnnotationType { } // The order is in terms of priority from least to greatest -#[derive(Clone, Copy, Debug, PartialEq, PartialOrd, Eq, Ord, Hash, Serialize)] +#[derive(Clone, Copy, Debug, PartialEq, PartialOrd, Eq, Ord, Hash, Deserialize, Serialize)] +#[serde(rename_all = "UPPERCASE")] pub enum AnnotationLevel { Auto, May, diff --git a/duvet/src/arguments.rs b/duvet/src/arguments.rs new file mode 100644 index 00000000..16ed9b4f --- /dev/null +++ b/duvet/src/arguments.rs @@ -0,0 +1,56 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +use crate::{ + extract, + manifest::{Requirement, Source}, + report, +}; +use clap::Parser; +use duvet_core::{env, path::Path, query, Result}; +use std::sync::Arc; + +#[derive(Debug, Parser)] +pub struct Arguments { + #[clap(short, long, global = true)] + pub config: Option, + + #[command(subcommand)] + pub command: Command, +} + +#[derive(Debug, Parser)] +#[allow(clippy::large_enum_variant)] +pub enum Command { + Extract(extract::Extract), + Report(report::Report), +} + +impl Arguments { + pub async fn exec(&self) -> Result<()> { + match &self.command { + Command::Extract(args) => args.exec().await, + Command::Report(args) => args.exec().await, + } + } + + pub fn load_sources(&self, sources: &mut Vec) { + match &self.command { + Command::Extract(_) => (), + Command::Report(args) => args.load_sources(sources), + } + } + + pub fn load_requirements(&self, requirements: &mut Vec) { + match &self.command { + Command::Extract(_) => (), + Command::Report(args) => args.load_requirements(requirements), + } + } +} + +#[query] +pub async fn get() -> Arc { + let args = env::args(); + Arc::new(Arguments::parse_from(args.iter())) +} diff --git a/duvet/src/comment.rs b/duvet/src/comment.rs new file mode 100644 index 00000000..dc54ed80 --- /dev/null +++ b/duvet/src/comment.rs @@ -0,0 +1,59 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +use crate::annotation::{AnnotationSet, AnnotationType}; +use anyhow::anyhow; +use duvet_core::{diagnostic::Error, file::SourceFile}; +use std::sync::Arc; + +pub mod parser; +pub mod tokenizer; + +#[cfg(test)] +mod tests; + +pub fn extract( + file: &SourceFile, + pattern: &Pattern, + default_type: AnnotationType, +) -> (AnnotationSet, Vec) { + let tokens = tokenizer::tokens(file, pattern); + let mut parser = parser::parse(tokens, default_type); + + let annotations = (&mut parser).collect(); + let errors = parser.errors(); + + (annotations, errors) +} + +#[derive(Clone, Debug, PartialEq, PartialOrd, Eq, Ord, Hash)] +pub struct Pattern { + pub meta: Arc, + pub content: Arc, +} + +impl Default for Pattern { + fn default() -> Self { + Self { + meta: "//=".into(), + content: "//#".into(), + } + } +} + +impl Pattern { + pub fn from_arg(arg: &str) -> Result { + let mut parts = arg.split(',').filter(|p| !p.is_empty()); + let meta = parts.next().expect("should have at least one pattern"); + if meta.is_empty() { + return Err(anyhow!("compliance pattern cannot be empty")); + } + + let content = parts.next().unwrap(); + + let meta = meta.into(); + let content = content.into(); + + Ok(Self { meta, content }) + } +} diff --git a/duvet/src/comment/parser.rs b/duvet/src/comment/parser.rs new file mode 100644 index 00000000..72e316d3 --- /dev/null +++ b/duvet/src/comment/parser.rs @@ -0,0 +1,242 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::tokenizer::Token; +use crate::{ + annotation::{Annotation, AnnotationLevel, AnnotationType}, + specification::Format, +}; +use anyhow::anyhow; +use duvet_core::{ + diagnostic::Error, + ensure, + file::{Slice, SourceFile}, + Result, +}; + +pub fn parse>( + tokens: T, + default_type: AnnotationType, +) -> Parser { + Parser { + prev_line: 0, + meta: Default::default(), + contents: Default::default(), + errors: Default::default(), + tokens: tokens.into_iter(), + default_type, + } +} + +#[derive(Debug, Default)] +struct Meta { + first_meta: Option, + target: Option, + anno: Option<(AnnotationType, Slice)>, + reason: Option, + line: u32, + column: u32, + feature: Option, + tracking_issue: Option, + level: Option<(AnnotationLevel, Slice)>, + format: Option<(Format, Slice)>, +} + +impl Meta { + fn set(&mut self, key: Option, value: Slice) -> Result<()> { + let source_value = value.clone(); + + let prev = match key.as_deref() { + Some("source") => core::mem::replace(&mut self.target, Some(value)), + Some("level") => { + let level = value.parse()?; + core::mem::replace(&mut self.level, Some((level, value))).map(|v| v.1) + } + Some("format") => { + let format = value.parse()?; + core::mem::replace(&mut self.format, Some((format, value))).map(|v| v.1) + } + Some("type") => { + let ty = value.parse()?; + core::mem::replace(&mut self.anno, Some((ty, value))).map(|v| v.1) + } + Some("reason") => core::mem::replace(&mut self.reason, Some(value)), + Some("feature") => core::mem::replace(&mut self.feature, Some(value)), + Some("tracking-issue") => core::mem::replace(&mut self.tracking_issue, Some(value)), + Some(_) => { + let key = key.unwrap(); + let err: Error = anyhow!("invalid metadata field {key:?}").into(); + let err = err.with_source_slice(key, "defined here"); + return Err(err); + } + None => core::mem::replace(&mut self.target, Some(value)), + }; + + if let Some(prev) = prev { + let key = key.as_deref().unwrap_or("source"); + let err: Error = anyhow!("{key:?} already specified").into(); + let err = err + .with_source_slice(source_value, "redefined here") + .with_source_slice(prev, "already defined here"); + return Err(err); + } + + Ok(()) + } + + fn build(self, contents: Vec, default_type: AnnotationType) -> Result { + let first_meta = self.first_meta.unwrap(); + let source = first_meta.file().path().clone(); + + let Some(target) = self.target else { + let err: Error = anyhow!("comment is missing source specification").into(); + let err = err.with_source_slice(first_meta, "defined here"); + return Err(err); + }; + + let anno = self.anno.map_or(default_type, |v| v.0); + + for (allowed, field) in [ + (AnnotationType::Exception, self.reason.as_ref()), + (AnnotationType::Todo, self.tracking_issue.as_ref()), + (AnnotationType::Todo, self.feature.as_ref()), + ] { + if anno != allowed { + if let Some(value) = field { + let err: Error = anyhow!("invalid key for type {anno}").into(); + let err = err.with_source_slice(value.clone(), "defined here"); + return Err(err); + } + } + } + + let mut quote = String::new(); + for (idx, part) in contents.into_iter().enumerate() { + if idx > 0 { + quote.push(' '); + } + quote.push_str(part.trim()); + } + + let annotation = Annotation { + source, + anno_line: self.line, + anno_column: self.column, + anno, + target: target.to_string(), + quote, + comment: self.reason.map(|v| v.to_string()).unwrap_or_default(), + level: self.level.map_or(AnnotationLevel::Auto, |v| v.0), + format: self.format.map_or(Format::Auto, |v| v.0), + tracking_issue: self + .tracking_issue + .map(|v| v.to_string()) + .unwrap_or_default(), + feature: self.feature.map(|v| v.to_string()).unwrap_or_default(), + tags: Default::default(), + }; + + Ok(annotation) + } +} + +pub struct Parser { + prev_line: usize, + default_type: AnnotationType, + meta: Meta, + contents: Vec, + errors: Vec, + tokens: T, +} + +impl> Parser { + pub fn errors(self) -> Vec { + self.errors + } + + fn on_token(&mut self, token: Token) -> Option { + let line_no = token.line_no(); + // if the line number isn't the next expected one then flush + let prev = self.flush_if(line_no > self.prev_line + 1); + self.prev_line = line_no; + + match token { + Token::Meta { + key, + value, + line: _, + } => self.push_meta(Some(key.clone()), value.clone()), + Token::UnnamedMeta { value, line: _ } => self.push_meta(None, value.clone()), + Token::Content { value, line: _ } => { + self.push_contents(value.clone()); + None + } + } + .or(prev) + } + + fn push_meta( + &mut self, + key: Option>, + value: Slice, + ) -> Option { + let prev = self.flush_if(!self.contents.is_empty()); + + if self.meta.first_meta.is_none() { + self.meta.first_meta = Some(value.clone()); + self.meta.line = (self.prev_line + 1) as _; + } + + if let Err(err) = self.meta.set(key, value) { + self.errors.push(err); + } + + prev + } + + fn push_contents(&mut self, value: Slice) { + let file = value.file(); + let value = value.trim(); + if !value.is_empty() { + self.contents.push(file.substr(value).unwrap()); + } + } + + fn flush_if(&mut self, cond: bool) -> Option { + if cond { + self.flush() + } else { + None + } + } + + fn flush(&mut self) -> Option { + let meta = core::mem::take(&mut self.meta); + let contents = core::mem::take(&mut self.contents); + + ensure!(meta.first_meta.is_some(), None); + + match meta.build(contents, self.default_type) { + Ok(anno) => Some(anno), + Err(err) => { + self.errors.push(err); + None + } + } + } +} + +impl> Iterator for Parser { + type Item = Annotation; + + fn next(&mut self) -> Option { + loop { + let Some(token) = self.tokens.next() else { + return self.flush(); + }; + if let Some(annotation) = self.on_token(token) { + return Some(annotation); + } + } + } +} diff --git a/duvet/src/pattern/snapshots/duvet__pattern__tests__content_without_meta.snap b/duvet/src/comment/snapshots/duvet__comment__tests__content_without_meta.snap similarity index 70% rename from duvet/src/pattern/snapshots/duvet__pattern__tests__content_without_meta.snap rename to duvet/src/comment/snapshots/duvet__comment__tests__content_without_meta.snap index f6995255..67a7e165 100644 --- a/duvet/src/pattern/snapshots/duvet__pattern__tests__content_without_meta.snap +++ b/duvet/src/comment/snapshots/duvet__comment__tests__content_without_meta.snap @@ -1,7 +1,8 @@ --- -source: src/pattern/tests.rs +source: duvet/src/comment/tests.rs expression: "parse(\"//=,//#\", r#\"\n //# This is some content without meta\n \"#)" --- -Ok( +( + {}, [], ) diff --git a/duvet/src/comment/snapshots/duvet__comment__tests__meta_without_content.snap b/duvet/src/comment/snapshots/duvet__comment__tests__meta_without_content.snap new file mode 100644 index 00000000..0e3b9f29 --- /dev/null +++ b/duvet/src/comment/snapshots/duvet__comment__tests__meta_without_content.snap @@ -0,0 +1,10 @@ +--- +source: duvet/src/comment/tests.rs +expression: "parse(\"//=,//#\", r#\"\n //= type=todo\n \"#)" +--- +( + {}, + [ + "comment is missing source specification", + ], +) diff --git a/duvet/src/pattern/snapshots/duvet__pattern__tests__missing_new_line.snap b/duvet/src/comment/snapshots/duvet__comment__tests__missing_new_line.snap similarity index 51% rename from duvet/src/pattern/snapshots/duvet__pattern__tests__missing_new_line.snap rename to duvet/src/comment/snapshots/duvet__comment__tests__missing_new_line.snap index 9962c71d..d99e3d59 100644 --- a/duvet/src/pattern/snapshots/duvet__pattern__tests__missing_new_line.snap +++ b/duvet/src/comment/snapshots/duvet__comment__tests__missing_new_line.snap @@ -1,26 +1,23 @@ --- -source: src/pattern/tests.rs -expression: "parse(\"//=,//#\",\n r#\"\n //= https://example.com/spec.txt\n //# Here is my citation\"#)" +source: duvet/src/comment/tests.rs +expression: "parse(\"//=,//#\",\nr#\"\n //= https://example.com/spec.txt\n //# Here is my citation\"#)" --- -Ok( - [ +( + { Annotation { source: "file.rs", anno_line: 2, - anno_column: 7, - item_line: 3, - item_column: 0, - path: "", - anno: Citation, + anno_column: 0, + anno: Implementation, target: "https://example.com/spec.txt", quote: "Here is my citation", comment: "", - manifest_dir: "/", level: Auto, format: Auto, tracking_issue: "", feature: "", tags: {}, }, - ], + }, + [], ) diff --git a/duvet/src/pattern/snapshots/duvet__pattern__tests__type_citation.snap b/duvet/src/comment/snapshots/duvet__comment__tests__type_citation.snap similarity index 51% rename from duvet/src/pattern/snapshots/duvet__pattern__tests__type_citation.snap rename to duvet/src/comment/snapshots/duvet__comment__tests__type_citation.snap index 426d4d10..779869ce 100644 --- a/duvet/src/pattern/snapshots/duvet__pattern__tests__type_citation.snap +++ b/duvet/src/comment/snapshots/duvet__comment__tests__type_citation.snap @@ -1,26 +1,23 @@ --- -source: src/pattern/tests.rs -expression: "parse(\"//=,//#\",\n r#\"\n //= https://example.com/spec.txt\n //# Here is my citation\n \"#)" +source: duvet/src/comment/tests.rs +expression: "parse(\"//=,//#\",\nr#\"\n //= https://example.com/spec.txt\n //# Here is my citation\n \"#)" --- -Ok( - [ +( + { Annotation { source: "file.rs", anno_line: 2, - anno_column: 7, - item_line: 4, - item_column: 0, - path: "", - anno: Citation, + anno_column: 0, + anno: Implementation, target: "https://example.com/spec.txt", quote: "Here is my citation", comment: "", - manifest_dir: "/", level: Auto, format: Auto, tracking_issue: "", feature: "", tags: {}, }, - ], + }, + [], ) diff --git a/duvet/src/pattern/snapshots/duvet__pattern__tests__type_exception.snap b/duvet/src/comment/snapshots/duvet__comment__tests__type_exception.snap similarity index 52% rename from duvet/src/pattern/snapshots/duvet__pattern__tests__type_exception.snap rename to duvet/src/comment/snapshots/duvet__comment__tests__type_exception.snap index 9d5011d1..2840785d 100644 --- a/duvet/src/pattern/snapshots/duvet__pattern__tests__type_exception.snap +++ b/duvet/src/comment/snapshots/duvet__comment__tests__type_exception.snap @@ -1,26 +1,23 @@ --- -source: src/pattern/tests.rs -expression: "parse(\"//=,//#\",\n r#\"\n //= https://example.com/spec.txt\n //= type=exception\n //= reason=This isn't possible currently\n //# Here is my citation\n \"#)" +source: duvet/src/comment/tests.rs +expression: "parse(\"//=,//#\",\nr#\"\n //= https://example.com/spec.txt\n //= type=exception\n //= reason=This isn't possible currently\n //# Here is my citation\n \"#)" --- -Ok( - [ +( + { Annotation { source: "file.rs", anno_line: 2, - anno_column: 7, - item_line: 6, - item_column: 0, - path: "", + anno_column: 0, anno: Exception, target: "https://example.com/spec.txt", quote: "Here is my citation", comment: "This isn't possible currently", - manifest_dir: "/", level: Auto, format: Auto, tracking_issue: "", feature: "", tags: {}, }, - ], + }, + [], ) diff --git a/duvet/src/pattern/snapshots/duvet__pattern__tests__type_test.snap b/duvet/src/comment/snapshots/duvet__comment__tests__type_test.snap similarity index 53% rename from duvet/src/pattern/snapshots/duvet__pattern__tests__type_test.snap rename to duvet/src/comment/snapshots/duvet__comment__tests__type_test.snap index a7ae5752..d38dc4aa 100644 --- a/duvet/src/pattern/snapshots/duvet__pattern__tests__type_test.snap +++ b/duvet/src/comment/snapshots/duvet__comment__tests__type_test.snap @@ -1,26 +1,23 @@ --- -source: src/pattern/tests.rs -expression: "parse(\"//=,//#\",\n r#\"\n //= https://example.com/spec.txt\n //= type=test\n //# Here is my citation\n \"#)" +source: duvet/src/comment/tests.rs +expression: "parse(\"//=,//#\",\nr#\"\n //= https://example.com/spec.txt\n //= type=test\n //# Here is my citation\n \"#)" --- -Ok( - [ +( + { Annotation { source: "file.rs", anno_line: 2, - anno_column: 7, - item_line: 5, - item_column: 0, - path: "", + anno_column: 0, anno: Test, target: "https://example.com/spec.txt", quote: "Here is my citation", comment: "", - manifest_dir: "/", level: Auto, format: Auto, tracking_issue: "", feature: "", tags: {}, }, - ], + }, + [], ) diff --git a/duvet/src/pattern/snapshots/duvet__pattern__tests__type_todo.snap b/duvet/src/comment/snapshots/duvet__comment__tests__type_todo.snap similarity index 50% rename from duvet/src/pattern/snapshots/duvet__pattern__tests__type_todo.snap rename to duvet/src/comment/snapshots/duvet__comment__tests__type_todo.snap index 2a5fd5fd..1100befe 100644 --- a/duvet/src/pattern/snapshots/duvet__pattern__tests__type_todo.snap +++ b/duvet/src/comment/snapshots/duvet__comment__tests__type_todo.snap @@ -1,26 +1,23 @@ --- -source: src/pattern/tests.rs -expression: "parse(\"//=,//#\",\n r#\"\n //= https://example.com/spec.txt\n //= type=todo\n //= feature=cool-things\n //= tracking-issue=123\n //# Here is my citation\n \"#)" +source: duvet/src/comment/tests.rs +expression: "parse(\"//=,//#\",\nr#\"\n //= https://example.com/spec.txt\n //= type=todo\n //= feature=cool-things\n //= tracking-issue=123\n //# Here is my citation\n \"#)" --- -Ok( - [ +( + { Annotation { source: "file.rs", anno_line: 2, - anno_column: 7, - item_line: 7, - item_column: 0, - path: "", + anno_column: 0, anno: Todo, target: "https://example.com/spec.txt", quote: "Here is my citation", comment: "", - manifest_dir: "/", level: Auto, format: Auto, tracking_issue: "123", feature: "cool-things", tags: {}, }, - ], + }, + [], ) diff --git a/duvet/src/comment/snapshots/duvet__comment__tokenizer__tests__basic.snap b/duvet/src/comment/snapshots/duvet__comment__tokenizer__tests__basic.snap new file mode 100644 index 00000000..acfac1ff --- /dev/null +++ b/duvet/src/comment/snapshots/duvet__comment__tokenizer__tests__basic.snap @@ -0,0 +1,28 @@ +--- +source: duvet/src/comment/tokenizer.rs +expression: tokens +--- +[ + UnnamedMeta { + value: "thing goes here", + line: 1, + }, + Meta { + key: "meta", + value: "foo", + line: 2, + }, + Meta { + key: "meta2", + value: "bar", + line: 3, + }, + Content { + value: "content goes", + line: 4, + }, + Content { + value: "here", + line: 5, + }, +] diff --git a/duvet/src/comment/snapshots/duvet__comment__tokenizer__tests__configured.snap b/duvet/src/comment/snapshots/duvet__comment__tokenizer__tests__configured.snap new file mode 100644 index 00000000..7dc15a19 --- /dev/null +++ b/duvet/src/comment/snapshots/duvet__comment__tokenizer__tests__configured.snap @@ -0,0 +1,15 @@ +--- +source: duvet/src/comment/tokenizer.rs +expression: tokens +--- +[ + Meta { + key: "meta", + value: "goes here", + line: 2, + }, + Content { + value: "content goes here", + line: 3, + }, +] diff --git a/duvet/src/comment/snapshots/duvet__comment__tokenizer__tests__duplicate_meta.snap b/duvet/src/comment/snapshots/duvet__comment__tokenizer__tests__duplicate_meta.snap new file mode 100644 index 00000000..2373014c --- /dev/null +++ b/duvet/src/comment/snapshots/duvet__comment__tokenizer__tests__duplicate_meta.snap @@ -0,0 +1,16 @@ +--- +source: duvet/src/comment/tokenizer.rs +expression: tokens +--- +[ + Meta { + key: "meta", + value: "1", + line: 1, + }, + Meta { + key: "meta", + value: "2", + line: 2, + }, +] diff --git a/duvet/src/comment/snapshots/duvet__comment__tokenizer__tests__empty.snap b/duvet/src/comment/snapshots/duvet__comment__tokenizer__tests__empty.snap new file mode 100644 index 00000000..c9955bed --- /dev/null +++ b/duvet/src/comment/snapshots/duvet__comment__tokenizer__tests__empty.snap @@ -0,0 +1,5 @@ +--- +source: duvet/src/comment/tokenizer.rs +expression: tokens +--- +[] diff --git a/duvet/src/comment/snapshots/duvet__comment__tokenizer__tests__only_unnamed.snap b/duvet/src/comment/snapshots/duvet__comment__tokenizer__tests__only_unnamed.snap new file mode 100644 index 00000000..bfdb2601 --- /dev/null +++ b/duvet/src/comment/snapshots/duvet__comment__tokenizer__tests__only_unnamed.snap @@ -0,0 +1,14 @@ +--- +source: duvet/src/comment/tokenizer.rs +expression: tokens +--- +[ + UnnamedMeta { + value: "this is meta", + line: 1, + }, + UnnamedMeta { + value: "this is other meta", + line: 2, + }, +] diff --git a/duvet/src/pattern/tests.rs b/duvet/src/comment/tests.rs similarity index 72% rename from duvet/src/pattern/tests.rs rename to duvet/src/comment/tests.rs index 5a35d52d..0aeff83d 100644 --- a/duvet/src/pattern/tests.rs +++ b/duvet/src/comment/tests.rs @@ -2,24 +2,11 @@ // SPDX-License-Identifier: Apache-2.0 use super::*; -use std::path::{Path, PathBuf}; -fn parse(pattern: &str, value: &str) -> Result, anyhow::Error> { +fn parse(pattern: &str, value: &str) -> (AnnotationSet, Vec) { + let file = SourceFile::new("file.rs", value).unwrap(); let pattern = Pattern::from_arg(pattern).unwrap(); - let path = Path::new("file.rs"); - let mut annotations = Default::default(); - pattern.extract(value, path, &mut annotations)?; - - let annotations = annotations - .into_iter() - .map(|mut annotation| { - // make the manifest dir consistent on all platforms - annotation.manifest_dir = PathBuf::from("/"); - annotation - }) - .collect(); - - Ok(annotations) + extract(&file, &pattern, Default::default()) } macro_rules! snapshot { diff --git a/duvet/src/comment/tokenizer.rs b/duvet/src/comment/tokenizer.rs new file mode 100644 index 00000000..8efb3d54 --- /dev/null +++ b/duvet/src/comment/tokenizer.rs @@ -0,0 +1,183 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +use crate::comment::Pattern; +use duvet_core::file::{Slice, SourceFile}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Token { + Meta { + key: Slice, + value: Slice, + line: usize, + }, + UnnamedMeta { + value: Slice, + line: usize, + }, + Content { + value: Slice, + line: usize, + }, +} + +impl Token { + pub fn line_no(&self) -> usize { + match self { + Token::Meta { line, .. } => *line, + Token::UnnamedMeta { line, .. } => *line, + Token::Content { line, .. } => *line, + } + } +} + +pub fn tokens<'a>(file: &'a SourceFile, style: &'a Pattern) -> Tokenizer<'a> { + Tokenizer::new(&style.meta, &style.content, file) +} + +pub struct Tokenizer<'a> { + meta_prefix: &'a str, + content_prefix: &'a str, + file: &'a SourceFile, + lines: core::iter::Enumerate>, +} + +impl<'a> Tokenizer<'a> { + fn new(meta_prefix: &'a str, content_prefix: &'a str, file: &'a SourceFile) -> Self { + Self { + meta_prefix, + content_prefix, + file, + lines: file.lines().enumerate(), + } + } + + fn on_line(&mut self, line: &str, line_no: usize) -> Option { + let content = line.trim_start(); + if content.is_empty() { + return None; + } + + if let Some(content) = content.strip_prefix(self.meta_prefix) { + let content = content.trim_start(); + return self.on_meta(content, line_no); + } + + if let Some(content) = content.strip_prefix(self.content_prefix) { + let content = content.trim_start(); + return self.on_content(content, line_no); + } + + None + } + + fn on_content(&mut self, content: &str, line_no: usize) -> Option { + let value = self.file.substr(content).unwrap(); + Some(Token::Content { + value, + line: line_no, + }) + } + + fn on_meta(&mut self, meta: &str, line_no: usize) -> Option { + let mut parts = meta.trim_start().splitn(2, '='); + + let key = parts.next().unwrap(); + let key = key.trim_end(); + let key = self.file.substr(key).unwrap(); + + if let Some(value) = parts.next() { + let value = value.trim(); + let value = self.file.substr(value).unwrap(); + Some(Token::Meta { + key, + value, + line: line_no, + }) + } else { + Some(Token::UnnamedMeta { + value: key, + line: line_no, + }) + } + } +} + +impl Iterator for Tokenizer<'_> { + type Item = Token; + + fn next(&mut self) -> Option { + loop { + let (line_no, line) = self.lines.next()?; + if let Some(token) = self.on_line(line, line_no) { + return Some(token); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + macro_rules! snapshot_test { + ($name:ident, $input:expr) => { + snapshot_test!( + $name, + $input, + Pattern { + meta: "//=".into(), + content: "//#".into(), + } + ); + }; + ($name:ident, $input:expr, $config:expr) => { + #[tokio::test] + async fn $name() { + let source = SourceFile::new(file!(), $input).unwrap(); + let config = $config; + let tokens: Vec<_> = tokens(&source, &config).collect(); + insta::assert_debug_snapshot!(stringify!($name), tokens); + } + }; + } + + snapshot_test!(empty, ""); + snapshot_test!( + basic, + r#" + //= thing goes here + //= meta=foo + //= meta2 = bar + //# content goes + //# here + "# + ); + snapshot_test!( + only_unnamed, + r#" + //= this is meta + //= this is other meta + "# + ); + snapshot_test!( + duplicate_meta, + r#" + //= meta=1 + //= meta=2 + "# + ); + snapshot_test!( + configured, + r#" + /* + *= meta=goes here + *# content goes here + */ + "#, + Pattern { + meta: "*=".into(), + content: "*#".into(), + } + ); +} diff --git a/duvet/src/config.rs b/duvet/src/config.rs new file mode 100644 index 00000000..19ee8e8a --- /dev/null +++ b/duvet/src/config.rs @@ -0,0 +1,61 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +use crate::{ + manifest::{Requirement, Source}, + Result, +}; +use duvet_core::{diagnostic::IntoDiagnostic, file::SourceFile, path::Path, vfs}; +use serde::Deserialize; +use std::sync::Arc; + +pub mod v1; + +#[derive(Debug, Deserialize)] +#[serde(tag = "version", deny_unknown_fields)] +pub enum Schema { + #[serde(rename = "1.0", alias = "1")] + V1(v1::Schema), +} + +#[derive(Clone, Debug)] +pub struct Config { + schema: Arc, + file: SourceFile, +} + +impl Config { + pub fn load_sources(&self, sources: &mut Vec) { + match &*self.schema { + Schema::V1(v1) => v1.load_sources(sources, self.file.path()), + } + } + + pub fn load_requirements(&self, requirements: &mut Vec) { + match &*self.schema { + Schema::V1(v1) => v1.load_requirements(requirements, self.file.path()), + } + } +} + +pub async fn load() -> Option> { + let args = crate::arguments::get().await; + let path = args.config.clone().or_else(default_path)?; + Some(load_from_path(path).await) +} + +async fn load_from_path(path: Path) -> Result { + let path = path.canonicalize().into_diagnostic()?; + + let file = vfs::read_string(path).await?; + + let schema = file.as_toml().await?; + + Ok(Config { schema, file }) +} + +fn default_path() -> Option { + let dir = duvet_core::env::current_dir().ok()?; + let path = dir.join("duvet.toml").canonicalize().ok()?; + Some(path.into()) +} diff --git a/duvet/src/config/v1.rs b/duvet/src/config/v1.rs new file mode 100644 index 00000000..0818c0f8 --- /dev/null +++ b/duvet/src/config/v1.rs @@ -0,0 +1,113 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +use crate::annotation::AnnotationType; +use duvet_core::{glob::Glob, path::Path}; +use serde::Deserialize; +use std::sync::Arc; + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Schema { + #[serde(default, rename = "source")] + pub sources: Arc<[Source]>, + + #[serde(default, rename = "requirement")] + pub requirements: Arc<[Requirement]>, +} + +impl Schema { + pub fn load_sources(&self, sources: &mut Vec, file: &Path) { + let root: Path = file.parent().unwrap().into(); + + for source in self.sources.iter() { + sources.push(crate::manifest::Source { + pattern: source.pattern.clone(), + comment_style: (&source.comment_style).into(), + default_type: source.default_type.into(), + root: root.clone(), + }); + } + } + + pub fn load_requirements( + &self, + requirements: &mut Vec, + file: &Path, + ) { + let root: Path = file.parent().unwrap().into(); + + for requirement in self.requirements.iter() { + requirements.push(crate::manifest::Requirement { + pattern: requirement.pattern.clone(), + root: root.clone(), + }); + } + } +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Source { + pub pattern: Glob, + #[serde(default, rename = "comment-style")] + pub comment_style: CommentStyle, + #[serde(rename = "type", default)] + pub default_type: DefaultType, +} + +#[derive(Clone, Copy, Debug, Default, Deserialize)] +#[serde(rename = "lowercase")] +pub enum DefaultType { + #[default] + Implementation, + Spec, + Test, + Exception, + Todo, + Implication, +} + +impl From for AnnotationType { + fn from(value: DefaultType) -> Self { + match value { + DefaultType::Implementation => Self::Implementation, + DefaultType::Spec => Self::Spec, + DefaultType::Test => Self::Test, + DefaultType::Todo => Self::Todo, + DefaultType::Exception => Self::Exception, + DefaultType::Implication => Self::Implication, + } + } +} + +#[derive(Clone, Debug, Deserialize, Hash)] +#[serde(deny_unknown_fields)] +pub struct CommentStyle { + pub meta: Arc, + pub content: Arc, +} + +impl Default for CommentStyle { + fn default() -> Self { + Self { + meta: "//=".into(), + content: "//#".into(), + } + } +} + +impl From<&CommentStyle> for crate::comment::Pattern { + fn from(value: &CommentStyle) -> Self { + Self { + meta: value.meta.clone(), + content: value.content.clone(), + } + } +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Requirement { + pub pattern: Glob, +} diff --git a/duvet/src/extract.rs b/duvet/src/extract.rs index 4530633e..1be46821 100644 --- a/duvet/src/extract.rs +++ b/duvet/src/extract.rs @@ -10,6 +10,7 @@ use crate::{ Error, }; use clap::Parser; +use duvet_core::Result; use lazy_static::lazy_static; use rayon::prelude::*; use regex::{Regex, RegexSet}; @@ -67,11 +68,11 @@ pub struct Extract { } impl Extract { - pub async fn exec(&self) -> Result<(), Error> { - let contents = self.target.load(self.spec_path.as_deref())?; - let local_path = self.target.local(self.spec_path.as_deref()); + pub async fn exec(&self) -> Result { + let spec_path = self.spec_path.as_deref(); + let local_path = self.target.local(spec_path); + let contents = self.target.load(spec_path).await?; let contents = duvet_core::file::SourceFile::new(&*local_path, contents).unwrap(); - let spec = self.format.parse(&contents)?; let sections = extract_sections(&spec); diff --git a/duvet/src/lib.rs b/duvet/src/lib.rs index caaeec02..83fbcbef 100644 --- a/duvet/src/lib.rs +++ b/duvet/src/lib.rs @@ -1,13 +1,15 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -use clap::Parser; -use std::sync::Arc; +use anyhow::Error; +use duvet_core::Result; mod annotation; +mod arguments; +mod comment; +mod config; mod extract; -mod parser; -mod pattern; +mod manifest; mod project; mod report; mod source; @@ -19,32 +21,8 @@ mod text; #[cfg(test)] mod tests; -pub use anyhow::Error; -pub use duvet_core::Result; - -#[allow(clippy::large_enum_variant)] -#[derive(Debug, Parser)] -pub enum Arguments { - Extract(extract::Extract), - Report(report::Report), -} - -#[duvet_core::query(cache)] -pub async fn arguments() -> Arc { - Arc::new(Arguments::parse()) -} - -impl Arguments { - pub async fn exec(&self) -> Result<(), Error> { - match self { - Self::Extract(args) => args.exec().await, - Self::Report(args) => args.exec().await, - } - } -} - pub async fn run() -> Result { - arguments().await.exec().await?; + arguments::get().await.exec().await?; Ok(()) } diff --git a/duvet/src/manifest.rs b/duvet/src/manifest.rs new file mode 100644 index 00000000..d5d82c87 --- /dev/null +++ b/duvet/src/manifest.rs @@ -0,0 +1,109 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +use crate::{annotation::AnnotationType, comment, source::SourceFile}; +use duvet_core::{glob::Glob, path::Path, query, vfs, Result}; +use futures::StreamExt; +use std::{collections::HashSet, sync::Arc}; + +#[derive(Debug)] +pub struct Manifest { + pub compliance: Compliance, +} + +#[derive(Debug)] +pub struct Compliance { + pub sources: Arc<[Source]>, + pub requirements: Arc<[Requirement]>, +} + +#[derive(Debug)] +pub struct Source { + pub pattern: Glob, + pub root: Path, + pub comment_style: comment::Pattern, + pub default_type: AnnotationType, +} + +#[derive(Debug)] +pub struct Requirement { + pub pattern: Glob, + pub root: Path, +} + +#[query] +pub async fn load() -> Result> { + let mut sources = vec![]; + let mut requirements = vec![]; + + let arguments = crate::arguments::get().await; + + arguments.load_sources(&mut sources); + arguments.load_requirements(&mut requirements); + + if let Some(config) = crate::config::load().await { + let config = config?; + config.load_sources(&mut sources); + config.load_requirements(&mut requirements); + } + + let manifest = Manifest { + compliance: Compliance { + sources: sources.into(), + requirements: requirements.into(), + }, + }; + + Ok(Arc::new(manifest)) +} + +// TODO pull this from `.gitignore`s +fn ignores() -> Glob { + Glob::try_from_iter(["**/.git", "**/node_modules", "**/target", "**/build"]).unwrap() +} + +#[query] +pub async fn sources() -> Result>> { + let manifest = load().await?; + + let mut sources = HashSet::new(); + + let ignores = ignores(); + + for source in manifest.compliance.sources.iter() { + let root = vfs::read_dir(&source.root).await?; + let glob = root.glob(source.pattern.clone(), ignores.clone()); + tokio::pin!(glob); + + while let Some(entry) = glob.next().await { + sources.insert(SourceFile::Text( + source.comment_style.clone(), + entry, + source.default_type, + )); + } + } + + Ok(Arc::new(sources)) +} + +#[query] +pub async fn requirements() -> Result>> { + let manifest = load().await?; + + let mut sources = HashSet::new(); + + let ignores = ignores(); + + for requirement in manifest.compliance.requirements.iter() { + let root = vfs::read_dir(&requirement.root).await?; + let glob = root.glob(requirement.pattern.clone(), ignores.clone()); + tokio::pin!(glob); + + while let Some(entry) = glob.next().await { + sources.insert(SourceFile::Spec(entry)); + } + } + + Ok(Arc::new(sources)) +} diff --git a/duvet/src/parser.rs b/duvet/src/parser.rs deleted file mode 100644 index 008aa452..00000000 --- a/duvet/src/parser.rs +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -use crate::{ - annotation::{Annotation, AnnotationLevel, AnnotationType}, - specification::Format, - Error, -}; -use anyhow::anyhow; -use core::convert::TryInto; - -pub struct Parser<'a>(pub &'a [u8]); - -#[derive(Debug, Default)] -pub struct ParsedAnnotation<'a> { - pub target: &'a str, - pub quote: &'a str, - pub anno: AnnotationType, - pub comment: &'a str, - pub source: &'a str, - pub anno_line: u32, - pub anno_column: u32, - pub item_line: u32, - pub item_column: u32, - pub path: &'a str, - pub manifest_dir: &'a str, - pub feature: &'a str, - pub tracking_issue: &'a str, - pub level: AnnotationLevel, - pub format: Format, -} - -const U32_SIZE: usize = core::mem::size_of::(); - -macro_rules! read_u32 { - ($buf:ident) => {{ - let (len, buf) = $buf.split_at(U32_SIZE); - let len = u32::from_le_bytes(len.try_into()?) as usize; - (len, buf) - }}; -} - -impl<'a> ParsedAnnotation<'a> { - fn parse(data: &'a [u8]) -> Result<(Self, &'a [u8]), Error> { - let mut parsed = Self::default(); - let (len_prefix, data) = read_u32!(data); - let (chunk, remaining) = data.split_at(len_prefix); - let (version, mut chunk) = read_u32!(chunk); - - if version != 0 { - return Err(anyhow!(format!("Invalid version {:?}", version))); - } - - while !chunk.is_empty() { - let (name, peek) = chunk.split_at(U32_SIZE); - let (len, peek) = read_u32!(peek); - let (value, peek) = peek.split_at(len); - - macro_rules! to_u32 { - () => { - u32::from_le_bytes(value.try_into()?) - }; - } - - macro_rules! to_str { - () => { - core::str::from_utf8(value)? - }; - } - - match name { - b"spec" => parsed.target = to_str!(), - b"quot" => parsed.quote = to_str!(), - b"anno" => parsed.anno = to_str!().parse()?, - b"comm" => parsed.comment = to_str!(), - b"file" => parsed.source = to_str!(), - b"ilin" => parsed.item_line = to_u32!(), - b"icol" => parsed.item_column = to_u32!(), - b"alin" => parsed.anno_line = to_u32!(), - b"acol" => parsed.anno_column = to_u32!(), - b"path" => parsed.path = to_str!(), - b"mand" => parsed.manifest_dir = to_str!(), - b"slvl" => parsed.level = to_str!().parse()?, - b"sfmt" => parsed.format = to_str!().parse()?, - other => { - if cfg!(debug_assertions) { - panic!("unhandled annotation field {:?}", other) - } - } - } - - chunk = peek; - } - - Ok((parsed, remaining)) - } -} - -impl<'a> From> for Annotation { - fn from(a: ParsedAnnotation<'a>) -> Self { - Annotation { - target: a.target.to_string(), - quote: a.quote.to_string(), - anno: a.anno, - comment: a.comment.to_string(), - source: a.source.into(), - path: a.path.to_string(), - anno_line: a.anno_line, - anno_column: a.anno_column, - item_line: a.item_line, - item_column: a.item_column, - manifest_dir: a.manifest_dir.into(), - level: a.level, - format: a.format, - feature: a.feature.to_string(), - tags: Default::default(), - tracking_issue: a.tracking_issue.to_string(), - } - } -} - -impl Iterator for Parser<'_> { - type Item = Result; - - fn next(&mut self) -> Option { - let data = self.0; - if data.is_empty() { - return None; - } - - match ParsedAnnotation::parse(data) { - Ok((annotation, data)) => { - self.0 = data; - Some(Ok(annotation.into())) - } - Err(err) => Some(Err(err)), - } - } -} diff --git a/duvet/src/pattern.rs b/duvet/src/pattern.rs deleted file mode 100644 index 5f56f471..00000000 --- a/duvet/src/pattern.rs +++ /dev/null @@ -1,211 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -use crate::{ - annotation::{Annotation, AnnotationSet, AnnotationType}, - parser::ParsedAnnotation, - sourcemap::{LinesIter, Str}, - Error, -}; -use anyhow::anyhow; -use std::path::Path; - -#[cfg(test)] -mod tests; - -#[derive(Clone, Copy, Debug, PartialEq, PartialOrd, Eq, Ord, Hash)] -pub struct Pattern<'a> { - meta: &'a str, - content: &'a str, -} - -impl Default for Pattern<'_> { - fn default() -> Self { - Self { - meta: "//=", - content: "//#", - } - } -} - -impl<'a> Pattern<'a> { - pub fn from_arg(arg: &'a str) -> Result { - let mut parts = arg.split(',').filter(|p| !p.is_empty()); - let meta = parts.next().expect("should have at least one pattern"); - if meta.is_empty() { - return Err(anyhow!("compliance pattern cannot be empty")); - } - - let content = parts.next().unwrap(); - - Ok(Self { meta, content }) - } - - pub fn extract( - &self, - source: &str, - path: &Path, - annotations: &mut AnnotationSet, - ) -> Result<(), Error> { - let mut state = ParserState::Search; - - let mut last_line = 0; - for Str { value, line, .. } in LinesIter::new(source) { - state.on_line(path, annotations, self, value, line)?; - last_line = line; - } - - // make sure we finish off the state machine - state.on_line(path, annotations, self, "", last_line)?; - - Ok(()) - } - - fn try_meta<'b>(&self, line: &'b str) -> Option<&'b str> { - line.strip_prefix(self.meta) - } - - fn try_content<'b>(&self, line: &'b str) -> Option<&'b str> { - line.strip_prefix(self.content) - } -} - -enum ParserState<'a> { - Search, - CapturingMeta(Capture<'a>), - CapturingContent(Capture<'a>), -} - -impl<'a> ParserState<'a> { - fn on_line( - &mut self, - path: &Path, - annotations: &mut AnnotationSet, - pattern: &Pattern, - line: &'a str, - line_no: usize, - ) -> Result<(), Error> { - let content = line.trim_start(); - - match core::mem::replace(self, ParserState::Search) { - ParserState::Search => { - let content = if let Some(content) = pattern.try_meta(content) { - content - } else { - return Ok(()); - }; - - if content.is_empty() { - return Ok(()); - } - - let indent = line.len() - content.len(); - let mut capture = Capture::new(line_no, indent); - capture.push_meta(content)?; - - *self = ParserState::CapturingMeta(capture); - } - ParserState::CapturingMeta(mut capture) => { - if let Some(meta) = pattern.try_meta(content) { - capture.push_meta(meta)?; - *self = ParserState::CapturingMeta(capture); - } else if let Some(content) = pattern.try_content(content) { - capture.push_content(content); - *self = ParserState::CapturingContent(capture); - } else { - annotations.insert(capture.done(line_no, path)?); - } - } - ParserState::CapturingContent(mut capture) => { - if pattern.try_meta(content).is_some() { - return Err(anyhow!("cannot set metadata while parsing content")); - } else if let Some(content) = pattern.try_content(content) { - capture.push_content(content); - *self = ParserState::CapturingContent(capture); - } else { - annotations.insert(capture.done(line_no, path)?); - } - } - } - - Ok(()) - } -} - -#[derive(Debug)] -struct Capture<'a> { - contents: String, - annotation: ParsedAnnotation<'a>, -} - -impl<'a> Capture<'a> { - fn new(line: usize, column: usize) -> Self { - Self { - contents: String::new(), - annotation: ParsedAnnotation { - anno_line: line as _, - anno_column: column as _, - item_line: line as _, - item_column: column as _, - ..Default::default() - }, - } - } - - fn push_meta(&mut self, value: &'a str) -> Result<(), Error> { - let mut parts = value.trim_start().splitn(2, '='); - - let key = parts.next().unwrap(); - let value = parts.next(); - - match (key, value) { - ("source", Some(value)) => self.annotation.target = value, - ("level", Some(value)) => self.annotation.level = value.parse()?, - ("format", Some(value)) => self.annotation.format = value.parse()?, - ("type", Some(value)) => self.annotation.anno = value.parse()?, - ("reason", Some(value)) if self.annotation.anno == AnnotationType::Exception => { - self.annotation.comment = value - } - ("feature", Some(value)) if self.annotation.anno == AnnotationType::Todo => { - self.annotation.feature = value - } - ("tracking-issue", Some(value)) if self.annotation.anno == AnnotationType::Todo => { - self.annotation.tracking_issue = value - } - (key, Some(_)) => return Err(anyhow!(format!("invalid metadata field {}", key))), - (value, None) if self.annotation.target.is_empty() => self.annotation.target = value, - (_, None) => return Err(anyhow!("annotation source already specified")), - } - - Ok(()) - } - - fn push_content(&mut self, value: &'a str) { - let value = value.trim(); - if !value.is_empty() { - self.contents.push_str(value); - self.contents.push(' '); - } - } - - fn done(self, item_line: usize, path: &Path) -> Result { - let mut annotation = Annotation { - item_line: item_line as _, - item_column: 0, - source: path.into(), - quote: self.contents, - manifest_dir: std::env::current_dir()?, - ..self.annotation.into() - }; - - while annotation.quote.ends_with(' ') { - annotation.quote.pop(); - } - - if annotation.target.is_empty() { - return Err(anyhow!("missing source information")); - } - - Ok(annotation) - } -} diff --git a/duvet/src/pattern/snapshots/duvet__pattern__tests__meta_without_content.snap b/duvet/src/pattern/snapshots/duvet__pattern__tests__meta_without_content.snap deleted file mode 100644 index 939c4212..00000000 --- a/duvet/src/pattern/snapshots/duvet__pattern__tests__meta_without_content.snap +++ /dev/null @@ -1,7 +0,0 @@ ---- -source: src/pattern/tests.rs -expression: "parse(\"//=,//#\", r#\"\n //= type=todo\n \"#)" ---- -Err( - "missing source information", -) diff --git a/duvet/src/project.rs b/duvet/src/project.rs index 0fc584dd..7f5ea08b 100644 --- a/duvet/src/project.rs +++ b/duvet/src/project.rs @@ -1,60 +1,56 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -use crate::{pattern::Pattern, source::SourceFile, Error}; +use crate::{ + comment::Pattern, + manifest::{Requirement, Source}, +}; use clap::Parser; -use glob::glob; -use std::collections::HashSet; +use duvet_core::glob::Glob; #[derive(Debug, PartialEq, PartialOrd, Eq, Ord, Hash, Parser)] -pub struct Project { - /// Package to run tests for - #[clap(long, short = 'p')] +struct Deprecated { + #[clap(long, short = 'p', hide = true)] package: Option, - /// Space or comma separated list of features to activate - #[clap(long)] + #[clap(long, hide = true)] features: Vec, - /// Build all packages in the workspace - #[clap(long)] + #[clap(long, hide = true)] workspace: bool, - /// Exclude packages from the test - #[clap(long = "exclude")] + #[clap(long = "exclude", hide = true)] + #[doc(hidden)] excludes: Vec, - /// Activate all available features - #[clap(long = "all-features")] + #[clap(long = "all-features", hide = true)] all_features: bool, - /// Do not activate the `default` feature - #[clap(long = "no-default-features")] + #[clap(long = "no-default-features", hide = true)] no_default_features: bool, - /// Disables running cargo commands - #[clap(long = "no-cargo")] + #[clap(long = "no-cargo", hide = true)] no_cargo: bool, - /// TRIPLE - #[clap(long)] + #[clap(long, hide = true)] target: Option, - /// Directory for all generated artifacts - #[clap(long = "target-dir", default_value = "target/compliance")] + #[clap(long = "target-dir", default_value = "target/compliance", hide = true)] target_dir: String, - /// Path to Cargo.toml - #[clap(long = "manifest-path")] + #[clap(long = "manifest-path", hide = true)] manifest_path: Option, +} +#[derive(Debug, Parser)] +pub struct Project { /// Glob patterns for additional source files #[clap(long = "source-pattern")] - source_patterns: Vec, + source_patterns: Vec, /// Glob patterns for spec files #[clap(long = "spec-pattern")] - spec_patterns: Vec, + spec_patterns: Vec, /// Path to store the collection of spec files /// @@ -63,56 +59,58 @@ pub struct Project { /// argument to override the default location. #[clap(long = "spec-path")] pub spec_path: Option, + + // Includes a list of deprecated options to avoid breakage + #[clap(flatten)] + #[doc(hidden)] + deprecated: Deprecated, } impl Project { - pub fn sources(&self) -> Result, Error> { - let mut sources = HashSet::new(); - - for pattern in &self.source_patterns { - self.source_file(pattern, &mut sources)?; + pub fn load_sources(&self, sources: &mut Vec) { + for p in &self.source_patterns { + sources.push(Source { + pattern: p.glob.clone(), + comment_style: p.pattern.clone(), + root: duvet_core::env::current_dir().unwrap(), + default_type: Default::default(), + }) } + } - for pattern in &self.spec_patterns { - self.spec_file(pattern, &mut sources)?; + pub fn load_requirements(&self, requirements: &mut Vec) { + for req in &self.spec_patterns { + requirements.push(Requirement { + pattern: req.clone(), + root: duvet_core::env::current_dir().unwrap(), + }) } - - Ok(sources) } +} - fn source_file<'a>( - &self, - pattern: &'a str, - files: &mut HashSet>, - ) -> Result<(), Error> { - let (compliance_pattern, file_pattern) = if let Some(pattern) = pattern.strip_prefix('(') { +#[derive(Clone, Debug)] +struct SourcePattern { + pattern: Pattern, + glob: Glob, +} + +impl core::str::FromStr for SourcePattern { + type Err = crate::Error; + + fn from_str(s: &str) -> Result { + if let Some(pattern) = s.strip_prefix('(') { let mut parts = pattern.splitn(2, ')'); let pattern = parts.next().expect("invalid pattern"); let file_pattern = parts.next().expect("invalid pattern"); let pattern = Pattern::from_arg(pattern)?; + let glob = file_pattern.parse()?; - (pattern, file_pattern) + Ok(Self { pattern, glob }) } else { - (Pattern::default(), pattern) - }; - - for entry in glob(file_pattern)? { - files.insert(SourceFile::Text(compliance_pattern, entry?)); - } - - Ok(()) - } - - fn spec_file<'a>( - &self, - pattern: &'a str, - files: &mut HashSet>, - ) -> Result<(), Error> { - for entry in glob(pattern)? { - files.insert(SourceFile::Spec(entry?)); + let pattern = Pattern::default(); + let glob = s.parse()?; + Ok(Self { pattern, glob }) } - - Ok(()) } } diff --git a/duvet/src/report/ci.rs b/duvet/src/report/ci.rs index b4514f0c..5fbb60f9 100644 --- a/duvet/src/report/ci.rs +++ b/duvet/src/report/ci.rs @@ -30,7 +30,7 @@ pub fn enforce_source(report: &TargetReport) -> Result<(), anyhow::Error> { AnnotationType::Test => { tested_lines.insert(line); } - AnnotationType::Citation => { + AnnotationType::Implementation => { cited_lines.insert(line); } AnnotationType::Exception => { diff --git a/duvet/src/report/json.rs b/duvet/src/report/json.rs index b4de7fa6..b395e25c 100644 --- a/duvet/src/report/json.rs +++ b/duvet/src/report/json.rs @@ -135,7 +135,11 @@ pub fn report_writer( item!( arr, obj!(|obj| { - kv!(obj, s!("source"), s!(annotation.source.to_string_lossy())); + kv!( + obj, + s!("source"), + s!(annotation.relative_source().to_string_lossy()) + ); kv!(obj, s!("target_path"), s!(annotation.resolve_target_path())); if let Some(section) = annotation.target_section() { @@ -146,7 +150,7 @@ pub fn report_writer( kv!(obj, s!("line"), w!(annotation.anno_line)); } - if annotation.anno != AnnotationType::Citation { + if annotation.anno != AnnotationType::Implementation { kv!(obj, s!("type"), su!(annotation.anno)); } @@ -523,7 +527,7 @@ impl RefStatus { self.level = self.level.max(r.annotation.level); match r.annotation.anno { AnnotationType::Spec => self.spec = true, - AnnotationType::Citation => self.citation = true, + AnnotationType::Implementation => self.citation = true, AnnotationType::Implication => self.implication = true, AnnotationType::Test => self.test = true, AnnotationType::Exception => self.exception = true, diff --git a/duvet/src/report/lcov.rs b/duvet/src/report/lcov.rs index 99c12ea9..81a9333e 100644 --- a/duvet/src/report/lcov.rs +++ b/duvet/src/report/lcov.rs @@ -116,7 +116,7 @@ fn report_source(report: &TargetReport, output: &mut Output) -> R citation!(0); test!(1); } - AnnotationType::Citation => { + AnnotationType::Implementation => { citation!(1); test!(0); } diff --git a/duvet/src/report/mod.rs b/duvet/src/report/mod.rs index 32cb6717..b8a8cc94 100644 --- a/duvet/src/report/mod.rs +++ b/duvet/src/report/mod.rs @@ -3,14 +3,15 @@ use crate::{ annotation::{Annotation, AnnotationLevel, AnnotationSet, AnnotationSetExt}, + manifest::{Requirement, Source}, project::Project, specification::Specification, target::Target, - Error, }; use anyhow::anyhow; use clap::Parser; use core::fmt; +use duvet_core::Result; use rayon::prelude::*; use std::{ collections::{BTreeMap, BTreeSet, HashMap}, @@ -97,31 +98,27 @@ impl fmt::Display for ReportError<'_> { } impl Report { - pub async fn exec(&self) -> Result<(), Error> { - let project_sources = self.project.sources()?; + pub fn load_sources(&self, sources: &mut Vec) { + self.project.load_sources(sources) + } - let annotations: AnnotationSet = project_sources - .par_iter() - .flat_map(|source| { - // TODO gracefully handle error - source - .annotations() - .unwrap_or_else(|_| panic!("could not extract annotations from {:?}", source)) - }) - .collect(); + pub fn load_requirements(&self, requirements: &mut Vec) { + self.project.load_requirements(requirements) + } + + pub async fn exec(&self) -> Result<()> { + let annotations = crate::annotation::query().await?; let targets = annotations.targets()?; - let contents: HashMap<_, _> = targets - .par_iter() - .map(|target| { - let spec_path = self.project.spec_path.as_deref(); - let path = target.path.local(spec_path); - let contents = target.path.load(spec_path).unwrap(); - let contents = duvet_core::file::SourceFile::new(path, contents).unwrap(); - (target, contents) - }) - .collect(); + let mut contents = HashMap::new(); + for target in targets.iter() { + let spec_path = self.project.spec_path.as_deref(); + let path = target.path.local(spec_path); + let value = target.path.load(spec_path).await?; + let t = duvet_core::file::SourceFile::new(path, value).unwrap(); + contents.insert(target, t); + } let specifications: HashMap<_, _> = contents .par_iter() @@ -240,9 +237,7 @@ impl Report { eprintln!("{}", error); } - return Err(anyhow!( - "source errors were found. no reports were generated" - )); + return Err(anyhow!("source errors were found. no reports were generated").into()); } report diff --git a/duvet/src/report/stats.rs b/duvet/src/report/stats.rs index 0a8dc816..0bccc5d3 100644 --- a/duvet/src/report/stats.rs +++ b/duvet/src/report/stats.rs @@ -46,7 +46,7 @@ impl AnnotationStatistics { fn record(&mut self, reference: &Reference) { self.total.record(reference); match reference.annotation.anno { - AnnotationType::Citation => { + AnnotationType::Implementation => { self.citations.record(reference); } AnnotationType::Test => { diff --git a/duvet/src/report/status.rs b/duvet/src/report/status.rs index b7bfecf3..90f13e67 100644 --- a/duvet/src/report/status.rs +++ b/duvet/src/report/status.rs @@ -87,7 +87,7 @@ impl SpecReport { fn insert(&mut self, offset: usize, reference: &Reference) { match reference.annotation.anno { AnnotationType::Spec => &mut self.spec_offsets, - AnnotationType::Citation => &mut self.citation_offsets, + AnnotationType::Implementation => &mut self.citation_offsets, AnnotationType::Implication => &mut self.implication_offsets, AnnotationType::Test => &mut self.test_offsets, AnnotationType::Exception => &mut self.exception_offsets, diff --git a/duvet/src/source.rs b/duvet/src/source.rs index 4c6aafc5..2bb6a8fa 100644 --- a/duvet/src/source.rs +++ b/duvet/src/source.rs @@ -3,142 +3,166 @@ use crate::{ annotation::{Annotation, AnnotationLevel, AnnotationSet, AnnotationType}, - pattern::Pattern, + comment, specification::Format, - Error, }; -use anyhow::{anyhow, Context}; +use anyhow::anyhow; +use duvet_core::{diagnostic, path::Path, vfs, Result}; use serde::Deserialize; -use std::{collections::BTreeSet, path::PathBuf}; +use std::{collections::BTreeSet, sync::Arc}; -#[derive(Debug, PartialEq, PartialOrd, Ord, Eq, Hash)] -pub enum SourceFile<'a> { - Text(Pattern<'a>, PathBuf), - Spec(PathBuf), +#[derive(Clone, Debug, PartialEq, PartialOrd, Ord, Eq, Hash)] +pub enum SourceFile { + Text(comment::Pattern, Path, AnnotationType), + Spec(Path), } -impl SourceFile<'_> { - pub fn annotations(&self) -> Result { - let mut annotations = AnnotationSet::new(); +impl SourceFile { + pub async fn annotations(&self) -> (AnnotationSet, Vec) { match self { - Self::Text(pattern, file) => { - let text = std::fs::read_to_string(file)?; - pattern - .extract(&text, file, &mut annotations) - .with_context(|| file.display().to_string())?; - Ok(annotations) + Self::Text(pattern, file, default_type) => { + read_source(file.clone(), pattern.clone(), *default_type).await } - Self::Spec(file) => { - let text = std::fs::read_to_string(file)?; - let specs = toml::from_str::(&text)?; - for anno in specs.specs { - annotations.insert(anno.into_annotation(file.clone(), &specs.target)?); - } - for anno in specs.exceptions { - annotations.insert(anno.into_annotation(file.clone(), &specs.target)?); - } - for anno in specs.todos { - annotations.insert(anno.into_annotation(file.clone(), &specs.target)?); - } - Ok(annotations) + Self::Spec(file) => read_requirement(file.clone()).await, + } + } +} + +async fn read_source( + file: Path, + pattern: comment::Pattern, + default_type: AnnotationType, +) -> (AnnotationSet, Vec) { + let source = match vfs::read_string(&file).await { + Ok(specs) => specs, + Err(err) => { + return (Default::default(), vec![err]); + } + }; + + comment::extract(&source, &pattern, default_type) +} + +async fn read_requirement(file: Path) -> (AnnotationSet, Vec) { + async fn read_file(file: &Path) -> Result> { + let text = vfs::read_string(file).await?; + let specs = text.as_toml().await?; + Ok(specs) + } + + let specs = match read_file(&file).await { + Ok(specs) => specs, + Err(err) => { + return (Default::default(), vec![err]); + } + }; + + let mut annotations = AnnotationSet::default(); + let mut errors = vec![]; + + let annos = None + .into_iter() + .chain( + specs + .specs + .iter() + .map(|anno| anno.as_annotation(file.clone(), &specs.target)), + ) + .chain( + specs + .exceptions + .iter() + .map(|anno| anno.as_annotation(file.clone(), &specs.target)), + ) + .chain( + specs + .todos + .iter() + .map(|anno| anno.as_annotation(file.clone(), &specs.target)), + ); + + for anno in annos { + match anno { + Ok(anno) => { + annotations.insert(anno); + } + Err(err) => { + errors.push(err); } } } + + (annotations, errors) } #[derive(Deserialize)] #[serde(deny_unknown_fields)] -struct Specs<'a> { +struct Specs { target: Option, - #[serde(borrow)] #[serde(alias = "spec", default)] - specs: Vec>, + specs: Vec, - #[serde(borrow)] #[serde(alias = "exception", default)] - exceptions: Vec>, + exceptions: Vec, - #[serde(borrow)] #[serde(alias = "TODO", alias = "todo", default)] - todos: Vec>, + todos: Vec, } #[derive(Deserialize)] #[serde(deny_unknown_fields)] -struct Spec<'a> { +struct Spec { target: Option, - level: Option<&'a str>, - format: Option<&'a str>, - quote: &'a str, + level: Option, + format: Option, + quote: String, } -impl Spec<'_> { - fn into_annotation( - self, - source: PathBuf, - default_target: &Option, - ) -> Result { +impl Spec { + fn as_annotation(&self, source: Path, default_target: &Option) -> Result { Ok(Annotation { anno_line: 0, anno_column: 0, - item_line: 0, - item_column: 0, - path: String::new(), anno: AnnotationType::Spec, target: self .target + .clone() .or_else(|| default_target.as_ref().cloned()) .ok_or_else(|| anyhow!("missing target"))?, - quote: normalize_quote(self.quote), + quote: normalize_quote(&self.quote), comment: self.quote.to_string(), - manifest_dir: source.clone(), feature: Default::default(), tags: Default::default(), tracking_issue: Default::default(), source, - level: if let Some(level) = self.level { - level.parse()? - } else { - AnnotationLevel::Auto - }, - format: if let Some(format) = self.format { - format.parse()? - } else { - Format::Auto - }, + level: self.level.unwrap_or(AnnotationLevel::Auto), + format: self.format.unwrap_or(Format::Auto), }) } } #[derive(Deserialize)] #[serde(deny_unknown_fields)] -struct Exception<'a> { +struct Exception { target: Option, - quote: &'a str, + quote: String, reason: String, } -impl Exception<'_> { - fn into_annotation( - self, - source: PathBuf, - default_target: &Option, - ) -> Result { +impl Exception { + fn as_annotation(&self, source: Path, default_target: &Option) -> Result { Ok(Annotation { anno_line: 0, anno_column: 0, - item_line: 0, - item_column: 0, - path: String::new(), anno: AnnotationType::Exception, target: self .target + .clone() .or_else(|| default_target.as_ref().cloned()) .ok_or_else(|| anyhow!("missing target"))?, - quote: normalize_quote(self.quote), - comment: self.reason, - manifest_dir: source.clone(), + quote: normalize_quote(&self.quote), + comment: self.reason.clone(), feature: Default::default(), tags: Default::default(), tracking_issue: Default::default(), @@ -151,9 +175,9 @@ impl Exception<'_> { #[derive(Deserialize)] #[serde(deny_unknown_fields)] -struct Todo<'a> { +struct Todo { target: Option, - quote: &'a str, + quote: String, feature: Option, #[serde(alias = "tracking-issue")] tracking_issue: Option, @@ -162,30 +186,23 @@ struct Todo<'a> { tags: BTreeSet, } -impl Todo<'_> { - fn into_annotation( - self, - source: PathBuf, - default_target: &Option, - ) -> Result { +impl Todo { + fn as_annotation(&self, source: Path, default_target: &Option) -> Result { Ok(Annotation { anno_line: 0, anno_column: 0, - item_line: 0, - item_column: 0, - path: String::new(), anno: AnnotationType::Todo, target: self .target + .clone() .or_else(|| default_target.as_ref().cloned()) .ok_or_else(|| anyhow!("missing target"))?, - quote: normalize_quote(self.quote), - comment: self.reason.unwrap_or_default(), - manifest_dir: source.clone(), + quote: normalize_quote(&self.quote), + comment: self.reason.clone().unwrap_or_default(), source, - tags: self.tags, - feature: self.feature.unwrap_or_default(), - tracking_issue: self.tracking_issue.unwrap_or_default(), + tags: self.tags.clone(), + feature: self.feature.clone().unwrap_or_default(), + tracking_issue: self.tracking_issue.clone().unwrap_or_default(), level: AnnotationLevel::Auto, format: Format::Auto, }) diff --git a/duvet/src/specification/mod.rs b/duvet/src/specification/mod.rs index bfbc98c5..8e1520f6 100644 --- a/duvet/src/specification/mod.rs +++ b/duvet/src/specification/mod.rs @@ -10,6 +10,7 @@ use core::{ str::FromStr, }; use duvet_core::file::SourceFile; +use serde::Deserialize; use std::collections::HashMap; pub mod ietf; @@ -65,7 +66,8 @@ impl<'a> Specification<'a> { } } -#[derive(Clone, Copy, Debug, PartialEq, PartialOrd, Ord, Eq, Hash)] +#[derive(Clone, Copy, Debug, PartialEq, PartialOrd, Ord, Eq, Hash, Deserialize)] +#[serde(rename = "lowercase")] pub enum Format { Auto, Ietf, diff --git a/duvet/src/target.rs b/duvet/src/target.rs index efb57659..6ff1d202 100644 --- a/duvet/src/target.rs +++ b/duvet/src/target.rs @@ -3,10 +3,8 @@ use crate::{annotation::Annotation, specification::Format, Error}; use core::{fmt, str::FromStr}; -use std::{ - collections::HashSet, - path::{Path, PathBuf}, -}; +use duvet_core::{path::Path, vfs}; +use std::collections::HashSet; use url::Url; pub type TargetSet = HashSet; @@ -41,7 +39,7 @@ impl FromStr for Target { #[derive(Clone, Debug, PartialEq, PartialOrd, Ord, Eq, Hash)] pub enum TargetPath { Url(Url), - Path(PathBuf), + Path(Path), } impl fmt::Display for TargetPath { @@ -68,54 +66,52 @@ impl TargetPath { return Ok(Self::Url(url)); } - let path = anno.resolve_file(Path::new(&path))?; + let path = anno.resolve_file(path.into())?; Ok(Self::Path(path)) } - pub fn load(&self, spec_download_path: Option<&str>) -> Result { - let mut contents = match self { + pub async fn load(&self, spec_download_path: Option<&str>) -> Result { + let contents = match self { Self::Url(url) => { let path = self.local(spec_download_path); - if !path.exists() { - std::fs::create_dir_all(path.parent().unwrap())?; - - let canonical_url = Self::canonical_url(url.as_str()); - - reqwest::blocking::Client::builder() - .build()? - .get(canonical_url) - .header("user-agent", "https://crates.io/crates/cargo-compliance") - .header("accept", "text/plain") - .send()? - .error_for_status()? - .copy_to(&mut std::fs::File::create(&path)?)?; - } - std::fs::read_to_string(path)? + tokio::fs::create_dir_all(path.parent().unwrap()).await?; + let url = Self::canonical_url(url.as_str()); + duvet_core::http::get_cached_string(url, path).await? } - Self::Path(path) => std::fs::read_to_string(path)?, + Self::Path(path) => vfs::read_string(path).await?, }; + let mut contents = contents.to_string(); + // make sure the file has a newline if !contents.ends_with('\n') { contents.push('\n'); } + if contents.trim_start().starts_with("") { + return Err(anyhow::anyhow!( + "target {self} returned HTML instead of plaintext" + )); + } + Ok(contents) } - pub fn local(&self, spec_download_path: Option<&str>) -> PathBuf { + pub fn local(&self, spec_download_path: Option<&str>) -> Path { match self { Self::Url(url) => { let mut path = if let Some(path_to_spec) = spec_download_path { - PathBuf::from_str(path_to_spec).unwrap() + path_to_spec.into() } else { std::env::current_dir().unwrap() }; path.push("specs"); + let url = Self::canonical_url(url.as_str()); + let url = Url::parse(&url).unwrap(); path.push(url.host_str().expect("url should have host")); path.extend(url.path_segments().expect("url should have path")); path.set_extension("txt"); - path + path.into() } Self::Path(path) => path.clone(), } @@ -125,12 +121,20 @@ impl TargetPath { // rewrite some of the IETF links for convenience if let Some(rfc) = url.strip_prefix("https://tools.ietf.org/rfc/") { let rfc = rfc.trim_end_matches(".txt").trim_end_matches(".html"); - return format!("https://www.rfc-editor.org/rfc/{}.txt", rfc); + return format!("https://www.rfc-editor.org/rfc/{rfc}.txt"); + } + + if let Some(rfc) = url.strip_prefix("https://datatracker.ietf.org/doc/html/rfc") { + let rfc = rfc + .trim_end_matches(".txt") + .trim_end_matches(".html") + .trim_end_matches('/'); + return format!("https://www.rfc-editor.org/rfc/rfc{rfc}.txt"); } if url.starts_with("https://www.rfc-editor.org/rfc/") { let rfc = url.trim_end_matches(".txt").trim_end_matches(".html"); - return format!("{}.txt", rfc); + return format!("{rfc}.txt"); } url.to_owned() @@ -147,7 +151,6 @@ impl FromStr for TargetPath { return Ok(Self::Url(url)); } - let path = PathBuf::from(path); - Ok(Self::Path(path)) + Ok(Self::Path(path.into())) } } diff --git a/duvet/src/tests.rs b/duvet/src/tests.rs index 45fea041..4ac29603 100644 --- a/duvet/src/tests.rs +++ b/duvet/src/tests.rs @@ -1,17 +1,11 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -use crate::{Arguments, Error}; -use clap::Parser; +use crate::Result; use insta::assert_json_snapshot; -use std::{ - ffi::OsString, - path::{Path, PathBuf}, -}; +use std::path::{Path, PathBuf}; use tempfile::TempDir; -type Result = core::result::Result; - struct Env { dir: TempDir, } @@ -51,15 +45,16 @@ impl Env { async fn exec(&self, args: I) -> Result where I: IntoIterator, - I::Item: Into + Clone, + I::Item: Into + Clone, { - Arguments::try_parse_from( + duvet_core::env::set_args( ["duvet".into()] .into_iter() - .chain(args.into_iter().map(|v| v.into())), - )? - .exec() - .await?; + .chain(args.into_iter().map(|v| v.into())) + .collect(), + ); + duvet_core::env::set_current_dir(self.dir.path().into()); + crate::run().await?; Ok(()) } }