From 88284e265b09483c3881fbc9520bb0879075de0c Mon Sep 17 00:00:00 2001 From: Stefnotch Date: Wed, 8 May 2024 12:07:01 +0200 Subject: [PATCH] Remove replaced code --- src/compose/compose_parser.rs | 334 +++++ src/compose/composer.rs | 1236 ++++++++++++++++++ src/compose/error.rs | 9 +- src/compose/mod.rs | 1536 +--------------------- src/compose/old_parse_imports.rs | 189 +++ src/compose/old_preprocess.rs | 1085 +++++++++++++++ src/compose/parse_imports.rs | 320 ----- src/compose/preprocess.rs | 2099 ++++++++++-------------------- src/compose/preprocess1.rs | 722 ---------- src/compose/test.rs | 41 +- 10 files changed, 3621 insertions(+), 3950 deletions(-) create mode 100644 src/compose/compose_parser.rs create mode 100644 src/compose/composer.rs create mode 100644 src/compose/old_parse_imports.rs create mode 100644 src/compose/old_preprocess.rs delete mode 100644 src/compose/parse_imports.rs delete mode 100644 src/compose/preprocess1.rs diff --git a/src/compose/compose_parser.rs b/src/compose/compose_parser.rs new file mode 100644 index 0000000..9a9209a --- /dev/null +++ b/src/compose/compose_parser.rs @@ -0,0 +1,334 @@ +use std::collections::HashMap; + +use winnow::{ + combinator::{alt, eof, fail, opt, peek}, + error::StrContext, + token::any, + PResult, Parser, Stateful, +}; + +use crate::compose::preprocess::ResolvedIfOp; + +use super::{ + composer::{ + ComposableModuleDefinition, Composer, ComposerError, ComposerErrorInner, ErrSource, + ImportDefWithOffset, ImportDefinition, ShaderDefValue, + }, + preprocess::{IfDefDirective, IfOpDirective, PreprocessorPart}, +}; + +type Stream<'a> = Stateful<&'a [PreprocessorPart], PreprocessingState<'a>>; + +pub(super) fn preprocess<'a>( + composer: &'a Composer, + module: &'a ComposableModuleDefinition, + shader_defs: &'a HashMap, +) -> Result { + let state = PreprocessingState { + composer, + module, + shader_defs, + }; + let stream = Stream { + input: &module.parsed.parts, + state, + }; + + top_level_preprocessor + .parse(stream) + .map_err(|inner| ComposerError { + inner: ComposerErrorInner::PreprocessorError( + vec![inner.into_inner().to_string()].into(), + ), + source: ErrSource::Module { + name: module.name.0.to_owned(), + offset: 0, + defs: shader_defs.clone(), + }, + }) +} + +#[derive(Debug, Clone)] +struct PreprocessingState<'a> { + // Only needs to know which other modules are defined so that we can resolve imports + composer: &'a Composer, + module: &'a ComposableModuleDefinition, + shader_defs: &'a HashMap, +} + +// TODO: Check what the error messages are like (do they include a location?) +fn top_level_preprocessor<'a>(input: &mut Stream<'a>) -> PResult { + let mut source = String::new(); + let mut imports = Vec::new(); + loop { + if let Some(result) = opt(if_statement).parse_next(input)? { + source += &result; + continue; + } + + let token = match opt(any).parse_next(input)? { + Some(v) => v, + None => break, + }; + match token { + PreprocessorPart::Version(_) => { /* ignore */ } + // TODO: Can I get away without the context? + PreprocessorPart::If(_) => { + return fail + .context(StrContext::Label("unexpected #ifdef")) + .parse_next(input)?; + } + PreprocessorPart::IfOp(_) => { + return fail + .context(StrContext::Label("unexpected #if")) + .parse_next(input)?; + } + PreprocessorPart::Else(_) => { + return fail + .context(StrContext::Label("unmatched else")) + .parse_next(input)?; + } + PreprocessorPart::EndIf(_) => { + return fail + .context(StrContext::Label("unmatched end-if")) + .parse_next(input)?; + } + PreprocessorPart::UseDefine(def) => { + let define = def.name(&input.state.module.source).unwrap(); + let value = input.state.shader_defs.get(define).unwrap(); + source += &value.value_as_string(); + } + PreprocessorPart::DefineShaderDef(_) => { /* ignore */ } + PreprocessorPart::DefineImportPath(_) => { /* ignore */ } + PreprocessorPart::Import(directive) => { + for import in directive + .get_import(&input.state.module.source) + .unwrap() + .into_iter() + { + let module_name = input.state.composer.get_imported_module(&import).unwrap(); // TODO: Error handling + let item = if &import.path == &module_name.0 { + module_name.0.clone() + } else { + import.path.rsplit_once("::").unwrap().0.to_owned() + }; + imports.push(ImportDefWithOffset { + definition: ImportDefinition { + module: module_name.clone(), + item, + }, + offset: import.offset, + }) + } + } + PreprocessorPart::UnknownDirective(_) => { + return fail + .context(StrContext::Label("unknown directive")) + .parse_next(input)?; + } + PreprocessorPart::Text(range) => { + source += &input.state.module.source[range.clone()]; + } + } + } + Ok(PreprocessOutput { source, imports }) +} + +#[derive(Debug, Clone)] +enum IfOrIfOp { + If(IfDefDirective), + IfOp(IfOpDirective), +} + +#[derive(Debug)] +enum IfEnd { + ElseIf(IfDefDirective), + ElseIfOp(IfOpDirective), + Else, + EndIf, + Eof, +} + +fn if_statement<'a>(input: &mut Stream<'a>) -> PResult { + let start = any + .verify_map(|token| match token { + PreprocessorPart::If(if_def) if !if_def.is_else_if => Some(IfOrIfOp::If(if_def)), + PreprocessorPart::IfOp(if_def) if !if_def.is_else_if => Some(IfOrIfOp::IfOp(if_def)), + _ => None, + }) + .parse_next(input)?; + + let mut source = String::new(); + let is_true_branch = match start { + IfOrIfOp::If(if_def) => { + let define = if_def.name(&input.state.module.source).unwrap(); + let mut result = input.state.shader_defs.contains_key(define); + if if_def.is_not { + result = !result; + } + result + } + IfOrIfOp::IfOp(if_op) => { + let ResolvedIfOp { + name, op, value, .. + } = if_op.resolve(&input.state.module.source).unwrap(); + let name_value = input.state.shader_defs.get(name).unwrap(); + act_on(name_value.value_as_string().as_str(), value, op).unwrap() // TODO: Error handling + } + }; + + if is_true_branch { + let (source_add, _) = block.parse_next(input)?; + source += &source_add; + // Skip all the next blocks until we reach the end + loop { + let next_block = block_end.parse_next(input)?.unwrap(); + match next_block { + IfEnd::ElseIf(_) | IfEnd::ElseIfOp(_) | IfEnd::Else => { + let _ = skip_block.parse_next(input)?; + } + IfEnd::EndIf => break, + IfEnd::Eof => fail + .context(StrContext::Label("expected #endif")) + .parse_next(input)?, + } + } + } else { + let peek_next_block = skip_block.parse_next(input)?; + // And handle the various else cases + match peek_next_block { + IfEnd::ElseIf(_) => source += &if_statement.parse_next(input)?, + IfEnd::ElseIfOp(_) => source += &if_statement.parse_next(input)?, + IfEnd::Else => { + let _ = block_end.parse_next(input)?; + let (source_add, peeked_block_end) = block.parse_next(input)?; + source += &source_add; + let _ = block_end.parse_next(input)?; + if matches!(peeked_block_end, IfEnd::EndIf) { + return fail + .context(StrContext::Label("else block must end with #endif")) + .parse_next(input)?; + } + } + IfEnd::EndIf => { /* done */ } + IfEnd::Eof => fail + .context(StrContext::Label("expected #endif")) + .parse_next(input)?, + }; + } + Ok(source) +} + +fn block_end<'a>(input: &mut Stream<'a>) -> PResult> { + alt(( + eof.map(|_| Some(IfEnd::Eof)), + any.map(|token| match token { + PreprocessorPart::If(if_def) if if_def.is_else_if => Some(IfEnd::ElseIf(if_def)), + PreprocessorPart::IfOp(if_op) if if_op.is_else_if => Some(IfEnd::ElseIfOp(if_op)), + PreprocessorPart::Else(_) => Some(IfEnd::Else), + PreprocessorPart::EndIf(_) => Some(IfEnd::EndIf), + _ => None, + }), + )) + .parse_next(input) +} + +fn block<'a>(input: &mut Stream<'a>) -> PResult<(String, IfEnd)> { + let mut source = String::new(); + loop { + if let Some(block_end) = peek(block_end).parse_next(input)? { + return Ok((source, block_end)); + } + + if let Some(result) = opt(if_statement).parse_next(input)? { + source += &result; + continue; + } + + let token = opt(any).parse_next(input)?.unwrap(); + match token { + PreprocessorPart::Version(_) => { + return fail + .context(StrContext::Label("#version must be at the top of the file")) + .parse_next(input)?; + } + PreprocessorPart::If(_) => { + return fail + .context(StrContext::Label("unexpected #ifdef")) + .parse_next(input)?; + } + PreprocessorPart::IfOp(_) => { + return fail + .context(StrContext::Label("unexpected #if")) + .parse_next(input)?; + } + PreprocessorPart::Else(_) => { + return fail + .context(StrContext::Label("unmatched else")) + .parse_next(input)?; + } + PreprocessorPart::EndIf(_) => { + return fail + .context(StrContext::Label("unmatched end-if")) + .parse_next(input)?; + } + PreprocessorPart::UseDefine(def) => { + let define = def.name(&input.state.module.source).unwrap(); + let value = input.state.shader_defs.get(define).unwrap(); + source += &value.value_as_string(); + } + PreprocessorPart::DefineShaderDef(_) => { + return fail + .context(StrContext::Label("#define must be at the top of the file")) + .parse_next(input)?; + } + PreprocessorPart::DefineImportPath(_) => { + return fail + .context(StrContext::Label( + "#define_import_path must be at the top of the file", + )) + .parse_next(input)?; + } + PreprocessorPart::Import(_) => { + return fail + .context(StrContext::Label("only top-level imports are allowed")) + .parse_next(input)?; + } + PreprocessorPart::UnknownDirective(_) => { + return fail + .context(StrContext::Label("unknown directive")) + .parse_next(input)?; + } + PreprocessorPart::Text(range) => { + source += &input.state.module.source[range.clone()]; + } + } + } +} + +fn skip_block<'a>(input: &mut Stream<'a>) -> PResult { + loop { + if let Some(if_end) = peek(block_end).parse_next(input)? { + return Ok(if_end); + } + any.parse_next(input)?; + } +} + +fn act_on(a: &str, b: &str, op: &str) -> Result { + match op { + "==" => Ok(a == b), + "!=" => Ok(a != b), + ">" => Ok(a > b), + ">=" => Ok(a >= b), + "<" => Ok(a < b), + "<=" => Ok(a <= b), + _ => Err(()), + } +} + +#[derive(Debug)] +pub struct PreprocessOutput { + pub source: String, + pub imports: Vec, +} diff --git a/src/compose/composer.rs b/src/compose/composer.rs new file mode 100644 index 0000000..d14379d --- /dev/null +++ b/src/compose/composer.rs @@ -0,0 +1,1236 @@ +use indexmap::IndexMap; + +use naga::EntryPoint; +use regex::Regex; +use std::{ + collections::{hash_map::Entry, BTreeMap, HashMap, HashSet}, + ops::Range, +}; +use tracing::{debug, trace}; +use winnow::RecoverableParser; + +use crate::{ + compose::preprocess::{DefineImportPath, PreprocessorPart}, + derive::DerivedModule, + redirect::Redirector, +}; + +pub use super::error::{ComposerError, ComposerErrorInner, ErrSource}; +use super::{ + compose_parser::PreprocessOutput, + error::StringsWithNewlines, + preprocess::{self, FlattenedImport, UseDefineDirective}, + ComposableModuleDescriptor, ShaderLanguage, +}; + +#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] +pub enum ShaderDefValue { + Bool(bool), + Int(i32), + UInt(u32), +} + +impl Default for ShaderDefValue { + fn default() -> Self { + ShaderDefValue::Bool(true) + } +} + +impl ShaderDefValue { + pub(super) fn value_as_string(&self) -> String { + match self { + ShaderDefValue::Bool(val) => val.to_string(), + ShaderDefValue::Int(val) => val.to_string(), + ShaderDefValue::UInt(val) => val.to_string(), + } + } + + pub(super) fn parse(value: &str) -> Self { + if let Ok(val) = value.parse::() { + ShaderDefValue::UInt(val) + } else if let Ok(val) = value.parse::() { + ShaderDefValue::Int(val) + } else if let Ok(val) = value.parse::() { + ShaderDefValue::Bool(val) + } else { + // TODO: Better error handling + ShaderDefValue::Bool(false) // this error will get picked up when we fully preprocess the module? + } + } +} + +#[derive(Clone, PartialEq, Eq, Hash, Debug, Default)] +pub struct OwnedShaderDefs(BTreeMap); + +#[derive(Clone, PartialEq, Eq, Hash, Debug)] +struct ModuleKey(OwnedShaderDefs); + +impl ModuleKey { + fn from_members(key: &HashMap, universe: &[String]) -> Self { + let mut acc = OwnedShaderDefs::default(); + for item in universe { + if let Some(value) = key.get(item) { + acc.0.insert(item.to_owned(), *value); + } + } + ModuleKey(acc) + } +} + +// a module built with a specific set of shader_defs +#[derive(Default, Debug)] +pub struct ComposableModule { + // module decoration, prefixed to all items from this module in the final source + pub decorated_name: String, + // module names required as imports, optionally with a list of items to import + pub imports: Vec, + // types exported + pub owned_types: HashSet, + // constants exported + pub owned_constants: HashSet, + // vars exported + pub owned_vars: HashSet, + // functions exported + pub owned_functions: HashSet, + // local functions that can be overridden + pub virtual_functions: HashSet, + // overriding functions defined in this module + // target function -> Vec + pub override_functions: IndexMap>, + // naga module, built against headers for any imports + module_ir: naga::Module, + // headers in different shader languages, used for building modules/shaders that import this module + // headers contain types, constants, global vars and empty function definitions - + // just enough to convert source strings that want to import this module into naga IR + // headers: HashMap, + header_ir: naga::Module, + // character offset of the start of the owned module string + start_offset: usize, +} + +// data used to build a ComposableModule +#[derive(Debug)] +pub struct ComposableModuleDefinition { + pub name: ModuleName, + // shader text (with auto bindings replaced - we do this on module add as we only want to do it once to avoid burning slots) + pub source: String, + // language + pub language: ShaderLanguage, + // source path for error display + pub file_path: String, + // shader defs that have been defined by this module + pub shader_defs: HashMap, + pub(super) parsed: preprocess::Preprocessed, + // list of shader_defs that can affect this module + pub(super) used_defs: Vec, + // full list of possible imports (regardless of shader_def configuration) + pub(super) shader_imports: Vec, + // additional imports to add (as though they were included in the source after any other imports) + pub(super) additional_imports: Vec<(ModuleName, ImportDefinition)>, + /// Which alias maps to which function/struct/module + pub(super) alias_to_path: HashMap, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct ModuleName(pub(super) String); +impl ModuleName { + pub fn new>(name: S) -> Self { + Self(name.into()) + } +} +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ImportDefinition { + pub module: ModuleName, + pub item: String, +} + +#[derive(Debug, Clone)] +pub struct ImportDefWithOffset { + pub(super) definition: ImportDefinition, + pub(super) offset: usize, +} + +/// module composer. +/// stores any modules that can be imported into a shader +/// and builds the final shader +#[derive(Debug)] +pub struct Composer { + pub validate: bool, + pub module_sets: HashMap, + pub capabilities: naga::valid::Capabilities, + check_decoration_regex: Regex, + undecorate_regex: Regex, + undecorate_override_regex: Regex, +} + +// shift for module index +// 21 gives +// max size for shader of 2m characters +// max 2048 modules +const SPAN_SHIFT: usize = 21; + +impl Default for Composer { + fn default() -> Self { + Self { + validate: true, + capabilities: Default::default(), + module_sets: Default::default(), + check_decoration_regex: Regex::new( + format!( + "({}|{})", + regex_syntax::escape(DECORATION_PRE), + regex_syntax::escape(DECORATION_OVERRIDE_PRE) + ) + .as_str(), + ) + .unwrap(), + undecorate_regex: Regex::new( + format!( + r"(\x1B\[\d+\w)?([\w\d_]+){}([A-Z0-9]*){}", + regex_syntax::escape(DECORATION_PRE), + regex_syntax::escape(DECORATION_POST) + ) + .as_str(), + ) + .unwrap(), + undecorate_override_regex: Regex::new( + format!( + "{}([A-Z0-9]*){}", + regex_syntax::escape(DECORATION_OVERRIDE_PRE), + regex_syntax::escape(DECORATION_POST) + ) + .as_str(), + ) + .unwrap(), + } + } +} + +// TODO: Change the mangling scheme +const DECORATION_PRE: &str = "X_naga_oil_mod_X"; +const DECORATION_POST: &str = "X"; + +// must be same length as DECORATION_PRE for spans to work +const DECORATION_OVERRIDE_PRE: &str = "X_naga_oil_vrt_X"; + +struct IrBuildResult { + module: naga::Module, + start_offset: usize, + override_functions: IndexMap>, +} + +impl Composer { + // TODO: Change the mangling scheme + pub fn decorated_name(module_name: Option<&str>, item_name: &str) -> String { + match module_name { + Some(module_name) => format!("{}{}", item_name, Self::decorate(module_name)), + None => item_name.to_owned(), + } + } + + pub(super) fn decorate(module: &str) -> String { + let encoded = data_encoding::BASE32_NOPAD.encode(module.as_bytes()); + format!("{DECORATION_PRE}{encoded}{DECORATION_POST}") + } + + fn decode(from: &str) -> String { + String::from_utf8(data_encoding::BASE32_NOPAD.decode(from.as_bytes()).unwrap()).unwrap() + } + + pub(super) fn undecorate(&self, string: &str) -> String { + let undecor = self + .undecorate_regex + .replace_all(string, |caps: ®ex::Captures| { + format!( + "{}{}::{}", + caps.get(1).map(|cc| cc.as_str()).unwrap_or(""), + Self::decode(caps.get(3).unwrap().as_str()), + caps.get(2).unwrap().as_str() + ) + }); + + let undecor = + self.undecorate_override_regex + .replace_all(&undecor, |caps: ®ex::Captures| { + format!( + "override fn {}::", + Self::decode(caps.get(1).unwrap().as_str()) + ) + }); + + undecor.to_string() + } + + fn naga_to_string( + &self, + naga_module: &mut naga::Module, + language: ShaderLanguage, + #[allow(unused)] header_for: &str, // Only used when GLSL is enabled + ) -> Result { + // TODO: cache headers again + let info = + naga::valid::Validator::new(naga::valid::ValidationFlags::all(), self.capabilities) + .validate(naga_module) + .map_err(ComposerErrorInner::HeaderValidationError)?; + + match language { + ShaderLanguage::Wgsl => naga::back::wgsl::write_string( + naga_module, + &info, + naga::back::wgsl::WriterFlags::EXPLICIT_TYPES, + ) + .map_err(ComposerErrorInner::WgslBackError), + #[cfg(feature = "glsl")] + ShaderLanguage::Glsl => { + let vec4 = naga_module.types.insert( + naga::Type { + name: None, + inner: naga::TypeInner::Vector { + size: naga::VectorSize::Quad, + scalar: naga::Scalar::F32, + }, + }, + naga::Span::UNDEFINED, + ); + // add a dummy entry point for glsl headers + let dummy_entry_point = "dummy_module_entry_point".to_owned(); + let func = naga::Function { + name: Some(dummy_entry_point.clone()), + arguments: Default::default(), + result: Some(naga::FunctionResult { + ty: vec4, + binding: Some(naga::Binding::BuiltIn(naga::BuiltIn::Position { + invariant: false, + })), + }), + local_variables: Default::default(), + expressions: Default::default(), + named_expressions: Default::default(), + body: Default::default(), + }; + let ep = EntryPoint { + name: dummy_entry_point.clone(), + stage: naga::ShaderStage::Vertex, + function: func, + early_depth_test: None, + workgroup_size: [0, 0, 0], + }; + + naga_module.entry_points.push(ep); + + let info = naga::valid::Validator::new( + naga::valid::ValidationFlags::all(), + self.capabilities, + ) + .validate(naga_module) + .map_err(ComposerErrorInner::HeaderValidationError)?; + + let mut string = String::new(); + let options = naga::back::glsl::Options { + version: naga::back::glsl::Version::Desktop(450), + writer_flags: naga::back::glsl::WriterFlags::INCLUDE_UNUSED_ITEMS, + ..Default::default() + }; + let pipeline_options = naga::back::glsl::PipelineOptions { + shader_stage: naga::ShaderStage::Vertex, + entry_point: dummy_entry_point, + multiview: None, + }; + let mut writer = naga::back::glsl::Writer::new( + &mut string, + naga_module, + &info, + &options, + &pipeline_options, + naga::proc::BoundsCheckPolicies::default(), + ) + .map_err(ComposerErrorInner::GlslBackError)?; + + writer.write().map_err(ComposerErrorInner::GlslBackError)?; + + // strip version decl and main() impl + let lines: Vec<_> = string.lines().collect(); + let string = lines[1..lines.len() - 3].join("\n"); + trace!("glsl header for {}:\n\"\n{:?}\n\"", header_for, string); + + Ok(string) + } + } + } + + // build naga module for a given shader_def configuration. builds a minimal self-contained module built against headers for imports + fn create_module_ir( + &self, + name: &str, + source: String, + language: ShaderLanguage, + imports: &[ImportDefinition], + shader_defs: &HashMap, + ) -> Result { + debug!("creating IR for {} with defs: {:?}", name, shader_defs); + + let mut module_string = match language { + ShaderLanguage::Wgsl => String::new(), + #[cfg(feature = "glsl")] + ShaderLanguage::Glsl => String::from("#version 450\n"), + }; + + let mut override_functions: IndexMap> = IndexMap::default(); + let mut added_imports: HashSet = HashSet::new(); + let mut header_module = DerivedModule::default(); + + for import in imports { + if added_imports.contains(&import.module) { + continue; + } + // add to header module + self.add_import( + &mut header_module, + import, + shader_defs, + true, + &mut added_imports, + ); + + // // we must have ensured these exist with Composer::ensure_imports() + trace!("looking for {}", import.module); + let import_module_set = self.module_sets.get(&import.module).unwrap(); + trace!("with defs {:?}", shader_defs); + let module = import_module_set.get_module(shader_defs).unwrap(); + trace!("ok"); + + // gather overrides + if !module.override_functions.is_empty() { + for (original, replacements) in &module.override_functions { + match override_functions.entry(original.clone()) { + indexmap::map::Entry::Occupied(o) => { + let existing = o.into_mut(); + let new_replacements: Vec<_> = replacements + .iter() + .filter(|rep| !existing.contains(rep)) + .cloned() + .collect(); + existing.extend(new_replacements); + } + indexmap::map::Entry::Vacant(v) => { + v.insert(replacements.clone()); + } + } + } + } + } + + let composed_header = self + .naga_to_string(&mut header_module.into(), language, name) + .map_err(|inner| ComposerError { + inner, + source: ErrSource::Module { + name: name.to_owned(), + offset: 0, + defs: shader_defs.clone(), + }, + })?; + module_string.push_str(&composed_header); + + let start_offset = module_string.len(); + + module_string.push_str(&source); + + trace!( + "parsing {}: {}, header len {}, total len {}", + name, + module_string, + start_offset, + module_string.len() + ); + let module = match language { + ShaderLanguage::Wgsl => naga::front::wgsl::parse_str(&module_string).map_err(|e| { + debug!("full err'd source file: \n---\n{}\n---", module_string); + ComposerError { + inner: ComposerErrorInner::WgslParseError(e), + source: ErrSource::Module { + name: name.to_owned(), + offset: start_offset, + defs: shader_defs.clone(), + }, + } + })?, + #[cfg(feature = "glsl")] + ShaderLanguage::Glsl => naga::front::glsl::Frontend::default() + .parse( + &naga::front::glsl::Options { + stage: naga::ShaderStage::Vertex, + defines: Default::default(), + }, + &module_string, + ) + .map_err(|e| { + debug!("full err'd source file: \n---\n{}\n---", module_string); + ComposerError { + inner: ComposerErrorInner::GlslParseError(e), + source: ErrSource::Module { + name: name.to_owned(), + offset: start_offset, + defs: shader_defs.clone(), + }, + } + })?, + }; + + Ok(IrBuildResult { + module, + start_offset, + override_functions, + }) + } + + // check that identifiers exported by a module do not get modified in string export + fn validate_identifiers( + source_ir: &naga::Module, + lang: ShaderLanguage, + header: &str, + module_decoration: &str, + owned_types: &HashSet, + ) -> Result<(), ComposerErrorInner> { + // TODO: remove this once glsl front support is complete + #[cfg(feature = "glsl")] + if lang == ShaderLanguage::Glsl { + return Ok(()); + } + + let recompiled = match lang { + ShaderLanguage::Wgsl => naga::front::wgsl::parse_str(header).unwrap(), + #[cfg(feature = "glsl")] + ShaderLanguage::Glsl => naga::front::glsl::Frontend::default() + .parse( + &naga::front::glsl::Options { + stage: naga::ShaderStage::Vertex, + defines: Default::default(), + }, + &format!("{}\n{}", header, "void main() {}"), + ) + .map_err(|e| { + debug!("full err'd source file: \n---\n{header}\n---"); + ComposerErrorInner::GlslParseError(e) + })?, + }; + + let recompiled_types: IndexMap<_, _> = recompiled + .types + .iter() + .flat_map(|(h, ty)| ty.name.as_deref().map(|name| (name, h))) + .collect(); + for (h, ty) in source_ir.types.iter() { + if let Some(name) = &ty.name { + let decorated_type_name = format!("{name}{module_decoration}"); + if !owned_types.contains(&decorated_type_name) { + continue; + } + match recompiled_types.get(decorated_type_name.as_str()) { + Some(recompiled_h) => { + if let naga::TypeInner::Struct { members, .. } = &ty.inner { + let recompiled_ty = recompiled.types.get_handle(*recompiled_h).unwrap(); + let naga::TypeInner::Struct { + members: recompiled_members, + .. + } = &recompiled_ty.inner + else { + panic!(); + }; + for (member, recompiled_member) in + members.iter().zip(recompiled_members) + { + if member.name != recompiled_member.name { + return Err(ComposerErrorInner::InvalidIdentifier { + original: member.name.clone().unwrap_or_default(), + at: source_ir.types.get_span(h), + }); + } + } + } + } + None => { + return Err(ComposerErrorInner::InvalidIdentifier { + original: name.clone(), + at: source_ir.types.get_span(h), + }) + } + } + } + } + + let recompiled_consts: HashSet<_> = recompiled + .constants + .iter() + .flat_map(|(_, c)| c.name.as_deref()) + .filter(|name| name.ends_with(module_decoration)) + .collect(); + for (h, c) in source_ir.constants.iter() { + if let Some(name) = &c.name { + if name.ends_with(module_decoration) && !recompiled_consts.contains(name.as_str()) { + return Err(ComposerErrorInner::InvalidIdentifier { + original: name.clone(), + at: source_ir.constants.get_span(h), + }); + } + } + } + + let recompiled_globals: HashSet<_> = recompiled + .global_variables + .iter() + .flat_map(|(_, c)| c.name.as_deref()) + .filter(|name| name.ends_with(module_decoration)) + .collect(); + for (h, gv) in source_ir.global_variables.iter() { + if let Some(name) = &gv.name { + if name.ends_with(module_decoration) && !recompiled_globals.contains(name.as_str()) + { + return Err(ComposerErrorInner::InvalidIdentifier { + original: name.clone(), + at: source_ir.global_variables.get_span(h), + }); + } + } + } + + let recompiled_fns: HashSet<_> = recompiled + .functions + .iter() + .flat_map(|(_, c)| c.name.as_deref()) + .filter(|name| name.ends_with(module_decoration)) + .collect(); + for (h, f) in source_ir.functions.iter() { + if let Some(name) = &f.name { + if name.ends_with(module_decoration) && !recompiled_fns.contains(name.as_str()) { + return Err(ComposerErrorInner::InvalidIdentifier { + original: name.clone(), + at: source_ir.functions.get_span(h), + }); + } + } + } + + Ok(()) + } + + // build a ComposableModule from a ComposableModuleDefinition, for a given set of shader defs + // - build the naga IR (against headers) + // - record any types/vars/constants/functions that are defined within this module + // - build headers for each supported language + #[allow(clippy::too_many_arguments)] + pub fn create_composable_module( + &mut self, + module_definition: &ComposableModuleDefinition, + module_decoration: String, + shader_defs: &HashMap, + create_headers: bool, + demote_entrypoints: bool, + source: &str, + imports: Vec, + ) -> Result { + let mut imports: Vec<_> = imports + .into_iter() + .map(|import_with_offset| import_with_offset.definition) + .collect(); + imports.extend(module_definition.additional_imports.to_vec()); + + trace!( + "create composable module {}: source len {}", + module_definition.name.0, + source.len() + ); + + trace!( + "create composable module {}: source len {}", + module_definition.name.0, + source.len() + ); + + let IrBuildResult { + module: mut source_ir, + start_offset, + mut override_functions, + } = self.create_module_ir( + &module_definition.name.0, + source, + module_definition.language, + &imports, + shader_defs, + )?; + + // from here on errors need to be reported using the modified source with start_offset + let wrap_err = |inner: ComposerErrorInner| -> ComposerError { + ComposerError { + inner, + source: ErrSource::Module { + name: module_definition.name.0.to_owned(), + offset: start_offset, + defs: shader_defs.clone(), + }, + } + }; + + // rename and record owned items (except types which can't be mutably accessed) + let mut owned_constants = IndexMap::new(); + for (h, c) in source_ir.constants.iter_mut() { + if let Some(name) = c.name.as_mut() { + if !name.contains(DECORATION_PRE) { + *name = format!("{name}{module_decoration}"); + owned_constants.insert(name.clone(), h); + } + } + } + + let mut owned_vars = IndexMap::new(); + for (h, gv) in source_ir.global_variables.iter_mut() { + if let Some(name) = gv.name.as_mut() { + if !name.contains(DECORATION_PRE) { + *name = format!("{name}{module_decoration}"); + + owned_vars.insert(name.clone(), h); + } + } + } + + let mut owned_functions = IndexMap::new(); + for (h_f, f) in source_ir.functions.iter_mut() { + if let Some(name) = f.name.as_mut() { + if !name.contains(DECORATION_PRE) { + *name = format!("{name}{module_decoration}"); + + // create dummy header function + let header_function = naga::Function { + name: Some(name.clone()), + arguments: f.arguments.to_vec(), + result: f.result.clone(), + local_variables: Default::default(), + expressions: Default::default(), + named_expressions: Default::default(), + body: Default::default(), + }; + + // record owned function + owned_functions.insert(name.clone(), (Some(h_f), header_function)); + } + } + } + + if demote_entrypoints { + // make normal functions out of the source entry points + for ep in &mut source_ir.entry_points { + ep.function.name = Some(format!( + "{}{}", + ep.function.name.as_deref().unwrap_or("main"), + module_decoration, + )); + let header_function = naga::Function { + name: ep.function.name.clone(), + arguments: ep + .function + .arguments + .iter() + .cloned() + .map(|arg| naga::FunctionArgument { + name: arg.name, + ty: arg.ty, + binding: None, + }) + .collect(), + result: ep.function.result.clone().map(|res| naga::FunctionResult { + ty: res.ty, + binding: None, + }), + local_variables: Default::default(), + expressions: Default::default(), + named_expressions: Default::default(), + body: Default::default(), + }; + + owned_functions.insert(ep.function.name.clone().unwrap(), (None, header_function)); + } + }; + + let mut module_builder = DerivedModule::default(); + let mut header_builder = DerivedModule::default(); + module_builder.set_shader_source(&source_ir, 0); + header_builder.set_shader_source(&source_ir, 0); + + let mut owned_types = HashSet::new(); + for (h, ty) in source_ir.types.iter() { + if let Some(name) = &ty.name { + // we need to exclude autogenerated struct names, i.e. those that begin with "__" + // "__" is a reserved prefix for naga so user variables cannot use it. + if !name.contains(DECORATION_PRE) && !name.starts_with("__") { + let name = format!("{name}{module_decoration}"); + owned_types.insert(name.clone()); + // copy and rename types + module_builder.rename_type(&h, Some(name.clone())); + header_builder.rename_type(&h, Some(name)); + continue; + } + } + + // copy all required types + module_builder.import_type(&h); + } + + // copy owned types into header and module + for h in owned_constants.values() { + header_builder.import_const(h); + module_builder.import_const(h); + } + + for h in owned_vars.values() { + header_builder.import_global(h); + module_builder.import_global(h); + } + + // only stubs of owned functions into the header + for (h_f, f) in owned_functions.values() { + let span = h_f + .map(|h_f| source_ir.functions.get_span(h_f)) + .unwrap_or(naga::Span::UNDEFINED); + header_builder.import_function(f, span); // header stub function + } + // all functions into the module (note source_ir only contains stubs for imported functions) + for (h_f, f) in source_ir.functions.iter() { + let span = source_ir.functions.get_span(h_f); + module_builder.import_function(f, span); + } + // // including entry points as vanilla functions if required + if demote_entrypoints { + for ep in &source_ir.entry_points { + let mut f = ep.function.clone(); + f.arguments = f + .arguments + .into_iter() + .map(|arg| naga::FunctionArgument { + name: arg.name, + ty: arg.ty, + binding: None, + }) + .collect(); + f.result = f.result.map(|res| naga::FunctionResult { + ty: res.ty, + binding: None, + }); + + module_builder.import_function(&f, naga::Span::UNDEFINED); + // todo figure out how to get span info for entrypoints + } + } + + let module_ir = module_builder.into_module_with_entrypoints(); + let mut header_ir: naga::Module = header_builder.into(); + + if self.validate && create_headers { + // check that identifiers haven't been renamed + #[allow(clippy::single_element_loop)] + for language in [ + ShaderLanguage::Wgsl, + #[cfg(feature = "glsl")] + ShaderLanguage::Glsl, + ] { + let header = self + .naga_to_string(&mut header_ir, language, &module_definition.name.0) + .map_err(wrap_err)?; + Self::validate_identifiers( + &source_ir, + language, + &header, + &module_decoration, + &owned_types, + ) + .map_err(wrap_err)?; + } + } + + let composable_module = ComposableModule { + decorated_name: module_decoration, + imports, + owned_types, + owned_constants: owned_constants.into_keys().collect(), + owned_vars: owned_vars.into_keys().collect(), + owned_functions: owned_functions.into_keys().collect(), + virtual_functions, + override_functions, + module_ir, + header_ir, + start_offset, + }; + + Ok(composable_module) + } + + // shunt all data owned by a composable into a derived module + fn add_composable_data<'a>( + derived: &mut DerivedModule<'a>, + composable: &'a ComposableModule, + items: Option<&Vec>, + span_offset: usize, + header: bool, + ) { + let items: Option> = items.map(|items| { + items + .iter() + .map(|item| format!("{}{}", item, composable.decorated_name)) + .collect() + }); + let items = items.as_ref(); + + let source_ir = match header { + true => &composable.header_ir, + false => &composable.module_ir, + }; + + derived.set_shader_source(source_ir, span_offset); + + for (h, ty) in source_ir.types.iter() { + if let Some(name) = &ty.name { + if composable.owned_types.contains(name) + && items.map_or(true, |items| items.contains(name)) + { + derived.import_type(&h); + } + } + } + + for (h, c) in source_ir.constants.iter() { + if let Some(name) = &c.name { + if composable.owned_constants.contains(name) + && items.map_or(true, |items| items.contains(name)) + { + derived.import_const(&h); + } + } + } + + for (h, v) in source_ir.global_variables.iter() { + if let Some(name) = &v.name { + if composable.owned_vars.contains(name) + && items.map_or(true, |items| items.contains(name)) + { + derived.import_global(&h); + } + } + } + + for (h_f, f) in source_ir.functions.iter() { + if let Some(name) = &f.name { + if composable.owned_functions.contains(name) + && (items.map_or(true, |items| items.contains(name)) + || composable + .override_functions + .values() + .any(|v| v.contains(name))) + { + let span = composable.module_ir.functions.get_span(h_f); + derived.import_function_if_new(f, span); + } + } + } + + derived.clear_shader_source(); + } + + // add an import (and recursive imports) into a derived module + fn add_import<'a>( + &'a self, + derived: &mut DerivedModule<'a>, + import: &ImportDefinition, + shader_defs: &HashMap, + header: bool, + already_added: &mut HashSet, + ) { + if already_added.contains(&import.module) { + trace!("skipping {}, already added", import.module); + return; + } + + let import_module_set = self.module_sets.get(&import.module).unwrap(); + let module = import_module_set.get_module(shader_defs).unwrap(); + + for import in &module.imports { + self.add_import(derived, import, shader_defs, header, already_added); + } + + Self::add_composable_data( + derived, + module, + Some(&import.items), + import_module_set.module_index << SPAN_SHIFT, + header, + ); + } + + fn ensure_import( + &mut self, + module_set: &ComposableModuleDefinition, + shader_defs: &HashMap, + ) -> Result { + let PreprocessOutput { + preprocessed_source, + imports, + } = self + .preprocessor + .preprocess(&module_set.source, shader_defs, self.validate) + .map_err(|inner| ComposerError { + inner, + source: ErrSource::Module { + name: module_set.name.to_owned(), + offset: 0, + defs: shader_defs.clone(), + }, + })?; + + self.ensure_imports(imports.iter().map(|import| &import.definition), shader_defs)?; + self.ensure_imports(&module_set.additional_imports, shader_defs)?; + + self.create_composable_module( + module_set, + Self::decorate(&module_set.name), + shader_defs, + true, + true, + &preprocessed_source, + imports, + ) + } + + pub(super) fn get_imported_module(&self, import: &FlattenedImport) -> Option { + let module_exists = self + .module_sets + .contains_key(&ModuleName(import.path.clone())); + let splitted_module_path = import.path.rsplit_once("::"); + // TODO: Change the syntax, or add #export s so that I don't need to rely on this hack where I check "which import could be correct". + let module = match (module_exists, splitted_module_path) { + (true, None) => ModuleName::new(import.path.clone()), + (true, Some((module, _item))) => { + eprintln!("Ambiguous import: {} could refer to either a module or a function. Please use the syntax `module::function` to disambiguate.", import.path); + ModuleName::new(module) + } + (false, None) => { + return None; + } + (false, Some((module, _item))) => ModuleName::new(module), + }; + + Some(module) + } + + pub(super) fn collect_all_imports( + &self, + entry_point: &ModuleName, + imports: &mut HashSet, + ) -> Result<(), ComposerError> { + let entry_module = self.module_sets.get(entry_point).unwrap(); + + // TODO: Document that conditional imports are not supported. (have to be at the very top, just like #defines) + // TODO: Verify that ^ + // Alternatively, we could support them by changing the #define semantics to no longer be global & time traveling. + + for import in entry_module.shader_imports.iter() { + let module = match self.get_imported_module(import) { + Some(v) => v, + None => { + return Err(ComposerError { + inner: ComposerErrorInner::ImportNotFound( + import.path.to_owned(), + import.offset, + ), + source: ErrSource::Module { + name: entry_point.0.to_string(), + offset: 0, + defs: Default::default(), // TODO: Set this properly + }, + }); + } + }; + + let is_new_import = imports.insert(module.clone()); + if is_new_import { + self.collect_all_imports(&module, imports)?; + } + } + + for (additional_module, _) in entry_module.additional_imports.iter() { + let is_new_import = imports.insert(additional_module.clone()); + if is_new_import { + self.collect_all_imports(&additional_module, imports)?; + } + } + Ok(()) + } + + pub(super) fn collect_shader_defs( + &self, + imports: &HashSet, + shader_defs: &mut HashMap, + ) -> Result<(), ComposerError> { + for import_name in imports { + // TODO: No unwrap pls + let module = self.module_sets.get(import_name).unwrap(); + for (def, value) in module.shader_defs.iter() { + match shader_defs.insert(def.clone(), value.clone()) { + Some(old_value) if &old_value != value => { + return Err(ComposerError { + inner: ComposerErrorInner::InconsistentShaderDefValue { + def: def.clone(), + }, + source: ErrSource::Constructing { + path: module.file_path.to_owned(), + source: module.source.to_owned(), + offset: 0, // TODO: Set this properly + }, + }); + } + _ => {} + } + } + } + + Ok(()) + } + + pub(super) fn make_composable_module( + &self, + desc: ComposableModuleDescriptor, + ) -> Result { + let ComposableModuleDescriptor { + source, + file_path, + language, + as_name, + additional_imports, + mut shader_defs, + } = desc; + + // reject a module containing the DECORATION strings + if let Some(decor) = self.check_decoration_regex.find(source) { + return Err(ComposerError { + inner: ComposerErrorInner::DecorationInSource(decor.range()), + source: ErrSource::Constructing { + path: file_path.to_owned(), + source: source.to_owned(), + offset: 0, + }, + }); + } + + let (_, parsed, errors) = + preprocess::preprocess.recoverable_parse(winnow::Located::new(source)); + + if !errors.is_empty() { + return Err(ComposerError { + inner: ComposerErrorInner::PreprocessorError( + // TODO: Prettier error messages + errors + .into_iter() + .map(|v| v.to_string()) + .collect::>() + .into(), + ), + source: ErrSource::Constructing { + path: file_path.to_owned(), + source: source.to_owned(), + offset: 0, + }, + }); + } + let parsed = match parsed { + Some(parsed) => parsed, + None => { + return Err(ComposerError { + inner: ComposerErrorInner::PreprocessorError( + vec!["preprocessor failed to parse source".to_owned()].into(), + ), + source: ErrSource::Constructing { + path: file_path.to_owned(), + source: source.to_owned(), + offset: 0, + }, + }); + } + }; + + let module_names = as_name + .into_iter() + .chain(parsed.get_module_names(source).map(|v| v.to_owned())) + .collect::>(); + if module_names.len() == 0 { + return Err(ComposerError { + inner: ComposerErrorInner::NoModuleName, + source: ErrSource::Constructing { + path: file_path.to_owned(), + source: source.to_owned(), + offset: 0, + }, + }); + } + if module_names.len() > 1 { + return Err(ComposerError { + inner: ComposerErrorInner::MultipleModuleNames(module_names.into()), + source: ErrSource::Constructing { + path: file_path.to_owned(), + source: source.to_owned(), + offset: 0, // TODO: Return the offset of the second module name + }, + }); + } + let module_name = ModuleName(module_names.into_iter().next().unwrap()); + let used_defs = parsed.get_used_defs(source); + let defined_defs = parsed.get_defined_defs(source); + shader_defs.extend(defined_defs); + + debug!( + "adding module definition for {:?} with defs: {:?}", + module_name, used_defs + ); + + let additional_imports = additional_imports + .into_iter() + .flat_map(|v| { + let items = if v.items.is_empty() { + vec![v.module.0.clone()] + } else { + v.items.clone() + }; + + items.into_iter().map(|item| { + ( + v.module.clone(), + ImportDefinition { + module: v.module.clone(), + item, + }, + ) + }) + }) + .collect::>(); + let shader_imports = parsed.get_imports(source); + let alias_to_path = shader_imports + .iter() + .filter_map(|v| { + v.alias + .as_ref() + .map(|alias| (alias.clone(), v.path.clone())) + }) + .collect(); + + let module_set = ComposableModuleDefinition { + name: module_name.clone(), + source: source.to_owned(), + file_path: file_path.to_owned(), + language, + used_defs: used_defs.into_iter().collect(), + additional_imports, + shader_imports, + shader_defs, + parsed, + alias_to_path, + }; + + Ok(module_set) + } +} diff --git a/src/compose/error.rs b/src/compose/error.rs index 1fe23bf..8db2dd6 100644 --- a/src/compose/error.rs +++ b/src/compose/error.rs @@ -13,7 +13,7 @@ use codespan_reporting::{ use thiserror::Error; use tracing::trace; -use super::{preprocess::PreprocessOutput, Composer, ShaderDefValue}; +use super::{ Composer, ShaderDefValue}; use crate::{compose::SPAN_SHIFT, redirect::RedirectError}; #[derive(Debug)] @@ -41,7 +41,7 @@ impl ErrSource { pub fn source<'a>(&'a self, composer: &'a Composer) -> Cow<'a, String> { match self { ErrSource::Module { name, defs, .. } => { - let raw_source = &composer.module_sets.get(name).unwrap().sanitized_source; + let raw_source = &composer.module_sets.get(name).unwrap().source; let Ok(PreprocessOutput { preprocessed_source: source, .. @@ -144,6 +144,8 @@ pub enum ComposerErrorInner { DefineInModule(usize), #[error("failed to preprocess shader {0}")] PreprocessorError(StringsWithNewlines), + #[error("module already exists, cannot overwrite {0}")] + ModuleAlreadyExists(String), } #[derive(Debug)] @@ -324,6 +326,9 @@ impl ComposerError { ComposerErrorInner::PreprocessorError(e) => { return format!("{path}: preprocessor errors: {e}"); } + ComposerErrorInner::ModuleAlreadyExists(name) => { + return format!("{path}: module already exists, cannot overwrite {name}"); + } }; let diagnostic = Diagnostic::error() diff --git a/src/compose/mod.rs b/src/compose/mod.rs index 70af484..0ed511d 100644 --- a/src/compose/mod.rs +++ b/src/compose/mod.rs @@ -1,4 +1,15 @@ -use indexmap::IndexMap; +use std::collections::{HashMap, HashSet}; + +use naga::EntryPoint; + +use crate::{derive::DerivedModule, redirect::Redirector}; + +use self::composer::{ + ComposableModuleDefinition, Composer, ComposerError, ComposerErrorInner, ErrSource, ModuleName, + ShaderDefValue, +}; + +mod compose_parser; /// the compose module allows construction of shaders from modules (which are themselves shaders). /// /// it does this by treating shaders as modules, and @@ -126,32 +137,20 @@ use indexmap::IndexMap; /// /// codespan reporting for errors is available using the error `emit_to_string` method. this requires validation to be enabled, which is true by default. `Composer::non_validating()` produces a non-validating composer that is not able to give accurate error reporting. /// -use naga::EntryPoint; -use regex::Regex; -use std::collections::{hash_map::Entry, BTreeMap, HashMap, HashSet}; -use tracing::{debug, trace}; -use winnow::RecoverableParser; - -use crate::{ - compose::{ - preprocess::{PreprocessOutput, PreprocessorMetaData}, - preprocess1::{DefineImportPath, PreprocessorPart}, - }, - derive::DerivedModule, - redirect::Redirector, -}; - -pub use self::error::{ComposerError, ComposerErrorInner, ErrSource}; -use self::preprocess::Preprocessor; - +pub mod composer; pub mod error; -pub mod parse_imports; pub mod preprocess; -pub mod preprocess1; mod test; pub mod tokenizer; pub mod util; +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AdditionalImport { + // TODO: Support aliases? + pub module: ModuleName, + pub items: Vec, +} + #[derive(Hash, PartialEq, Eq, Clone, Copy, Debug, Default)] pub enum ShaderLanguage { #[default] @@ -180,1189 +179,13 @@ impl From for ShaderLanguage { } } -#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] -pub enum ShaderDefValue { - Bool(bool), - Int(i32), - UInt(u32), -} - -impl Default for ShaderDefValue { - fn default() -> Self { - ShaderDefValue::Bool(true) - } -} - -impl ShaderDefValue { - fn value_as_string(&self) -> String { - match self { - ShaderDefValue::Bool(val) => val.to_string(), - ShaderDefValue::Int(val) => val.to_string(), - ShaderDefValue::UInt(val) => val.to_string(), - } - } -} - -#[derive(Clone, PartialEq, Eq, Hash, Debug, Default)] -pub struct OwnedShaderDefs(BTreeMap); - -#[derive(Clone, PartialEq, Eq, Hash, Debug)] -struct ModuleKey(OwnedShaderDefs); - -impl ModuleKey { - fn from_members(key: &HashMap, universe: &[String]) -> Self { - let mut acc = OwnedShaderDefs::default(); - for item in universe { - if let Some(value) = key.get(item) { - acc.0.insert(item.to_owned(), *value); - } - } - ModuleKey(acc) - } -} - -// a module built with a specific set of shader_defs -#[derive(Default, Debug)] -pub struct ComposableModule { - // module decoration, prefixed to all items from this module in the final source - pub decorated_name: String, - // module names required as imports, optionally with a list of items to import - pub imports: Vec, - // types exported - pub owned_types: HashSet, - // constants exported - pub owned_constants: HashSet, - // vars exported - pub owned_vars: HashSet, - // functions exported - pub owned_functions: HashSet, - // local functions that can be overridden - pub virtual_functions: HashSet, - // overriding functions defined in this module - // target function -> Vec - pub override_functions: IndexMap>, - // naga module, built against headers for any imports - module_ir: naga::Module, - // headers in different shader languages, used for building modules/shaders that import this module - // headers contain types, constants, global vars and empty function definitions - - // just enough to convert source strings that want to import this module into naga IR - // headers: HashMap, - header_ir: naga::Module, - // character offset of the start of the owned module string - start_offset: usize, -} - -// data used to build a ComposableModule -#[derive(Debug)] -pub struct ComposableModuleDefinition { - pub name: String, - // shader text (with auto bindings replaced - we do this on module add as we only want to do it once to avoid burning slots) - pub sanitized_source: String, - // language - pub language: ShaderLanguage, - // source path for error display - pub file_path: String, - // shader def values bound to this module - pub shader_defs: HashMap, - // list of shader_defs that can affect this module - effective_defs: Vec, - // full list of possible imports (regardless of shader_def configuration) - all_imports: HashSet, - // additional imports to add (as though they were included in the source after any other imports) - additional_imports: Vec, - // built composable modules for a given set of shader defs - modules: HashMap, - // used in spans when this module is included - module_index: usize, - // preprocessor meta data - // metadata: PreprocessorMetaData, -} - -impl ComposableModuleDefinition { - fn get_module( - &self, - shader_defs: &HashMap, - ) -> Option<&ComposableModule> { - self.modules - .get(&ModuleKey::from_members(shader_defs, &self.effective_defs)) - } - - fn insert_module( - &mut self, - shader_defs: &HashMap, - module: ComposableModule, - ) -> &ComposableModule { - match self - .modules - .entry(ModuleKey::from_members(shader_defs, &self.effective_defs)) - { - Entry::Occupied(_) => panic!("entry already populated"), - Entry::Vacant(v) => v.insert(module), - } - } -} - -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct ImportDefinition { - pub import: String, - pub items: Vec, -} - -#[derive(Debug, Clone)] -pub struct ImportDefWithOffset { - definition: ImportDefinition, - offset: usize, -} - -/// module composer. -/// stores any modules that can be imported into a shader -/// and builds the final shader -#[derive(Debug)] -pub struct Composer { - pub validate: bool, - pub module_sets: HashMap, - pub module_index: HashMap, - pub capabilities: naga::valid::Capabilities, - preprocessor: Preprocessor, - check_decoration_regex: Regex, - undecorate_regex: Regex, - virtual_fn_regex: Regex, - override_fn_regex: Regex, - undecorate_override_regex: Regex, - auto_binding_regex: Regex, - auto_binding_index: u32, -} - -// shift for module index -// 21 gives -// max size for shader of 2m characters -// max 2048 modules -const SPAN_SHIFT: usize = 21; - -impl Default for Composer { - fn default() -> Self { - Self { - validate: true, - capabilities: Default::default(), - module_sets: Default::default(), - module_index: Default::default(), - preprocessor: Preprocessor, - check_decoration_regex: Regex::new( - format!( - "({}|{})", - regex_syntax::escape(DECORATION_PRE), - regex_syntax::escape(DECORATION_OVERRIDE_PRE) - ) - .as_str(), - ) - .unwrap(), - undecorate_regex: Regex::new( - format!( - r"(\x1B\[\d+\w)?([\w\d_]+){}([A-Z0-9]*){}", - regex_syntax::escape(DECORATION_PRE), - regex_syntax::escape(DECORATION_POST) - ) - .as_str(), - ) - .unwrap(), - virtual_fn_regex: Regex::new( - r"(?P[\s]*virtual\s+fn\s+)(?P[^\s]+)(?P\s*)\(", - ) - .unwrap(), - override_fn_regex: Regex::new( - format!( - r"(override\s+fn\s+)([^\s]+){}([\w\d]+){}(\s*)\(", - regex_syntax::escape(DECORATION_PRE), - regex_syntax::escape(DECORATION_POST) - ) - .as_str(), - ) - .unwrap(), - undecorate_override_regex: Regex::new( - format!( - "{}([A-Z0-9]*){}", - regex_syntax::escape(DECORATION_OVERRIDE_PRE), - regex_syntax::escape(DECORATION_POST) - ) - .as_str(), - ) - .unwrap(), - auto_binding_regex: Regex::new(r"@binding\(auto\)").unwrap(), - auto_binding_index: 0, - } - } -} - -const DECORATION_PRE: &str = "X_naga_oil_mod_X"; -const DECORATION_POST: &str = "X"; - -// must be same length as DECORATION_PRE for spans to work -const DECORATION_OVERRIDE_PRE: &str = "X_naga_oil_vrt_X"; - -struct IrBuildResult { - module: naga::Module, - start_offset: usize, - override_functions: IndexMap>, -} - -impl Composer { - pub fn decorated_name(module_name: Option<&str>, item_name: &str) -> String { - match module_name { - Some(module_name) => format!("{}{}", item_name, Self::decorate(module_name)), - None => item_name.to_owned(), - } - } - - fn decorate(module: &str) -> String { - let encoded = data_encoding::BASE32_NOPAD.encode(module.as_bytes()); - format!("{DECORATION_PRE}{encoded}{DECORATION_POST}") - } - - fn decode(from: &str) -> String { - String::from_utf8(data_encoding::BASE32_NOPAD.decode(from.as_bytes()).unwrap()).unwrap() - } - - fn undecorate(&self, string: &str) -> String { - let undecor = self - .undecorate_regex - .replace_all(string, |caps: ®ex::Captures| { - format!( - "{}{}::{}", - caps.get(1).map(|cc| cc.as_str()).unwrap_or(""), - Self::decode(caps.get(3).unwrap().as_str()), - caps.get(2).unwrap().as_str() - ) - }); - - let undecor = - self.undecorate_override_regex - .replace_all(&undecor, |caps: ®ex::Captures| { - format!( - "override fn {}::", - Self::decode(caps.get(1).unwrap().as_str()) - ) - }); - - undecor.to_string() - } - - fn sanitize_and_set_auto_bindings(&mut self, source: &str) -> String { - let mut substituted_source = source.replace("\r\n", "\n").replace('\r', "\n"); - if !substituted_source.ends_with('\n') { - substituted_source.push('\n'); - } - - // replace @binding(auto) with an incrementing index - struct AutoBindingReplacer<'a> { - auto: &'a mut u32, - } - - impl<'a> regex::Replacer for AutoBindingReplacer<'a> { - fn replace_append(&mut self, _: ®ex::Captures<'_>, dst: &mut String) { - dst.push_str(&format!("@binding({})", self.auto)); - *self.auto += 1; - } - } - - let substituted_source = self.auto_binding_regex.replace_all( - &substituted_source, - AutoBindingReplacer { - auto: &mut self.auto_binding_index, - }, - ); - - substituted_source.into_owned() - } - - fn naga_to_string( - &self, - naga_module: &mut naga::Module, - language: ShaderLanguage, - #[allow(unused)] header_for: &str, // Only used when GLSL is enabled - ) -> Result { - // TODO: cache headers again - let info = - naga::valid::Validator::new(naga::valid::ValidationFlags::all(), self.capabilities) - .validate(naga_module) - .map_err(ComposerErrorInner::HeaderValidationError)?; - - match language { - ShaderLanguage::Wgsl => naga::back::wgsl::write_string( - naga_module, - &info, - naga::back::wgsl::WriterFlags::EXPLICIT_TYPES, - ) - .map_err(ComposerErrorInner::WgslBackError), - #[cfg(feature = "glsl")] - ShaderLanguage::Glsl => { - let vec4 = naga_module.types.insert( - naga::Type { - name: None, - inner: naga::TypeInner::Vector { - size: naga::VectorSize::Quad, - scalar: naga::Scalar::F32, - }, - }, - naga::Span::UNDEFINED, - ); - // add a dummy entry point for glsl headers - let dummy_entry_point = "dummy_module_entry_point".to_owned(); - let func = naga::Function { - name: Some(dummy_entry_point.clone()), - arguments: Default::default(), - result: Some(naga::FunctionResult { - ty: vec4, - binding: Some(naga::Binding::BuiltIn(naga::BuiltIn::Position { - invariant: false, - })), - }), - local_variables: Default::default(), - expressions: Default::default(), - named_expressions: Default::default(), - body: Default::default(), - }; - let ep = EntryPoint { - name: dummy_entry_point.clone(), - stage: naga::ShaderStage::Vertex, - function: func, - early_depth_test: None, - workgroup_size: [0, 0, 0], - }; - - naga_module.entry_points.push(ep); - - let info = naga::valid::Validator::new( - naga::valid::ValidationFlags::all(), - self.capabilities, - ) - .validate(naga_module) - .map_err(ComposerErrorInner::HeaderValidationError)?; - - let mut string = String::new(); - let options = naga::back::glsl::Options { - version: naga::back::glsl::Version::Desktop(450), - writer_flags: naga::back::glsl::WriterFlags::INCLUDE_UNUSED_ITEMS, - ..Default::default() - }; - let pipeline_options = naga::back::glsl::PipelineOptions { - shader_stage: naga::ShaderStage::Vertex, - entry_point: dummy_entry_point, - multiview: None, - }; - let mut writer = naga::back::glsl::Writer::new( - &mut string, - naga_module, - &info, - &options, - &pipeline_options, - naga::proc::BoundsCheckPolicies::default(), - ) - .map_err(ComposerErrorInner::GlslBackError)?; - - writer.write().map_err(ComposerErrorInner::GlslBackError)?; - - // strip version decl and main() impl - let lines: Vec<_> = string.lines().collect(); - let string = lines[1..lines.len() - 3].join("\n"); - trace!("glsl header for {}:\n\"\n{:?}\n\"", header_for, string); - - Ok(string) - } - } - } - - // build naga module for a given shader_def configuration. builds a minimal self-contained module built against headers for imports - fn create_module_ir( - &self, - name: &str, - source: String, - language: ShaderLanguage, - imports: &[ImportDefinition], - shader_defs: &HashMap, - ) -> Result { - debug!("creating IR for {} with defs: {:?}", name, shader_defs); - - let mut module_string = match language { - ShaderLanguage::Wgsl => String::new(), - #[cfg(feature = "glsl")] - ShaderLanguage::Glsl => String::from("#version 450\n"), - }; - - let mut override_functions: IndexMap> = IndexMap::default(); - let mut added_imports: HashSet = HashSet::new(); - let mut header_module = DerivedModule::default(); - - for import in imports { - if added_imports.contains(&import.import) { - continue; - } - // add to header module - self.add_import( - &mut header_module, - import, - shader_defs, - true, - &mut added_imports, - ); - - // // we must have ensured these exist with Composer::ensure_imports() - trace!("looking for {}", import.import); - let import_module_set = self.module_sets.get(&import.import).unwrap(); - trace!("with defs {:?}", shader_defs); - let module = import_module_set.get_module(shader_defs).unwrap(); - trace!("ok"); - - // gather overrides - if !module.override_functions.is_empty() { - for (original, replacements) in &module.override_functions { - match override_functions.entry(original.clone()) { - indexmap::map::Entry::Occupied(o) => { - let existing = o.into_mut(); - let new_replacements: Vec<_> = replacements - .iter() - .filter(|rep| !existing.contains(rep)) - .cloned() - .collect(); - existing.extend(new_replacements); - } - indexmap::map::Entry::Vacant(v) => { - v.insert(replacements.clone()); - } - } - } - } - } - - let composed_header = self - .naga_to_string(&mut header_module.into(), language, name) - .map_err(|inner| ComposerError { - inner, - source: ErrSource::Module { - name: name.to_owned(), - offset: 0, - defs: shader_defs.clone(), - }, - })?; - module_string.push_str(&composed_header); - - let start_offset = module_string.len(); - - module_string.push_str(&source); - - trace!( - "parsing {}: {}, header len {}, total len {}", - name, - module_string, - start_offset, - module_string.len() - ); - let module = match language { - ShaderLanguage::Wgsl => naga::front::wgsl::parse_str(&module_string).map_err(|e| { - debug!("full err'd source file: \n---\n{}\n---", module_string); - ComposerError { - inner: ComposerErrorInner::WgslParseError(e), - source: ErrSource::Module { - name: name.to_owned(), - offset: start_offset, - defs: shader_defs.clone(), - }, - } - })?, - #[cfg(feature = "glsl")] - ShaderLanguage::Glsl => naga::front::glsl::Frontend::default() - .parse( - &naga::front::glsl::Options { - stage: naga::ShaderStage::Vertex, - defines: Default::default(), - }, - &module_string, - ) - .map_err(|e| { - debug!("full err'd source file: \n---\n{}\n---", module_string); - ComposerError { - inner: ComposerErrorInner::GlslParseError(e), - source: ErrSource::Module { - name: name.to_owned(), - offset: start_offset, - defs: shader_defs.clone(), - }, - } - })?, - }; - - Ok(IrBuildResult { - module, - start_offset, - override_functions, - }) - } - - // check that identifiers exported by a module do not get modified in string export - fn validate_identifiers( - source_ir: &naga::Module, - lang: ShaderLanguage, - header: &str, - module_decoration: &str, - owned_types: &HashSet, - ) -> Result<(), ComposerErrorInner> { - // TODO: remove this once glsl front support is complete - #[cfg(feature = "glsl")] - if lang == ShaderLanguage::Glsl { - return Ok(()); - } - - let recompiled = match lang { - ShaderLanguage::Wgsl => naga::front::wgsl::parse_str(header).unwrap(), - #[cfg(feature = "glsl")] - ShaderLanguage::Glsl => naga::front::glsl::Frontend::default() - .parse( - &naga::front::glsl::Options { - stage: naga::ShaderStage::Vertex, - defines: Default::default(), - }, - &format!("{}\n{}", header, "void main() {}"), - ) - .map_err(|e| { - debug!("full err'd source file: \n---\n{header}\n---"); - ComposerErrorInner::GlslParseError(e) - })?, - }; - - let recompiled_types: IndexMap<_, _> = recompiled - .types - .iter() - .flat_map(|(h, ty)| ty.name.as_deref().map(|name| (name, h))) - .collect(); - for (h, ty) in source_ir.types.iter() { - if let Some(name) = &ty.name { - let decorated_type_name = format!("{name}{module_decoration}"); - if !owned_types.contains(&decorated_type_name) { - continue; - } - match recompiled_types.get(decorated_type_name.as_str()) { - Some(recompiled_h) => { - if let naga::TypeInner::Struct { members, .. } = &ty.inner { - let recompiled_ty = recompiled.types.get_handle(*recompiled_h).unwrap(); - let naga::TypeInner::Struct { - members: recompiled_members, - .. - } = &recompiled_ty.inner - else { - panic!(); - }; - for (member, recompiled_member) in - members.iter().zip(recompiled_members) - { - if member.name != recompiled_member.name { - return Err(ComposerErrorInner::InvalidIdentifier { - original: member.name.clone().unwrap_or_default(), - at: source_ir.types.get_span(h), - }); - } - } - } - } - None => { - return Err(ComposerErrorInner::InvalidIdentifier { - original: name.clone(), - at: source_ir.types.get_span(h), - }) - } - } - } - } - - let recompiled_consts: HashSet<_> = recompiled - .constants - .iter() - .flat_map(|(_, c)| c.name.as_deref()) - .filter(|name| name.ends_with(module_decoration)) - .collect(); - for (h, c) in source_ir.constants.iter() { - if let Some(name) = &c.name { - if name.ends_with(module_decoration) && !recompiled_consts.contains(name.as_str()) { - return Err(ComposerErrorInner::InvalidIdentifier { - original: name.clone(), - at: source_ir.constants.get_span(h), - }); - } - } - } - - let recompiled_globals: HashSet<_> = recompiled - .global_variables - .iter() - .flat_map(|(_, c)| c.name.as_deref()) - .filter(|name| name.ends_with(module_decoration)) - .collect(); - for (h, gv) in source_ir.global_variables.iter() { - if let Some(name) = &gv.name { - if name.ends_with(module_decoration) && !recompiled_globals.contains(name.as_str()) - { - return Err(ComposerErrorInner::InvalidIdentifier { - original: name.clone(), - at: source_ir.global_variables.get_span(h), - }); - } - } - } - - let recompiled_fns: HashSet<_> = recompiled - .functions - .iter() - .flat_map(|(_, c)| c.name.as_deref()) - .filter(|name| name.ends_with(module_decoration)) - .collect(); - for (h, f) in source_ir.functions.iter() { - if let Some(name) = &f.name { - if name.ends_with(module_decoration) && !recompiled_fns.contains(name.as_str()) { - return Err(ComposerErrorInner::InvalidIdentifier { - original: name.clone(), - at: source_ir.functions.get_span(h), - }); - } - } - } - - Ok(()) - } - - // build a ComposableModule from a ComposableModuleDefinition, for a given set of shader defs - // - build the naga IR (against headers) - // - record any types/vars/constants/functions that are defined within this module - // - build headers for each supported language - #[allow(clippy::too_many_arguments)] - fn create_composable_module( - &mut self, - module_definition: &ComposableModuleDefinition, - module_decoration: String, - shader_defs: &HashMap, - create_headers: bool, - demote_entrypoints: bool, - source: &str, - imports: Vec, - ) -> Result { - let mut imports: Vec<_> = imports - .into_iter() - .map(|import_with_offset| import_with_offset.definition) - .collect(); - imports.extend(module_definition.additional_imports.to_vec()); - - trace!( - "create composable module {}: source len {}", - module_definition.name, - source.len() - ); - - // record virtual/overridable functions - let mut virtual_functions: HashSet = Default::default(); - let source = self - .virtual_fn_regex - .replace_all(source, |cap: ®ex::Captures| { - let target_function = cap.get(2).unwrap().as_str().to_owned(); - - let replacement_str = format!( - "{}fn {}{}(", - " ".repeat(cap.get(1).unwrap().range().len() - 3), - target_function, - " ".repeat(cap.get(3).unwrap().range().len()), - ); - - virtual_functions.insert(target_function); - - replacement_str - }); - - // record and rename override functions - let mut local_override_functions: IndexMap = Default::default(); - - #[cfg(not(feature = "override_any"))] - let mut override_error = None; - - let source = - self.override_fn_regex - .replace_all(&source, |cap: ®ex::Captures| { - let target_module = cap.get(3).unwrap().as_str().to_owned(); - let target_function = cap.get(2).unwrap().as_str().to_owned(); - - #[cfg(not(feature = "override_any"))] - { - let wrap_err = |inner: ComposerErrorInner| -> ComposerError { - ComposerError { - inner, - source: ErrSource::Module { - name: module_definition.name.to_owned(), - offset: 0, - defs: shader_defs.clone(), - }, - } - }; - - // ensure overrides are applied to virtual functions - let raw_module_name = Self::decode(&target_module); - let module_set = self.module_sets.get(&raw_module_name); - - match module_set { - None => { - // TODO this should be unreachable? - let pos = cap.get(3).unwrap().start(); - override_error = Some(wrap_err( - ComposerErrorInner::ImportNotFound(raw_module_name, pos), - )); - } - Some(module_set) => { - let module = module_set.get_module(shader_defs).unwrap(); - if !module.virtual_functions.contains(&target_function) { - let pos = cap.get(2).unwrap().start(); - override_error = - Some(wrap_err(ComposerErrorInner::OverrideNotVirtual { - name: target_function.clone(), - pos, - })); - } - } - } - } - - let base_name = format!( - "{}{}{}{}", - target_function.as_str(), - DECORATION_PRE, - target_module.as_str(), - DECORATION_POST, - ); - let rename = format!( - "{}{}{}{}", - target_function.as_str(), - DECORATION_OVERRIDE_PRE, - target_module.as_str(), - DECORATION_POST, - ); - - let replacement_str = format!( - "{}fn {}{}(", - " ".repeat(cap.get(1).unwrap().range().len() - 3), - rename, - " ".repeat(cap.get(4).unwrap().range().len()), - ); - - local_override_functions.insert(rename, base_name); - - replacement_str - }) - .to_string(); - - #[cfg(not(feature = "override_any"))] - if let Some(err) = override_error { - return Err(err); - } - - trace!("local overrides: {:?}", local_override_functions); - trace!( - "create composable module {}: source len {}", - module_definition.name, - source.len() - ); - - let IrBuildResult { - module: mut source_ir, - start_offset, - mut override_functions, - } = self.create_module_ir( - &module_definition.name, - source, - module_definition.language, - &imports, - shader_defs, - )?; - - // from here on errors need to be reported using the modified source with start_offset - let wrap_err = |inner: ComposerErrorInner| -> ComposerError { - ComposerError { - inner, - source: ErrSource::Module { - name: module_definition.name.to_owned(), - offset: start_offset, - defs: shader_defs.clone(), - }, - } - }; - - // add our local override to the total set of overrides for the given function - for (rename, base_name) in &local_override_functions { - override_functions - .entry(base_name.clone()) - .or_default() - .push(format!("{rename}{module_decoration}")); - } - - // rename and record owned items (except types which can't be mutably accessed) - let mut owned_constants = IndexMap::new(); - for (h, c) in source_ir.constants.iter_mut() { - if let Some(name) = c.name.as_mut() { - if !name.contains(DECORATION_PRE) { - *name = format!("{name}{module_decoration}"); - owned_constants.insert(name.clone(), h); - } - } - } - - let mut owned_vars = IndexMap::new(); - for (h, gv) in source_ir.global_variables.iter_mut() { - if let Some(name) = gv.name.as_mut() { - if !name.contains(DECORATION_PRE) { - *name = format!("{name}{module_decoration}"); - - owned_vars.insert(name.clone(), h); - } - } - } - - let mut owned_functions = IndexMap::new(); - for (h_f, f) in source_ir.functions.iter_mut() { - if let Some(name) = f.name.as_mut() { - if !name.contains(DECORATION_PRE) { - *name = format!("{name}{module_decoration}"); - - // create dummy header function - let header_function = naga::Function { - name: Some(name.clone()), - arguments: f.arguments.to_vec(), - result: f.result.clone(), - local_variables: Default::default(), - expressions: Default::default(), - named_expressions: Default::default(), - body: Default::default(), - }; - - // record owned function - owned_functions.insert(name.clone(), (Some(h_f), header_function)); - } - } - } - - if demote_entrypoints { - // make normal functions out of the source entry points - for ep in &mut source_ir.entry_points { - ep.function.name = Some(format!( - "{}{}", - ep.function.name.as_deref().unwrap_or("main"), - module_decoration, - )); - let header_function = naga::Function { - name: ep.function.name.clone(), - arguments: ep - .function - .arguments - .iter() - .cloned() - .map(|arg| naga::FunctionArgument { - name: arg.name, - ty: arg.ty, - binding: None, - }) - .collect(), - result: ep.function.result.clone().map(|res| naga::FunctionResult { - ty: res.ty, - binding: None, - }), - local_variables: Default::default(), - expressions: Default::default(), - named_expressions: Default::default(), - body: Default::default(), - }; - - owned_functions.insert(ep.function.name.clone().unwrap(), (None, header_function)); - } - }; - - let mut module_builder = DerivedModule::default(); - let mut header_builder = DerivedModule::default(); - module_builder.set_shader_source(&source_ir, 0); - header_builder.set_shader_source(&source_ir, 0); - - let mut owned_types = HashSet::new(); - for (h, ty) in source_ir.types.iter() { - if let Some(name) = &ty.name { - // we need to exclude autogenerated struct names, i.e. those that begin with "__" - // "__" is a reserved prefix for naga so user variables cannot use it. - if !name.contains(DECORATION_PRE) && !name.starts_with("__") { - let name = format!("{name}{module_decoration}"); - owned_types.insert(name.clone()); - // copy and rename types - module_builder.rename_type(&h, Some(name.clone())); - header_builder.rename_type(&h, Some(name)); - continue; - } - } - - // copy all required types - module_builder.import_type(&h); - } - - // copy owned types into header and module - for h in owned_constants.values() { - header_builder.import_const(h); - module_builder.import_const(h); - } - - for h in owned_vars.values() { - header_builder.import_global(h); - module_builder.import_global(h); - } - - // only stubs of owned functions into the header - for (h_f, f) in owned_functions.values() { - let span = h_f - .map(|h_f| source_ir.functions.get_span(h_f)) - .unwrap_or(naga::Span::UNDEFINED); - header_builder.import_function(f, span); // header stub function - } - // all functions into the module (note source_ir only contains stubs for imported functions) - for (h_f, f) in source_ir.functions.iter() { - let span = source_ir.functions.get_span(h_f); - module_builder.import_function(f, span); - } - // // including entry points as vanilla functions if required - if demote_entrypoints { - for ep in &source_ir.entry_points { - let mut f = ep.function.clone(); - f.arguments = f - .arguments - .into_iter() - .map(|arg| naga::FunctionArgument { - name: arg.name, - ty: arg.ty, - binding: None, - }) - .collect(); - f.result = f.result.map(|res| naga::FunctionResult { - ty: res.ty, - binding: None, - }); - - module_builder.import_function(&f, naga::Span::UNDEFINED); - // todo figure out how to get span info for entrypoints - } - } - - let module_ir = module_builder.into_module_with_entrypoints(); - let mut header_ir: naga::Module = header_builder.into(); - - if self.validate && create_headers { - // check that identifiers haven't been renamed - #[allow(clippy::single_element_loop)] - for language in [ - ShaderLanguage::Wgsl, - #[cfg(feature = "glsl")] - ShaderLanguage::Glsl, - ] { - let header = self - .naga_to_string(&mut header_ir, language, &module_definition.name) - .map_err(wrap_err)?; - Self::validate_identifiers( - &source_ir, - language, - &header, - &module_decoration, - &owned_types, - ) - .map_err(wrap_err)?; - } - } - - let composable_module = ComposableModule { - decorated_name: module_decoration, - imports, - owned_types, - owned_constants: owned_constants.into_keys().collect(), - owned_vars: owned_vars.into_keys().collect(), - owned_functions: owned_functions.into_keys().collect(), - virtual_functions, - override_functions, - module_ir, - header_ir, - start_offset, - }; - - Ok(composable_module) - } - - // shunt all data owned by a composable into a derived module - fn add_composable_data<'a>( - derived: &mut DerivedModule<'a>, - composable: &'a ComposableModule, - items: Option<&Vec>, - span_offset: usize, - header: bool, - ) { - let items: Option> = items.map(|items| { - items - .iter() - .map(|item| format!("{}{}", item, composable.decorated_name)) - .collect() - }); - let items = items.as_ref(); - - let source_ir = match header { - true => &composable.header_ir, - false => &composable.module_ir, - }; - - derived.set_shader_source(source_ir, span_offset); - - for (h, ty) in source_ir.types.iter() { - if let Some(name) = &ty.name { - if composable.owned_types.contains(name) - && items.map_or(true, |items| items.contains(name)) - { - derived.import_type(&h); - } - } - } - - for (h, c) in source_ir.constants.iter() { - if let Some(name) = &c.name { - if composable.owned_constants.contains(name) - && items.map_or(true, |items| items.contains(name)) - { - derived.import_const(&h); - } - } - } - - for (h, v) in source_ir.global_variables.iter() { - if let Some(name) = &v.name { - if composable.owned_vars.contains(name) - && items.map_or(true, |items| items.contains(name)) - { - derived.import_global(&h); - } - } - } - - for (h_f, f) in source_ir.functions.iter() { - if let Some(name) = &f.name { - if composable.owned_functions.contains(name) - && (items.map_or(true, |items| items.contains(name)) - || composable - .override_functions - .values() - .any(|v| v.contains(name))) - { - let span = composable.module_ir.functions.get_span(h_f); - derived.import_function_if_new(f, span); - } - } - } - - derived.clear_shader_source(); - } - - // add an import (and recursive imports) into a derived module - fn add_import<'a>( - &'a self, - derived: &mut DerivedModule<'a>, - import: &ImportDefinition, - shader_defs: &HashMap, - header: bool, - already_added: &mut HashSet, - ) { - if already_added.contains(&import.import) { - trace!("skipping {}, already added", import.import); - return; - } - - let import_module_set = self.module_sets.get(&import.import).unwrap(); - let module = import_module_set.get_module(shader_defs).unwrap(); - - for import in &module.imports { - self.add_import(derived, import, shader_defs, header, already_added); - } - - Self::add_composable_data( - derived, - module, - Some(&import.items), - import_module_set.module_index << SPAN_SHIFT, - header, - ); - } - - fn ensure_import( - &mut self, - module_set: &ComposableModuleDefinition, - shader_defs: &HashMap, - ) -> Result { - let PreprocessOutput { - preprocessed_source, - imports, - } = self - .preprocessor - .preprocess(&module_set.sanitized_source, shader_defs, self.validate) - .map_err(|inner| ComposerError { - inner, - source: ErrSource::Module { - name: module_set.name.to_owned(), - offset: 0, - defs: shader_defs.clone(), - }, - })?; - - self.ensure_imports(imports.iter().map(|import| &import.definition), shader_defs)?; - self.ensure_imports(&module_set.additional_imports, shader_defs)?; - - self.create_composable_module( - module_set, - Self::decorate(&module_set.name), - shader_defs, - true, - true, - &preprocessed_source, - imports, - ) - } - - // build required ComposableModules for a given set of shader_defs - fn ensure_imports<'a>( - &mut self, - imports: impl IntoIterator, - shader_defs: &HashMap, - ) -> Result<(), ComposerError> { - for ImportDefinition { import, .. } in imports.into_iter() { - // we've already ensured imports exist when they were added - let module_set = self.module_sets.get(import).unwrap(); - if module_set.get_module(shader_defs).is_some() { - continue; - } - - // we need to build the module - // take the set so we can recurse without borrowing - let (set_key, mut module_set) = self.module_sets.remove_entry(import).unwrap(); - - match self.ensure_import(&module_set, shader_defs) { - Ok(module) => { - module_set.insert_module(shader_defs, module); - self.module_sets.insert(set_key, module_set); - } - Err(e) => { - self.module_sets.insert(set_key, module_set); - return Err(e); - } - } - } - - Ok(()) - } -} - #[derive(Default)] pub struct ComposableModuleDescriptor<'a> { pub source: &'a str, pub file_path: &'a str, pub language: ShaderLanguage, pub as_name: Option, - pub additional_imports: &'a [ImportDefinition], + pub additional_imports: &'a [AdditionalImport], pub shader_defs: HashMap, } @@ -1371,8 +194,8 @@ pub struct NagaModuleDescriptor<'a> { pub source: &'a str, pub file_path: &'a str, pub shader_type: ShaderType, + pub additional_imports: &'a [AdditionalImport], pub shader_defs: HashMap, - pub additional_imports: &'a [ImportDefinition], } // public api @@ -1399,304 +222,77 @@ impl Composer { } /// check if a module with the given name has been added - pub fn contains_module(&self, module_name: &str) -> bool { + pub fn contains_module(&self, module_name: &ModuleName) -> bool { self.module_sets.contains_key(module_name) } - /// add a composable module to the composer. - /// all modules imported by this module must already have been added + /// add a composable module to the composer pub fn add_composable_module( &mut self, desc: ComposableModuleDescriptor, ) -> Result<&ComposableModuleDefinition, ComposerError> { - let ComposableModuleDescriptor { - source, - file_path, - language, - as_name, - additional_imports, - mut shader_defs, - } = desc; - - // reject a module containing the DECORATION strings - if let Some(decor) = self.check_decoration_regex.find(source) { - return Err(ComposerError { - inner: ComposerErrorInner::DecorationInSource(decor.range()), - source: ErrSource::Constructing { - path: file_path.to_owned(), - source: source.to_owned(), - offset: 0, - }, - }); - } - - let substituted_source = self.sanitize_and_set_auto_bindings(source); - - let (_, parsed, errors) = preprocess1::preprocess - .recoverable_parse(winnow::Located::new(substituted_source.as_str())); - - if !errors.is_empty() { - return Err(ComposerError { - inner: ComposerErrorInner::PreprocessorError( - // TODO: Prettier error messages - errors - .into_iter() - .map(|v| v.to_string()) - .collect::>() - .into(), - ), - source: ErrSource::Constructing { - path: file_path.to_owned(), - source: source.to_owned(), - offset: 0, - }, - }); - } - let parsed = match parsed { - Some(parsed) => parsed, - None => { - return Err(ComposerError { - inner: ComposerErrorInner::PreprocessorError( - vec!["preprocessor failed to parse source".to_owned()].into(), - ), - source: ErrSource::Constructing { - path: file_path.to_owned(), - source: source.to_owned(), - offset: 0, - }, - }); - } - }; + let module_set = self.make_composable_module(desc)?; - let module_names = as_name - .into_iter() - .chain( - parsed - .get_module_names(&substituted_source) - .map(|v| v.to_owned()), - ) - .collect::>(); - if module_names.len() == 0 { + if self.module_sets.contains_key(&module_set.name) { return Err(ComposerError { - inner: ComposerErrorInner::NoModuleName, + inner: ComposerErrorInner::ModuleAlreadyExists(module_set.name.0.clone()), source: ErrSource::Constructing { - path: file_path.to_owned(), - source: source.to_owned(), + path: module_set.file_path.to_owned(), + source: module_set.source.to_owned(), offset: 0, }, }); } - if module_names.len() > 1 { - return Err(ComposerError { - inner: ComposerErrorInner::MultipleModuleNames(module_names.into()), - source: ErrSource::Constructing { - path: file_path.to_owned(), - source: source.to_owned(), - offset: 0, // TODO: Return the offset of the second module name - }, - }); - } - let module_name = module_names.into_iter().next().unwrap(); - - let all_imports = parsed.get_imports(&substituted_source); - - let used_defs = parsed.get_used_defs(&substituted_source); - /* - // TODO: What are the the ImportDefWithOffset s? - let imports: Vec; - // TODO: Why are effective_defs so weird? - let effective_defs: HashSet; - let PreprocessorMetaData { - mut imports, - mut effective_defs, - .. - } = self - .preprocessor - .get_preprocessor_metadata(&substituted_source, false) - */ - - debug!( - "adding module definition for {} with defs: {:?}", - module_name, used_defs - ); - - // add custom imports - let additional_imports = additional_imports.to_vec(); - imports.extend( - additional_imports - .iter() - .cloned() - .map(|def| ImportDefWithOffset { - definition: def, - offset: 0, - }), - ); - - for import in &imports { - // we require modules already added so that we can capture the shader_defs that may impact us by impacting our dependencies - let module_set = self - .module_sets - .get(&import.definition.import) - .ok_or_else(|| ComposerError { - inner: ComposerErrorInner::ImportNotFound( - import.definition.import.clone(), - import.offset, - ), - source: ErrSource::Constructing { - path: file_path.to_owned(), - source: substituted_source.to_owned(), - offset: 0, - }, - })?; - effective_defs.extend(module_set.effective_defs.iter().cloned()); - shader_defs.extend( - module_set - .shader_defs - .iter() - .map(|def| (def.0.clone(), *def.1)), - ); - } - - // remove defs that are already specified through our imports - effective_defs.retain(|name| !shader_defs.contains_key(name)); - - // can't gracefully report errors for more modules. perhaps this should be a warning - assert!((self.module_sets.len() as u32) < u32::MAX >> SPAN_SHIFT); - let module_index = self.module_sets.len() + 1; - - let module_set = ComposableModuleDefinition { - name: module_name.clone(), - sanitized_source: substituted_source, - file_path: file_path.to_owned(), - language, - effective_defs: effective_defs.into_iter().collect(), - all_imports: imports.into_iter().map(|id| id.definition.import).collect(), - additional_imports, - shader_defs, - module_index, - modules: Default::default(), - }; - - // invalidate dependent modules if this module already exists - self.remove_composable_module(&module_name); - - self.module_sets.insert(module_name.clone(), module_set); - self.module_index.insert(module_index, module_name.clone()); - Ok(self.module_sets.get(&module_name).unwrap()) + let name = module_set.name.clone(); + self.module_sets.insert(name.clone(), module_set); + Ok(self.module_sets.get(&name).unwrap()) } - /// remove a composable module. also removes modules that depend on this module, as we cannot be sure about - /// the completeness of their effective shader defs any more... - pub fn remove_composable_module(&mut self, module_name: &str) { - // todo this could be improved by making effective defs an Option and populating on demand? - let mut dependent_sets = Vec::new(); - - if self.module_sets.remove(module_name).is_some() { - dependent_sets.extend(self.module_sets.iter().filter_map(|(dependent_name, set)| { - if set.all_imports.contains(module_name) { - Some(dependent_name.clone()) - } else { - None - } - })); - } - - for dependent_set in dependent_sets { - self.remove_composable_module(&dependent_set); - } + /// remove a composable module + pub fn remove_composable_module(&mut self, module_name: &ModuleName) { + self.module_sets.remove(module_name); } + /// TODO: + /// - @binding(auto) for auto-binding + /// - virtual and override /// build a naga shader module pub fn make_naga_module( &mut self, desc: NagaModuleDescriptor, ) -> Result { - let NagaModuleDescriptor { - source, - file_path, - shader_type, - mut shader_defs, - additional_imports, - } = desc; + let definition = self.make_composable_module(ComposableModuleDescriptor { + source: desc.source, + file_path: desc.file_path, + language: desc.shader_type.into(), + as_name: None, + additional_imports: desc.additional_imports, + shader_defs: desc.shader_defs, + })?; + + let shader_defs = { + let defs = definition.shader_defs.clone(); + let mut all_imported_modules = HashSet::new(); + self.collect_all_imports(&definition.name, &mut all_imported_modules)?; + self.collect_shader_defs(&all_imported_modules, &mut defs); + defs + }; - let sanitized_source = self.sanitize_and_set_auto_bindings(source); + let processed = compose_parser::preprocess(self, &definition, &shader_defs)?; - let PreprocessorMetaData { - name, - defines, - imports, - .. - } = self - .preprocessor - .get_preprocessor_metadata(&sanitized_source, true) - .map_err(|inner| ComposerError { - inner, - source: ErrSource::Constructing { - path: file_path.to_owned(), - source: sanitized_source.to_owned(), - offset: 0, - }, - })?; - shader_defs.extend(defines); - - let name = name.unwrap_or_default(); + // TODO: + // - Replace all :: names with randomly generated names. Merge that logic into compose_parser::preprocess + // - Build naga modules, starting with the ones that have zero dependencies. + // - Then construct a header, and build the next module. Use the aliases here. + // - Name mangling. - // make sure imports have been added - // and gather additional defs specified at module level - for (import_name, offset) in imports - .iter() - .map(|id| (&id.definition.import, id.offset)) - .chain(additional_imports.iter().map(|ai| (&ai.import, 0))) - { - if let Some(module_set) = self.module_sets.get(import_name) { - for (def, value) in &module_set.shader_defs { - if let Some(prior_value) = shader_defs.insert(def.clone(), *value) { - if prior_value != *value { - return Err(ComposerError { - inner: ComposerErrorInner::InconsistentShaderDefValue { - def: def.clone(), - }, - source: ErrSource::Constructing { - path: file_path.to_owned(), - source: sanitized_source.to_owned(), - offset: 0, - }, - }); - } - } - } - } else { - return Err(ComposerError { - inner: ComposerErrorInner::ImportNotFound(import_name.clone(), offset), - source: ErrSource::Constructing { - path: file_path.to_owned(), - source: sanitized_source, - offset: 0, - }, - }); - } - } self.ensure_imports( imports.iter().map(|import| &import.definition), &shader_defs, )?; self.ensure_imports(additional_imports, &shader_defs)?; - let definition = ComposableModuleDefinition { - name, - sanitized_source: sanitized_source.clone(), - language: shader_type.into(), - file_path: file_path.to_owned(), - module_index: 0, - additional_imports: additional_imports.to_vec(), - // we don't care about these for creating a top-level module - effective_defs: Default::default(), - all_imports: Default::default(), - shader_defs: Default::default(), - modules: Default::default(), - }; - let PreprocessOutput { preprocessed_source, imports, @@ -1746,7 +342,7 @@ impl Composer { Self::add_composable_data(&mut derived, &composable, None, 0, false); - let stage = match shader_type { + let stage = match desc.shader_type { #[cfg(feature = "glsl")] ShaderType::GlslVertex => Some(naga::ShaderStage::Vertex), #[cfg(feature = "glsl")] @@ -1861,6 +457,7 @@ impl Composer { } } +/* TODO: Implement this /// Get module name and all required imports (ignoring shader_defs) from a shader string pub fn get_preprocessor_data( source: &str, @@ -1869,6 +466,13 @@ pub fn get_preprocessor_data( Vec, HashMap, ) { + let (_, parsed, errors) = + preprocess1::preprocess.recoverable_parse(winnow::Located::new(source)); + + // Returning the defines correctly is impossible at the moment. + todo!() + + if let Ok(PreprocessorMetaData { name, imports, @@ -1888,4 +492,4 @@ pub fn get_preprocessor_data( // if errors occur we return nothing; the actual error will be displayed when the caller attempts to use the shader Default::default() } -} +}*/ diff --git a/src/compose/old_parse_imports.rs b/src/compose/old_parse_imports.rs new file mode 100644 index 0000000..8bdfc63 --- /dev/null +++ b/src/compose/old_parse_imports.rs @@ -0,0 +1,189 @@ +use std::ops::Range; + +use indexmap::IndexMap; +use winnow::{stream::Recoverable, Located, Parser}; + +use crate::compose::preprocess1::{import_directive, ImportTree}; + +use super::{ + composer::{ImportDefWithOffset, ImportDefinition}, + tokenizer::{Token, Tokenizer}, + Composer, +}; + +#[cfg(test)] +fn test_parse(input: &str) -> Result>, (&str, usize)> { + let mut declared_imports = IndexMap::default(); + parse_imports(input, &mut declared_imports)?; + Ok(declared_imports) +} + +#[test] +fn import_tokens() { + let input = r" + #import a::b + "; + assert_eq!( + test_parse(input), + Ok(IndexMap::from_iter([( + "b".to_owned(), + vec!("a::b".to_owned()) + )])) + ); + + let input = r" + #import a::{b, c} + "; + assert_eq!( + test_parse(input), + Ok(IndexMap::from_iter([ + ("b".to_owned(), vec!("a::b".to_owned())), + ("c".to_owned(), vec!("a::c".to_owned())), + ])) + ); + + let input = r" + #import a::{b as d, c} + "; + assert_eq!( + test_parse(input), + Ok(IndexMap::from_iter([ + ("d".to_owned(), vec!("a::b".to_owned())), + ("c".to_owned(), vec!("a::c".to_owned())), + ])) + ); + + let input = r" + #import a::{b::{c, d}, e} + "; + assert_eq!( + test_parse(input), + Ok(IndexMap::from_iter([ + ("c".to_owned(), vec!("a::b::c".to_owned())), + ("d".to_owned(), vec!("a::b::d".to_owned())), + ("e".to_owned(), vec!("a::e".to_owned())), + ])) + ); + + let input = r" + #import a::b::{c, d}, e + "; + assert_eq!( + test_parse(input), + Ok(IndexMap::from_iter([ + ("c".to_owned(), vec!("a::b::c".to_owned())), + ("d".to_owned(), vec!("a::b::d".to_owned())), + ("e".to_owned(), vec!("e".to_owned())), + ])) + ); + + let input = r" + #import a, b + "; + assert_eq!( + test_parse(input), + Ok(IndexMap::from_iter([ + ("a".to_owned(), vec!("a".to_owned())), + ("b".to_owned(), vec!("b".to_owned())), + ])) + ); + + let input = r" + #import a::b c, d + "; + assert_eq!( + test_parse(input), + Ok(IndexMap::from_iter([ + ("c".to_owned(), vec!("a::b::c".to_owned())), + ("d".to_owned(), vec!("a::b::d".to_owned())), + ])) + ); + + let input = r" + #import a::b c + "; + assert_eq!( + test_parse(input), + Ok(IndexMap::from_iter([( + "c".to_owned(), + vec!("a::b::c".to_owned()) + ),])) + ); + + let input = r" + #import a::b::{c::{d, e}, f, g::{h as i, j}} + "; + assert_eq!( + test_parse(input), + Ok(IndexMap::from_iter([ + ("d".to_owned(), vec!("a::b::c::d".to_owned())), + ("e".to_owned(), vec!("a::b::c::e".to_owned())), + ("f".to_owned(), vec!("a::b::f".to_owned())), + ("i".to_owned(), vec!("a::b::g::h".to_owned())), + ("j".to_owned(), vec!("a::b::g::j".to_owned())), + ])) + ); + + let input = r" + #import a::b::{ + c::{d, e}, + f, + g::{ + h as i, + j::k::l as m, + } + } + "; + assert_eq!( + test_parse(input), + Ok(IndexMap::from_iter([ + ("d".to_owned(), vec!("a::b::c::d".to_owned())), + ("e".to_owned(), vec!("a::b::c::e".to_owned())), + ("f".to_owned(), vec!("a::b::f".to_owned())), + ("i".to_owned(), vec!("a::b::g::h".to_owned())), + ("m".to_owned(), vec!("a::b::g::j::k::l".to_owned())), + ])) + ); + + let input = r#" + #import "path//with\ all sorts of .stuff"::{a, b} + "#; + assert_eq!( + test_parse(input), + Ok(IndexMap::from_iter([ + ( + "a".to_owned(), + vec!(r#""path//with\ all sorts of .stuff"::a"#.to_owned()) + ), + ( + "b".to_owned(), + vec!(r#""path//with\ all sorts of .stuff"::b"#.to_owned()) + ), + ])) + ); + + let input = r" + #import a::b::{ + "; + assert!(test_parse(input).is_err()); + + let input = r" + #import a::b::{{c} + "; + assert!(test_parse(input).is_err()); + + let input = r" + #import a::b::{c}} + "; + assert!(test_parse(input).is_err()); + + let input = r" + #import a::b{{c,d}} + "; + assert!(test_parse(input).is_err()); + + let input = r" + #import a:b + "; + assert!(test_parse(input).is_err()); +} diff --git a/src/compose/old_preprocess.rs b/src/compose/old_preprocess.rs new file mode 100644 index 0000000..fb7fc47 --- /dev/null +++ b/src/compose/old_preprocess.rs @@ -0,0 +1,1085 @@ +use std::collections::{HashMap, HashSet}; + +use indexmap::IndexMap; +use winnow::Parser; + +use super::{ + composer::ImportDefWithOffset, + old_parse_imports::substitute_identifiers, + preprocess1::{self, input_new}, + ComposerErrorInner, ShaderDefValue, +}; + +#[derive(Debug)] +pub struct Preprocessor; + +#[derive(Debug)] +pub struct PreprocessorMetaData { + pub name: Option, + pub imports: Vec, + pub defines: HashMap, + pub effective_defs: HashSet, +} + +#[cfg(test)] +mod test { + use super::*; + + #[rustfmt::skip] + const WGSL_ELSE_IFDEF: &str = r" +struct View { + view_proj: mat4x4, + world_position: vec3, +}; +@group(0) @binding(0) +var view: View; + +#ifdef TEXTURE +// Main texture +@group(1) @binding(0) +var sprite_texture: texture_2d; +#else ifdef SECOND_TEXTURE +// Second texture +@group(1) @binding(0) +var sprite_texture: texture_2d; +#else ifdef THIRD_TEXTURE +// Third texture +@group(1) @binding(0) +var sprite_texture: texture_2d; +#else +@group(1) @binding(0) +var sprite_texture: texture_2d_array; +#endif + +struct VertexOutput { + @location(0) uv: vec2, + @builtin(position) position: vec4, +}; + +@vertex +fn vertex( + @location(0) vertex_position: vec3, + @location(1) vertex_uv: vec2 +) -> VertexOutput { + var out: VertexOutput; + out.uv = vertex_uv; + out.position = view.view_proj * vec4(vertex_position, 1.0); + return out; +} +"; + + //preprocessor tests + #[test] + fn process_shader_def_unknown_operator() { + #[rustfmt::skip] + const WGSL: &str = r" +struct View { + view_proj: mat4x4, + world_position: vec3, +}; +@group(0) @binding(0) +var view: View; +#if TEXTURE !! true +@group(1) @binding(0) +var sprite_texture: texture_2d; +#endif +struct VertexOutput { + @location(0) uv: vec2, + @builtin(position) position: vec4, +}; +@vertex +fn vertex( + @location(0) vertex_position: vec3, + @location(1) vertex_uv: vec2 +) -> VertexOutput { + var out: VertexOutput; + out.uv = vertex_uv; + out.position = view.view_proj * vec4(vertex_position, 1.0); + return out; +} +"; + + let processor = Preprocessor::default(); + + let result_missing = processor.preprocess( + WGSL, + &[("TEXTURE".to_owned(), ShaderDefValue::Bool(true))].into(), + true, + ); + + let expected: Result = + Err(ComposerErrorInner::UnknownShaderDefOperator { + pos: 124, + operator: "!!".to_string(), + }); + + assert_eq!(format!("{result_missing:?}"), format!("{expected:?}"),); + } + #[test] + fn process_shader_def_equal_int() { + #[rustfmt::skip] + const WGSL: &str = r" +struct View { + view_proj: mat4x4, + world_position: vec3, +}; +@group(0) @binding(0) +var view: View; +#if TEXTURE == 3 +@group(1) @binding(0) +var sprite_texture: texture_2d; +#endif +struct VertexOutput { + @location(0) uv: vec2, + @builtin(position) position: vec4, +}; +@vertex +fn vertex( + @location(0) vertex_position: vec3, + @location(1) vertex_uv: vec2 +) -> VertexOutput { + var out: VertexOutput; + out.uv = vertex_uv; + out.position = view.view_proj * vec4(vertex_position, 1.0); + return out; +} +"; + + #[rustfmt::skip] + const EXPECTED_EQ: &str = r" +struct View { + view_proj: mat4x4, + world_position: vec3, +}; +@group(0) @binding(0) +var view: View; + +@group(1) @binding(0) +var sprite_texture: texture_2d; + +struct VertexOutput { + @location(0) uv: vec2, + @builtin(position) position: vec4, +}; +@vertex +fn vertex( + @location(0) vertex_position: vec3, + @location(1) vertex_uv: vec2 +) -> VertexOutput { + var out: VertexOutput; + out.uv = vertex_uv; + out.position = view.view_proj * vec4(vertex_position, 1.0); + return out; +} +"; + + #[rustfmt::skip] + const EXPECTED_NEQ: &str = r" +struct View { + view_proj: mat4x4, + world_position: vec3, +}; +@group(0) @binding(0) +var view: View; + + + + +struct VertexOutput { + @location(0) uv: vec2, + @builtin(position) position: vec4, +}; +@vertex +fn vertex( + @location(0) vertex_position: vec3, + @location(1) vertex_uv: vec2 +) -> VertexOutput { + var out: VertexOutput; + out.uv = vertex_uv; + out.position = view.view_proj * vec4(vertex_position, 1.0); + return out; +} +"; + let processor = Preprocessor::default(); + let result_eq = processor + .preprocess( + WGSL, + &[("TEXTURE".to_string(), ShaderDefValue::Int(3))].into(), + true, + ) + .unwrap(); + assert_eq!(result_eq.preprocessed_source, EXPECTED_EQ); + + let result_neq = processor + .preprocess( + WGSL, + &[("TEXTURE".to_string(), ShaderDefValue::Int(7))].into(), + true, + ) + .unwrap(); + assert_eq!(result_neq.preprocessed_source, EXPECTED_NEQ); + + let result_missing = processor.preprocess(WGSL, &Default::default(), true); + + let expected_err: Result< + (Option, String, Vec), + ComposerErrorInner, + > = Err(ComposerErrorInner::UnknownShaderDef { + pos: 124, + shader_def_name: "TEXTURE".to_string(), + }); + assert_eq!(format!("{result_missing:?}"), format!("{expected_err:?}"),); + + let result_wrong_type = processor.preprocess( + WGSL, + &[("TEXTURE".to_string(), ShaderDefValue::Bool(true))].into(), + true, + ); + + let expected_err: Result< + (Option, String, Vec), + ComposerErrorInner, + > = Err(ComposerErrorInner::InvalidShaderDefComparisonValue { + pos: 124, + shader_def_name: "TEXTURE".to_string(), + expected: "bool".to_string(), + value: "3".to_string(), + }); + + assert_eq!( + format!("{result_wrong_type:?}"), + format!("{expected_err:?}") + ); + } + + #[test] + fn process_shader_def_equal_bool() { + #[rustfmt::skip] + const WGSL: &str = r" +struct View { + view_proj: mat4x4, + world_position: vec3, +}; +@group(0) @binding(0) +var view: View; +#if TEXTURE == true +@group(1) @binding(0) +var sprite_texture: texture_2d; +#endif +struct VertexOutput { + @location(0) uv: vec2, + @builtin(position) position: vec4, +}; +@vertex +fn vertex( + @location(0) vertex_position: vec3, + @location(1) vertex_uv: vec2 +) -> VertexOutput { + var out: VertexOutput; + out.uv = vertex_uv; + out.position = view.view_proj * vec4(vertex_position, 1.0); + return out; +} +"; + + #[rustfmt::skip] + const EXPECTED_EQ: &str = r" +struct View { + view_proj: mat4x4, + world_position: vec3, +}; +@group(0) @binding(0) +var view: View; + +@group(1) @binding(0) +var sprite_texture: texture_2d; + +struct VertexOutput { + @location(0) uv: vec2, + @builtin(position) position: vec4, +}; +@vertex +fn vertex( + @location(0) vertex_position: vec3, + @location(1) vertex_uv: vec2 +) -> VertexOutput { + var out: VertexOutput; + out.uv = vertex_uv; + out.position = view.view_proj * vec4(vertex_position, 1.0); + return out; +} +"; + + #[rustfmt::skip] + const EXPECTED_NEQ: &str = r" +struct View { + view_proj: mat4x4, + world_position: vec3, +}; +@group(0) @binding(0) +var view: View; + + + + +struct VertexOutput { + @location(0) uv: vec2, + @builtin(position) position: vec4, +}; +@vertex +fn vertex( + @location(0) vertex_position: vec3, + @location(1) vertex_uv: vec2 +) -> VertexOutput { + var out: VertexOutput; + out.uv = vertex_uv; + out.position = view.view_proj * vec4(vertex_position, 1.0); + return out; +} +"; + let processor = Preprocessor::default(); + let result_eq = processor + .preprocess( + WGSL, + &[("TEXTURE".to_string(), ShaderDefValue::Bool(true))].into(), + true, + ) + .unwrap(); + assert_eq!(result_eq.preprocessed_source, EXPECTED_EQ); + + let result_neq = processor + .preprocess( + WGSL, + &[("TEXTURE".to_string(), ShaderDefValue::Bool(false))].into(), + true, + ) + .unwrap(); + assert_eq!(result_neq.preprocessed_source, EXPECTED_NEQ); + } + + #[test] + fn process_shader_def_not_equal_bool() { + #[rustfmt::skip] + const WGSL: &str = r" +struct View { + view_proj: mat4x4, + world_position: vec3, +}; +@group(0) @binding(0) +var view: View; +#if TEXTURE != false +@group(1) @binding(0) +var sprite_texture: texture_2d; +#endif +struct VertexOutput { + @location(0) uv: vec2, + @builtin(position) position: vec4, +}; +@vertex +fn vertex( + @location(0) vertex_position: vec3, + @location(1) vertex_uv: vec2 +) -> VertexOutput { + var out: VertexOutput; + out.uv = vertex_uv; + out.position = view.view_proj * vec4(vertex_position, 1.0); + return out; +} +"; + + #[rustfmt::skip] + const EXPECTED_EQ: &str = r" +struct View { + view_proj: mat4x4, + world_position: vec3, +}; +@group(0) @binding(0) +var view: View; + +@group(1) @binding(0) +var sprite_texture: texture_2d; + +struct VertexOutput { + @location(0) uv: vec2, + @builtin(position) position: vec4, +}; +@vertex +fn vertex( + @location(0) vertex_position: vec3, + @location(1) vertex_uv: vec2 +) -> VertexOutput { + var out: VertexOutput; + out.uv = vertex_uv; + out.position = view.view_proj * vec4(vertex_position, 1.0); + return out; +} +"; + + #[rustfmt::skip] + const EXPECTED_NEQ: &str = r" +struct View { + view_proj: mat4x4, + world_position: vec3, +}; +@group(0) @binding(0) +var view: View; + + + + +struct VertexOutput { + @location(0) uv: vec2, + @builtin(position) position: vec4, +}; +@vertex +fn vertex( + @location(0) vertex_position: vec3, + @location(1) vertex_uv: vec2 +) -> VertexOutput { + var out: VertexOutput; + out.uv = vertex_uv; + out.position = view.view_proj * vec4(vertex_position, 1.0); + return out; +} +"; + let processor = Preprocessor::default(); + let result_eq = processor + .preprocess( + WGSL, + &[("TEXTURE".to_string(), ShaderDefValue::Bool(true))].into(), + true, + ) + .unwrap(); + assert_eq!(result_eq.preprocessed_source, EXPECTED_EQ); + + let result_neq = processor + .preprocess( + WGSL, + &[("TEXTURE".to_string(), ShaderDefValue::Bool(false))].into(), + true, + ) + .unwrap(); + assert_eq!(result_neq.preprocessed_source, EXPECTED_NEQ); + + let result_missing = processor.preprocess(WGSL, &[].into(), true); + let expected_err: Result< + (Option, String, Vec), + ComposerErrorInner, + > = Err(ComposerErrorInner::UnknownShaderDef { + pos: 124, + shader_def_name: "TEXTURE".to_string(), + }); + assert_eq!(format!("{result_missing:?}"), format!("{expected_err:?}"),); + + let result_wrong_type = processor.preprocess( + WGSL, + &[("TEXTURE".to_string(), ShaderDefValue::Int(7))].into(), + true, + ); + + let expected_err: Result< + (Option, String, Vec), + ComposerErrorInner, + > = Err(ComposerErrorInner::InvalidShaderDefComparisonValue { + pos: 124, + shader_def_name: "TEXTURE".to_string(), + expected: "int".to_string(), + value: "false".to_string(), + }); + assert_eq!( + format!("{result_wrong_type:?}"), + format!("{expected_err:?}"), + ); + } + + #[test] + fn process_shader_def_replace() { + #[rustfmt::skip] + const WGSL: &str = r" +struct View { + view_proj: mat4x4, + world_position: vec3, +}; +@group(0) @binding(0) +var view: View; +struct VertexOutput { + @location(0) uv: vec2, + @builtin(position) position: vec4, +}; +@vertex +fn vertex( + @location(0) vertex_position: vec3, + @location(1) vertex_uv: vec2 +) -> VertexOutput { + var out: VertexOutput; + out.uv = vertex_uv; + var a: i32 = #FIRST_VALUE; + var b: i32 = #FIRST_VALUE * #SECOND_VALUE; + var c: i32 = #MISSING_VALUE; + var d: bool = #BOOL_VALUE; + out.position = view.view_proj * vec4(vertex_position, 1.0); + return out; +} +"; + + #[rustfmt::skip] + const EXPECTED_REPLACED: &str = r" +struct View { + view_proj: mat4x4, + world_position: vec3, +}; +@group(0) @binding(0) +var view: View; +struct VertexOutput { + @location(0) uv: vec2, + @builtin(position) position: vec4, +}; +@vertex +fn vertex( + @location(0) vertex_position: vec3, + @location(1) vertex_uv: vec2 +) -> VertexOutput { + var out: VertexOutput; + out.uv = vertex_uv; + var a: i32 = 5; + var b: i32 = 5 * 3; + var c: i32 = #MISSING_VALUE; + var d: bool = true; + out.position = view.view_proj * vec4(vertex_position, 1.0); + return out; +} +"; + let processor = Preprocessor::default(); + let result = processor + .preprocess( + WGSL, + &[ + ("BOOL_VALUE".to_string(), ShaderDefValue::Bool(true)), + ("FIRST_VALUE".to_string(), ShaderDefValue::Int(5)), + ("SECOND_VALUE".to_string(), ShaderDefValue::Int(3)), + ] + .into(), + true, + ) + .unwrap(); + assert_eq!(result.preprocessed_source, EXPECTED_REPLACED); + } + + #[test] + fn process_shader_define_in_shader() { + #[rustfmt::skip] + const WGSL: &str = r" +#define NOW_DEFINED +#ifdef NOW_DEFINED +defined +#endif +"; + + #[rustfmt::skip] + const EXPECTED: &str = r" + + +defined + +"; + let processor = Preprocessor::default(); + let PreprocessorMetaData { + defines: shader_defs, + .. + } = processor.get_preprocessor_metadata(&WGSL, true).unwrap(); + println!("defines: {:?}", shader_defs); + let result = processor.preprocess(&WGSL, &shader_defs, true).unwrap(); + assert_eq!(result.preprocessed_source, EXPECTED); + } + + #[test] + fn process_shader_define_in_shader_with_value() { + #[rustfmt::skip] + const WGSL: &str = r" +#define DEFUINT 1 +#define DEFINT -1 +#define DEFBOOL false +#if DEFUINT == 1 +uint: #DEFUINT +#endif +#if DEFINT == -1 +int: #DEFINT +#endif +#if DEFBOOL == false +bool: #DEFBOOL +#endif +"; + + #[rustfmt::skip] + const EXPECTED: &str = r" + + + + +uint: 1 + + +int: -1 + + +bool: false + +"; + let processor = Preprocessor::default(); + let PreprocessorMetaData { + defines: shader_defs, + .. + } = processor.get_preprocessor_metadata(&WGSL, true).unwrap(); + println!("defines: {:?}", shader_defs); + let result = processor.preprocess(&WGSL, &shader_defs, true).unwrap(); + assert_eq!(result.preprocessed_source, EXPECTED); + } + + #[test] + fn process_shader_def_else_ifdef_ends_up_in_else() { + #[rustfmt::skip] + const EXPECTED: &str = r" +struct View { + view_proj: mat4x4, + world_position: vec3, +}; +@group(0) @binding(0) +var view: View; +@group(1) @binding(0) +var sprite_texture: texture_2d_array; +struct VertexOutput { + @location(0) uv: vec2, + @builtin(position) position: vec4, +}; +@vertex +fn vertex( + @location(0) vertex_position: vec3, + @location(1) vertex_uv: vec2 +) -> VertexOutput { + var out: VertexOutput; + out.uv = vertex_uv; + out.position = view.view_proj * vec4(vertex_position, 1.0); + return out; +} +"; + let processor = Preprocessor::default(); + let result = processor + .preprocess(&WGSL_ELSE_IFDEF, &[].into(), true) + .unwrap(); + assert_eq!( + result + .preprocessed_source + .replace(" ", "") + .replace("\n", "") + .replace("\r", ""), + EXPECTED + .replace(" ", "") + .replace("\n", "") + .replace("\r", "") + ); + } + + #[test] + fn process_shader_def_else_ifdef_no_match_and_no_fallback_else() { + #[rustfmt::skip] + const WGSL_ELSE_IFDEF_NO_ELSE_FALLBACK: &str = r" +struct View { + view_proj: mat4x4, + world_position: vec3, +}; +@group(0) @binding(0) +var view: View; + +#ifdef TEXTURE +// Main texture +@group(1) @binding(0) +var sprite_texture: texture_2d; +#else ifdef OTHER_TEXTURE +// Other texture +@group(1) @binding(0) +var sprite_texture: texture_2d; +#endif + +struct VertexOutput { + @location(0) uv: vec2, + @builtin(position) position: vec4, +}; + +@vertex +fn vertex( + @location(0) vertex_position: vec3, + @location(1) vertex_uv: vec2 +) -> VertexOutput { + var out: VertexOutput; + out.uv = vertex_uv; + out.position = view.view_proj * vec4(vertex_position, 1.0); + return out; +} +"; + + #[rustfmt::skip] + const EXPECTED: &str = r" +struct View { + view_proj: mat4x4, + world_position: vec3, +}; +@group(0) @binding(0) +var view: View; +struct VertexOutput { + @location(0) uv: vec2, + @builtin(position) position: vec4, +}; +@vertex +fn vertex( + @location(0) vertex_position: vec3, + @location(1) vertex_uv: vec2 +) -> VertexOutput { + var out: VertexOutput; + out.uv = vertex_uv; + out.position = view.view_proj * vec4(vertex_position, 1.0); + return out; +} +"; + let processor = Preprocessor::default(); + let result = processor + .preprocess(&WGSL_ELSE_IFDEF_NO_ELSE_FALLBACK, &[].into(), true) + .unwrap(); + assert_eq!( + result + .preprocessed_source + .replace(" ", "") + .replace("\n", "") + .replace("\r", ""), + EXPECTED + .replace(" ", "") + .replace("\n", "") + .replace("\r", "") + ); + } + + #[test] + fn process_shader_def_else_ifdef_ends_up_in_first_clause() { + #[rustfmt::skip] + const EXPECTED: &str = r" +struct View { + view_proj: mat4x4, + world_position: vec3, +}; +@group(0) @binding(0) +var view: View; + +// Main texture +@group(1) @binding(0) +var sprite_texture: texture_2d; + +struct VertexOutput { + @location(0) uv: vec2, + @builtin(position) position: vec4, +}; + +@vertex +fn vertex( + @location(0) vertex_position: vec3, + @location(1) vertex_uv: vec2 +) -> VertexOutput { + var out: VertexOutput; + out.uv = vertex_uv; + out.position = view.view_proj * vec4(vertex_position, 1.0); + return out; +} +"; + let processor = Preprocessor::default(); + let result = processor + .preprocess( + &WGSL_ELSE_IFDEF, + &[("TEXTURE".to_string(), ShaderDefValue::Bool(true))].into(), + true, + ) + .unwrap(); + assert_eq!( + result + .preprocessed_source + .replace(" ", "") + .replace("\n", "") + .replace("\r", ""), + EXPECTED + .replace(" ", "") + .replace("\n", "") + .replace("\r", "") + ); + } + + #[test] + fn process_shader_def_else_ifdef_ends_up_in_second_clause() { + #[rustfmt::skip] + const EXPECTED: &str = r" +struct View { + view_proj: mat4x4, + world_position: vec3, +}; +@group(0) @binding(0) +var view: View; +// Second texture +@group(1) @binding(0) +var sprite_texture: texture_2d; +struct VertexOutput { + @location(0) uv: vec2, + @builtin(position) position: vec4, +}; +@vertex +fn vertex( + @location(0) vertex_position: vec3, + @location(1) vertex_uv: vec2 +) -> VertexOutput { + var out: VertexOutput; + out.uv = vertex_uv; + out.position = view.view_proj * vec4(vertex_position, 1.0); + return out; +} +"; + let processor = Preprocessor::default(); + let result = processor + .preprocess( + &WGSL_ELSE_IFDEF, + &[("SECOND_TEXTURE".to_string(), ShaderDefValue::Bool(true))].into(), + true, + ) + .unwrap(); + assert_eq!( + result + .preprocessed_source + .replace(" ", "") + .replace("\n", "") + .replace("\r", ""), + EXPECTED + .replace(" ", "") + .replace("\n", "") + .replace("\r", "") + ); + } + + #[test] + fn process_shader_def_else_ifdef_ends_up_in_third_clause() { + #[rustfmt::skip] + const EXPECTED: &str = r" +struct View { + view_proj: mat4x4, + world_position: vec3, +}; +@group(0) @binding(0) +var view: View; +// Third texture +@group(1) @binding(0) +var sprite_texture: texture_2d; +struct VertexOutput { + @location(0) uv: vec2, + @builtin(position) position: vec4, +}; +@vertex +fn vertex( + @location(0) vertex_position: vec3, + @location(1) vertex_uv: vec2 +) -> VertexOutput { + var out: VertexOutput; + out.uv = vertex_uv; + out.position = view.view_proj * vec4(vertex_position, 1.0); + return out; +} +"; + let processor = Preprocessor::default(); + let result = processor + .preprocess( + &WGSL_ELSE_IFDEF, + &[("THIRD_TEXTURE".to_string(), ShaderDefValue::Bool(true))].into(), + true, + ) + .unwrap(); + assert_eq!( + result + .preprocessed_source + .replace(" ", "") + .replace("\n", "") + .replace("\r", ""), + EXPECTED + .replace(" ", "") + .replace("\n", "") + .replace("\r", "") + ); + } + + #[test] + fn process_shader_def_else_ifdef_only_accepts_one_valid_else_ifdef() { + #[rustfmt::skip] + const EXPECTED: &str = r" +struct View { + view_proj: mat4x4, + world_position: vec3, +}; +@group(0) @binding(0) +var view: View; +// Second texture +@group(1) @binding(0) +var sprite_texture: texture_2d; +struct VertexOutput { + @location(0) uv: vec2, + @builtin(position) position: vec4, +}; +@vertex +fn vertex( + @location(0) vertex_position: vec3, + @location(1) vertex_uv: vec2 +) -> VertexOutput { + var out: VertexOutput; + out.uv = vertex_uv; + out.position = view.view_proj * vec4(vertex_position, 1.0); + return out; +} +"; + let processor = Preprocessor::default(); + let result = processor + .preprocess( + &WGSL_ELSE_IFDEF, + &[ + ("SECOND_TEXTURE".to_string(), ShaderDefValue::Bool(true)), + ("THIRD_TEXTURE".to_string(), ShaderDefValue::Bool(true)), + ] + .into(), + true, + ) + .unwrap(); + assert_eq!( + result + .preprocessed_source + .replace(" ", "") + .replace("\n", "") + .replace("\r", ""), + EXPECTED + .replace(" ", "") + .replace("\n", "") + .replace("\r", "") + ); + } + + #[test] + fn process_shader_def_else_ifdef_complicated_nesting() { + // Test some nesting including #else ifdef statements + // 1. Enter an #else ifdef + // 2. Then enter an #else + // 3. Then enter another #else ifdef + + #[rustfmt::skip] + const WGSL_COMPLICATED_ELSE_IFDEF: &str = r" +#ifdef NOT_DEFINED +// not defined +#else ifdef IS_DEFINED +// defined 1 +#ifdef NOT_DEFINED +// not defined +#else +// should be here +#ifdef NOT_DEFINED +// not defined +#else ifdef ALSO_NOT_DEFINED +// not defined +#else ifdef IS_DEFINED +// defined 2 +#endif +#endif +#endif +"; + + #[rustfmt::skip] + const EXPECTED: &str = r" +// defined 1 +// should be here +// defined 2 +"; + let processor = Preprocessor::default(); + let result = processor + .preprocess( + &WGSL_COMPLICATED_ELSE_IFDEF, + &[("IS_DEFINED".to_string(), ShaderDefValue::Bool(true))].into(), + true, + ) + .unwrap(); + assert_eq!( + result + .preprocessed_source + .replace(" ", "") + .replace("\n", "") + .replace("\r", ""), + EXPECTED + .replace(" ", "") + .replace("\n", "") + .replace("\r", "") + ); + } + + #[test] + fn process_shader_def_else_ifndef() { + #[rustfmt::skip] + const INPUT: &str = r" +#ifdef NOT_DEFINED +fail 1 +#else ifdef ALSO_NOT_DEFINED +fail 2 +#else ifndef ALSO_ALSO_NOT_DEFINED +ok +#else +fail 3 +#endif +"; + + const EXPECTED: &str = r"ok"; + let processor = Preprocessor::default(); + let result = processor.preprocess(&INPUT, &[].into(), true).unwrap(); + assert_eq!( + result + .preprocessed_source + .replace(" ", "") + .replace("\n", "") + .replace("\r", ""), + EXPECTED + .replace(" ", "") + .replace("\n", "") + .replace("\r", "") + ); + } + + #[test] + fn process_shader_def_else_if() { + #[rustfmt::skip] + const INPUT: &str = r" +#ifdef NOT_DEFINED +fail 1 +#else if x == 1 +fail 2 +#else if x == 2 +ok +#else +fail 3 +#endif +"; + + const EXPECTED: &str = r"ok"; + let processor = Preprocessor::default(); + let result = processor + .preprocess( + &INPUT, + &[("x".to_owned(), ShaderDefValue::Int(2))].into(), + true, + ) + .unwrap(); + assert_eq!( + result + .preprocessed_source + .replace(" ", "") + .replace("\n", "") + .replace("\r", ""), + EXPECTED + .replace(" ", "") + .replace("\n", "") + .replace("\r", "") + ); + } +} diff --git a/src/compose/parse_imports.rs b/src/compose/parse_imports.rs deleted file mode 100644 index 6068944..0000000 --- a/src/compose/parse_imports.rs +++ /dev/null @@ -1,320 +0,0 @@ -use std::ops::Range; - -use indexmap::IndexMap; -use winnow::{stream::Recoverable, Located, Parser}; - -use crate::compose::preprocess1::{import_directive, ImportTree}; - -use super::{ - tokenizer::{Token, Tokenizer}, - Composer, ImportDefWithOffset, ImportDefinition, -}; - -pub fn parse_imports<'a>( - input: &'a str, - declared_imports: &mut IndexMap>, -) -> Result<(), (&'a str, usize)> { - let input = input.trim(); - let imports = import_directive - .parse(Recoverable::new(Located::new(input))) - .map_err(|_v| { - // panic!("{:#?}", _v); - ("failed to parse imports", 0) - })?; - - fn to_stack<'a>(input: &'a str, ranges: &[Range]) -> Vec<&'a str> { - ranges - .iter() - .map(|range| &input[range.clone()]) - .collect::>() - } - - fn walk_import_tree<'a>( - input: &'a str, - tree: &ImportTree, - stack: &[&'a str], - declared_imports: &mut IndexMap>, - ) { - let (name_range, path_ranges) = match tree { - ImportTree::Path(path_ranges) => (path_ranges.last().unwrap().clone(), path_ranges), - ImportTree::Alias { - path: path_ranges, - alias: alias_range, - } => (alias_range.clone(), path_ranges), - ImportTree::Children { path, children } => { - let extended_stack = [stack, &to_stack(input, path)].concat(); - for child in children { - walk_import_tree(input, child, &extended_stack, declared_imports); - } - return; - } - }; - - let name = input[name_range].to_string(); - let extended_stack = [stack, &to_stack(input, &path_ranges)].concat(); - declared_imports - .entry(name) - .or_default() - .push(extended_stack.join("::")); - } - - walk_import_tree(input, &imports.tree, &[], declared_imports); - - Ok(()) -} - -pub fn substitute_identifiers( - input: &str, - offset: usize, - declared_imports: &IndexMap>, - used_imports: &mut IndexMap, - allow_ambiguous: bool, -) -> Result { - let tokens = Tokenizer::new(input, true); - let mut output = String::with_capacity(input.len()); - let mut in_substitution_position = true; - - for token in tokens { - match token { - Token::Identifier(ident, token_pos) => { - if in_substitution_position { - let (first, residual) = ident.split_once("::").unwrap_or((ident, "")); - let full_paths = declared_imports - .get(first) - .cloned() - .unwrap_or(vec![first.to_owned()]); - - if !allow_ambiguous && full_paths.len() > 1 { - return Err(offset + token_pos); - } - - for mut full_path in full_paths { - if !residual.is_empty() { - full_path.push_str("::"); - full_path.push_str(residual); - } - - if let Some((module, item)) = full_path.rsplit_once("::") { - used_imports - .entry(module.to_owned()) - .or_insert_with(|| ImportDefWithOffset { - definition: ImportDefinition { - import: module.to_owned(), - ..Default::default() - }, - offset: offset + token_pos, - }) - .definition - .items - .push(item.to_owned()); - output.push_str(item); - output.push_str(&Composer::decorate(module)); - } else if full_path.find('"').is_some() { - // we don't want to replace local variables that shadow quoted module imports with the - // quoted name as that won't compile. - // since quoted items always refer to modules, we can just emit the original ident - // in this case - output.push_str(ident); - } else { - // if there are no quotes we do the replacement. this means that individually imported - // items can be used, and any shadowing local variables get harmlessly renamed. - // TODO: it can lead to weird errors, but such is life - output.push_str(&full_path); - } - } - } else { - output.push_str(ident); - } - } - Token::Other(other, _) => { - output.push(other); - if other == '.' || other == '@' { - in_substitution_position = false; - continue; - } - } - Token::Whitespace(ws, _) => output.push_str(ws), - } - - in_substitution_position = true; - } - - Ok(output) -} - -#[cfg(test)] -fn test_parse(input: &str) -> Result>, (&str, usize)> { - let mut declared_imports = IndexMap::default(); - parse_imports(input, &mut declared_imports)?; - Ok(declared_imports) -} - -#[test] -fn import_tokens() { - let input = r" - #import a::b - "; - assert_eq!( - test_parse(input), - Ok(IndexMap::from_iter([( - "b".to_owned(), - vec!("a::b".to_owned()) - )])) - ); - - let input = r" - #import a::{b, c} - "; - assert_eq!( - test_parse(input), - Ok(IndexMap::from_iter([ - ("b".to_owned(), vec!("a::b".to_owned())), - ("c".to_owned(), vec!("a::c".to_owned())), - ])) - ); - - let input = r" - #import a::{b as d, c} - "; - assert_eq!( - test_parse(input), - Ok(IndexMap::from_iter([ - ("d".to_owned(), vec!("a::b".to_owned())), - ("c".to_owned(), vec!("a::c".to_owned())), - ])) - ); - - let input = r" - #import a::{b::{c, d}, e} - "; - assert_eq!( - test_parse(input), - Ok(IndexMap::from_iter([ - ("c".to_owned(), vec!("a::b::c".to_owned())), - ("d".to_owned(), vec!("a::b::d".to_owned())), - ("e".to_owned(), vec!("a::e".to_owned())), - ])) - ); - - let input = r" - #import a::b::{c, d}, e - "; - assert_eq!( - test_parse(input), - Ok(IndexMap::from_iter([ - ("c".to_owned(), vec!("a::b::c".to_owned())), - ("d".to_owned(), vec!("a::b::d".to_owned())), - ("e".to_owned(), vec!("e".to_owned())), - ])) - ); - - let input = r" - #import a, b - "; - assert_eq!( - test_parse(input), - Ok(IndexMap::from_iter([ - ("a".to_owned(), vec!("a".to_owned())), - ("b".to_owned(), vec!("b".to_owned())), - ])) - ); - - let input = r" - #import a::b c, d - "; - assert_eq!( - test_parse(input), - Ok(IndexMap::from_iter([ - ("c".to_owned(), vec!("a::b::c".to_owned())), - ("d".to_owned(), vec!("a::b::d".to_owned())), - ])) - ); - - let input = r" - #import a::b c - "; - assert_eq!( - test_parse(input), - Ok(IndexMap::from_iter([( - "c".to_owned(), - vec!("a::b::c".to_owned()) - ),])) - ); - - let input = r" - #import a::b::{c::{d, e}, f, g::{h as i, j}} - "; - assert_eq!( - test_parse(input), - Ok(IndexMap::from_iter([ - ("d".to_owned(), vec!("a::b::c::d".to_owned())), - ("e".to_owned(), vec!("a::b::c::e".to_owned())), - ("f".to_owned(), vec!("a::b::f".to_owned())), - ("i".to_owned(), vec!("a::b::g::h".to_owned())), - ("j".to_owned(), vec!("a::b::g::j".to_owned())), - ])) - ); - - let input = r" - #import a::b::{ - c::{d, e}, - f, - g::{ - h as i, - j::k::l as m, - } - } - "; - assert_eq!( - test_parse(input), - Ok(IndexMap::from_iter([ - ("d".to_owned(), vec!("a::b::c::d".to_owned())), - ("e".to_owned(), vec!("a::b::c::e".to_owned())), - ("f".to_owned(), vec!("a::b::f".to_owned())), - ("i".to_owned(), vec!("a::b::g::h".to_owned())), - ("m".to_owned(), vec!("a::b::g::j::k::l".to_owned())), - ])) - ); - - let input = r#" - #import "path//with\ all sorts of .stuff"::{a, b} - "#; - assert_eq!( - test_parse(input), - Ok(IndexMap::from_iter([ - ( - "a".to_owned(), - vec!(r#""path//with\ all sorts of .stuff"::a"#.to_owned()) - ), - ( - "b".to_owned(), - vec!(r#""path//with\ all sorts of .stuff"::b"#.to_owned()) - ), - ])) - ); - - let input = r" - #import a::b::{ - "; - assert!(test_parse(input).is_err()); - - let input = r" - #import a::b::{{c} - "; - assert!(test_parse(input).is_err()); - - let input = r" - #import a::b::{c}} - "; - assert!(test_parse(input).is_err()); - - let input = r" - #import a::b{{c,d}} - "; - assert!(test_parse(input).is_err()); - - let input = r" - #import a:b - "; - assert!(test_parse(input).is_err()); -} diff --git a/src/compose/preprocess.rs b/src/compose/preprocess.rs index 366f1ae..31080ee 100644 --- a/src/compose/preprocess.rs +++ b/src/compose/preprocess.rs @@ -1,1534 +1,791 @@ -use std::collections::{HashMap, HashSet}; +use std::{collections::HashSet, ops::Range}; -use indexmap::IndexMap; -use winnow::Parser; - -use super::{ - parse_imports::{parse_imports, substitute_identifiers}, - preprocess1::{self, input_new}, - ComposerErrorInner, ImportDefWithOffset, ShaderDefValue, +use winnow::{ + combinator::{alt, empty, eof, fail, opt, peek, preceded, repeat, separated, seq, terminated}, + error::{ContextError, StrContext}, + stream::Recoverable, + token::{any, none_of, one_of, take_till, take_while}, + Located, PResult, Parser, }; -#[derive(Debug)] -pub struct Preprocessor; +use super::{composer::ImportDefWithOffset, ShaderDefValue}; -#[derive(Debug)] -pub struct PreprocessorMetaData { - pub name: Option, - pub imports: Vec, - pub defines: HashMap, - pub effective_defs: HashSet, +/** + * The abstract syntax trees do not include spaces or comments. They are implicity there between adjacent tokens. + * It is also missing a lot of filler tokens, like semicolons, commas, and braces. + * The syntax tree only has ranges, and needs the original source code to extract the actual text. + * + * If we ever want to have a full concrete syntax tree, we should look into https://github.com/domenicquirl/cstree + */ + +pub type Input<'a> = Recoverable, ContextError>; +pub fn input_new(input: &str) -> Input { + Recoverable::new(Located::new(input)) } -enum ScopeLevel { - Active, // conditions have been met - PreviouslyActive, // conditions have previously been met - NotActive, // no conditions yet met +#[derive(Debug)] +pub struct Preprocessed { + pub parts: Vec, } +impl Preprocessed { + pub fn get_module_names<'a, 'b>(&'b self, input: &'a str) -> impl Iterator + 'b + where + 'a: 'b, + { + self.parts + .iter() + .filter_map(|v: &PreprocessorPart| match v { + PreprocessorPart::DefineImportPath(DefineImportPath { path }) => { + path.as_ref().map(|v| &input[v.clone()]) + } + _ => None, + }) + } + pub fn get_imports(&self, input: &str) -> Vec { + self.parts + .iter() + .filter_map(move |v| match v { + PreprocessorPart::Import(v) => v.get_import(input), + _ => None, + }) + .flatten() + .collect::>() + } -struct Scope(Vec); + pub fn get_used_defs(&self, input: &str) -> HashSet { + self.parts + .iter() + .filter_map(|v| match v { + PreprocessorPart::If(v) => v.name.as_ref(), + PreprocessorPart::IfOp(v) => v.name.as_ref(), + PreprocessorPart::UseDefine(v) => v.name.as_ref(), + _ => None, + }) + .map(|v| input[v.clone()].to_owned()) + .collect() + } -impl Scope { - fn new() -> Self { - Self(vec![ScopeLevel::Active]) + pub fn get_defined_defs<'a>( + &'a self, + input: &'a str, + ) -> impl Iterator + 'a { + self.parts.iter().filter_map(|v| match v { + PreprocessorPart::DefineShaderDef(v) => { + let name = match v.name.as_ref() { + Some(v) => input[v.clone()].to_owned(), + None => return None, + }; + let value = match v.value.as_ref() { + Some(v) => ShaderDefValue::parse(&input[v.clone()]), + None => ShaderDefValue::default(), + }; + Some((name, value)) + } + _ => None, + }) } +} - fn branch( - &mut self, - is_else: bool, - condition: bool, - offset: usize, - ) -> Result<(), ComposerErrorInner> { - if is_else { - let prev_scope = self.0.pop().unwrap(); - let parent_scope = self - .0 - .last() - .ok_or(ComposerErrorInner::ElseWithoutCondition(offset))?; - let new_scope = if !matches!(parent_scope, ScopeLevel::Active) { - ScopeLevel::NotActive - } else if !matches!(prev_scope, ScopeLevel::NotActive) { - ScopeLevel::PreviouslyActive - } else if condition { - ScopeLevel::Active - } else { - ScopeLevel::NotActive - }; +impl ImportDirective { + pub fn get_import(&self, input: &str) -> Option> { + fn to_stack<'a>(input: &'a str, ranges: &[Range]) -> Vec<&'a str> { + ranges + .iter() + .map(|range| &input[range.clone()]) + .collect::>() + } - self.0.push(new_scope); - } else { - let parent_scope = self.0.last().unwrap_or(&ScopeLevel::Active); - let new_scope = if matches!(parent_scope, ScopeLevel::Active) && condition { - ScopeLevel::Active - } else { - ScopeLevel::NotActive + fn walk_import_tree<'a>( + input: &'a str, + tree: &ImportTree, + stack: &[&'a str], + offset: usize, + ) -> Vec { + let (alias_range, path_ranges) = match tree { + ImportTree::Path(path_ranges) => (None, path_ranges), + ImportTree::Alias { path, alias } => (alias.clone(), path), + ImportTree::Children { path, children } => { + let extended_stack = [stack, &to_stack(input, path)].concat(); + return children + .iter() + .flat_map(|child| walk_import_tree(input, child, &extended_stack, offset)) + .collect(); + } }; - self.0.push(new_scope); + let alias = alias_range.map(|v| input[v.clone()].to_owned()); + let path = [stack, &to_stack(input, &path_ranges)].concat().join("::"); + vec![FlattenedImport { + alias, + path, + offset, + }] } - Ok(()) - } - - fn pop(&mut self, offset: usize) -> Result<(), ComposerErrorInner> { - self.0.pop(); - if self.0.is_empty() { - Err(ComposerErrorInner::TooManyEndIfs(offset)) - } else { - Ok(()) + match &self.tree { + Some((tree, range)) => Some(walk_import_tree(input, tree, &[], range.start)), + None => None, } } +} - fn active(&self) -> bool { - matches!(self.0.last().unwrap(), ScopeLevel::Active) - } +#[derive(Debug, Clone)] +pub struct ResolvedIfOp<'a> { + pub is_else_if: bool, + pub name: &'a str, + pub op: &'a str, + pub value: &'a str, +} - fn finish(&self, offset: usize) -> Result<(), ComposerErrorInner> { - if self.0.len() != 1 { - Err(ComposerErrorInner::NotEnoughEndIfs(offset)) - } else { - Ok(()) - } +impl IfOpDirective { + pub fn resolve<'a>(&self, input: &'a str) -> Option> { + Some(ResolvedIfOp { + is_else_if: self.is_else_if, + name: &input[self.name.as_ref()?.clone()], + op: &input[self.op.as_ref()?.clone()], + value: &input[self.value.as_ref()?.clone()], + }) } } #[derive(Debug)] -pub struct PreprocessOutput { - pub preprocessed_source: String, - pub imports: Vec, +pub struct FlattenedImport { + pub offset: usize, + pub path: String, + pub alias: Option, } -impl Preprocessor { - fn check_scope<'a>( - &self, - shader_defs: &HashMap, - line: &'a str, - scope: Option<&mut Scope>, - offset: usize, - ) -> Result<(bool, Option<&'a str>), ComposerErrorInner> { - if let Some(cap) = self.ifdef_regex.captures(line) { - let is_else = cap.get(1).is_some(); - let def = cap.get(2).unwrap().as_str(); - let cond = shader_defs.contains_key(def); - scope.map_or(Ok(()), |scope| scope.branch(is_else, cond, offset))?; - return Ok((true, Some(def))); - } else if let Some(cap) = self.ifndef_regex.captures(line) { - let is_else = cap.get(1).is_some(); - let def = cap.get(2).unwrap().as_str(); - let cond = !shader_defs.contains_key(def); - scope.map_or(Ok(()), |scope| scope.branch(is_else, cond, offset))?; - return Ok((true, Some(def))); - } else if let Some(cap) = self.ifop_regex.captures(line) { - let is_else = cap.get(1).is_some(); - let def = cap.get(2).unwrap().as_str(); - let op = cap.get(3).unwrap(); - let val = cap.get(4).unwrap(); - - if scope.is_none() { - // don't try to evaluate if we don't have a scope - return Ok((true, Some(def))); - } +#[derive(Debug, Clone)] +pub enum PreprocessorPart { + Version(VersionDirective), + If(IfDefDirective), + IfOp(IfOpDirective), + Else(ElseDirective), + EndIf(EndIfDirective), + UseDefine(UseDefineDirective), + DefineShaderDef(DefineShaderDef), + DefineImportPath(DefineImportPath), + Import(ImportDirective), + UnknownDirective(Range), + /// Normal shader code + Text(Range), +} - fn act_on( - a: T, - b: T, - op: &str, - pos: usize, - ) -> Result { - match op { - "==" => Ok(a == b), - "!=" => Ok(a != b), - ">" => Ok(a > b), - ">=" => Ok(a >= b), - "<" => Ok(a < b), - "<=" => Ok(a <= b), - _ => Err(ComposerErrorInner::UnknownShaderDefOperator { - pos, - operator: op.to_string(), +// Note: This is a public API that lower level tools may use. It's a recoverable parser. +pub fn preprocess(input: &mut Input<'_>) -> PResult { + // All of the directives start with a #. + // And most of the directives have to be on their own line. + let mut parts = Vec::new(); + let mut start_text = empty.span().parse_next(input)?.start; + loop { + // I'm at the start of a line. Let's try parsing a preprocessor directive. + if let Some(_) = opt(spaces_single_line).parse_next(input)? { + if let Some(_) = opt(peek('#').span()).parse_next(input)? { + // It's a preprocessor directive + let (part, span): (Option<_>, _) = alt(( + version.map(PreprocessorPart::Version), + if_directive.map(|v| match v { + IfDirective::If(v) => PreprocessorPart::If(v), + IfDirective::IfOp(v) => PreprocessorPart::IfOp(v), + IfDirective::Else(v) => PreprocessorPart::Else(v), }), - } + end_if_directive.map(PreprocessorPart::EndIf), + use_define_directive.map(PreprocessorPart::UseDefine), + define_import_path.map(PreprocessorPart::DefineImportPath), + define_shader_def.map(PreprocessorPart::DefineShaderDef), + import_directive.map(PreprocessorPart::Import), + fail.context(StrContext::Label("Unknown directive")), + )) + .resume_after(take_till(0.., is_newline_start).map(|_| ())) + .with_span() + .parse_next(input)?; + parts.push(PreprocessorPart::Text(start_text..span.start)); + start_text = span.end; + parts.push(part.unwrap_or_else(move || PreprocessorPart::UnknownDirective(span))); + continue; } - - let def_value = shader_defs - .get(def) - .ok_or(ComposerErrorInner::UnknownShaderDef { - pos: offset, - shader_def_name: def.to_string(), - })?; - - let invalid_def = |ty: &str| ComposerErrorInner::InvalidShaderDefComparisonValue { - pos: offset, - shader_def_name: def.to_string(), - value: val.as_str().to_string(), - expected: ty.to_string(), - }; - - let new_scope = match def_value { - ShaderDefValue::Bool(def_value) => { - let val = val.as_str().parse().map_err(|_| invalid_def("bool"))?; - act_on(*def_value, val, op.as_str(), offset)? - } - ShaderDefValue::Int(def_value) => { - let val = val.as_str().parse().map_err(|_| invalid_def("int"))?; - act_on(*def_value, val, op.as_str(), offset)? - } - ShaderDefValue::UInt(def_value) => { - let val = val.as_str().parse().map_err(|_| invalid_def("uint"))?; - act_on(*def_value, val, op.as_str(), offset)? - } - }; - - scope.map_or(Ok(()), |scope| scope.branch(is_else, new_scope, offset))?; - return Ok((true, Some(def))); - } else if self.else_regex.is_match(line) { - scope.map_or(Ok(()), |scope| scope.branch(true, true, offset))?; - return Ok((true, None)); - } else if self.endif_regex.is_match(line) { - scope.map_or(Ok(()), |scope| scope.pop(offset))?; - return Ok((true, None)); } - Ok((false, None)) - } - - // process #if[(n)?def]? / #else / #endif preprocessor directives, - // strip module name and imports - // also strip "#version xxx" - // replace items with resolved decorated names - pub fn preprocess( - &self, - shader_str: &str, - shader_defs: &HashMap, - validate_len: bool, - ) -> Result { - let mut declared_imports = IndexMap::new(); - let mut used_imports = IndexMap::new(); - let mut scope = Scope::new(); - let mut final_string = String::new(); - let mut offset = 0; - - #[cfg(debug)] - let len = shader_str.len(); - - // this code broadly stolen from bevy_render::ShaderProcessor - let mut lines = replace_comments(shader_str).peekable(); - - while let Some((mut line, original_line)) = lines.next() { - let mut output = false; - let trimmed_line = line.trim(); - - if let Some(cap) = { - let a = preprocess1::version.parse(input_new(trimmed_line)).ok(); - a - } { - let version_number = cap.version_number(trimmed_line); - if version_number != Some(440) && version_number != Some(450) { - return Err(ComposerErrorInner::GlslInvalidVersion(offset)); - } - } else if self - .check_scope(shader_defs, &line, Some(&mut scope), offset)? - .0 - || preprocess1::define_import_path - .parse(input_new(trimmed_line)) - .is_ok() - || preprocess1::define_shader_def - .parse(input_new(trimmed_line)) - .is_ok() + // Normal line + loop { + let text = take_till(1.., |c: char| is_newline_start(c) || c == '#') + .span() + .parse_next(input)?; + + if let Some(_) = opt(new_line).parse_next(input)? { + // Nice, we finished a line + break; + } else if let Some((use_define, span)) = + opt(use_define_directive.with_span()).parse_next(input)? { - // ignore - } else if scope.active() { - if preprocess1::import_start.parse(input_new(&line)).is_ok() { - let mut import_lines = String::default(); - let mut open_count = 0; - let initial_offset = offset; - - loop { - // output spaces for removed lines to keep spans consistent (errors report against substituted_source, which is not preprocessed) - final_string.extend(std::iter::repeat(" ").take(line.len())); - offset += line.len() + 1; - - // PERF: Ideally we don't do multiple `match_indices` passes over `line` - // in addition to the final pass for the import parse - open_count += line.match_indices('{').count(); - open_count = open_count.saturating_sub(line.match_indices('}').count()); - - // PERF: it's bad that we allocate here. ideally we would use something like - // let import_lines = &shader_str[initial_offset..offset] - // but we need the comments removed, and the iterator approach doesn't make that easy - import_lines.push_str(&line); - import_lines.push('\n'); - - if open_count == 0 || lines.peek().is_none() { - break; - } - - final_string.push('\n'); - line = lines.next().unwrap().0; - } - - parse_imports(import_lines.as_str(), &mut declared_imports).map_err( - |(err, line_offset)| { - ComposerErrorInner::ImportParseError( - err.to_owned(), - initial_offset + line_offset, - ) - }, - )?; - output = true; - } else { - let replaced_lines = [original_line, &line].map(|input| { - let mut output = input.to_string(); - for capture in self.def_regex.captures_iter(input) { - let def = capture.get(1).unwrap(); - if let Some(def) = shader_defs.get(def.as_str()) { - output = self - .def_regex - .replace(&output, def.value_as_string()) - .to_string(); - } - } - for capture in self.def_regex_delimited.captures_iter(input) { - let def = capture.get(1).unwrap(); - if let Some(def) = shader_defs.get(def.as_str()) { - output = self - .def_regex_delimited - .replace(&output, def.value_as_string()) - .to_string(); - } - } - output - }); - - let original_line = &replaced_lines[0]; - let decommented_line = &replaced_lines[1]; - - // we don't want to capture imports from comments so we run using a dummy used_imports, and disregard any errors - let item_replaced_line = substitute_identifiers( - original_line, - offset, - &declared_imports, - &mut Default::default(), - true, - ) - .unwrap(); - // we also run against the de-commented line to replace real imports, and throw an error if appropriate - let _ = substitute_identifiers( - decommented_line, - offset, - &declared_imports, - &mut used_imports, - false, - ) - .map_err(|pos| { - ComposerErrorInner::ImportParseError( - "Ambiguous import path for item".to_owned(), - pos, - ) - })?; - - final_string.push_str(&item_replaced_line); - let diff = line.len().saturating_sub(item_replaced_line.len()); - final_string.extend(std::iter::repeat(" ").take(diff)); - offset += original_line.len() + 1; - output = true; - } - } - - if !output { - // output spaces for removed lines to keep spans consistent (errors report against substituted_source, which is not preprocessed) - final_string.extend(std::iter::repeat(" ").take(line.len())); - offset += line.len() + 1; + parts.push(PreprocessorPart::Text(start_text..span.start)); + start_text = span.end; + parts.push(PreprocessorPart::UseDefine(use_define)); + // Continue parsing the line + } else if let Some(_) = opt(eof).parse_next(input)? { + // We reached the end of the file + parts.push(PreprocessorPart::Text(start_text..text.end)); + return Ok(Preprocessed { parts }); + } else { + // It's a # that we don't care about + // Skip it and continue parsing the line + let _ = any.parse_next(input)?; } - final_string.push('\n'); - } - - scope.finish(offset)?; - - #[cfg(debug)] - if validate_len { - let revised_len = final_string.len(); - assert_eq!(len, revised_len); } - #[cfg(not(debug))] - let _ = validate_len; - - Ok(PreprocessOutput { - preprocessed_source: final_string, - imports: used_imports.into_values().collect(), - }) } +} - // extract module name and all possible imports - pub fn get_preprocessor_metadata( - &self, - shader_str: &str, - allow_defines: bool, - ) -> Result { - let mut declared_imports = IndexMap::default(); - let mut used_imports = IndexMap::default(); - let mut name = None; - let mut offset = 0; - let mut defines = HashMap::default(); - let mut effective_defs = HashSet::default(); - - let mut lines = replace_comments(shader_str).peekable(); - - while let Some((mut line, _)) = lines.next() { - let (is_scope, def) = self.check_scope(&HashMap::default(), &line, None, offset)?; - - if is_scope { - if let Some(def) = def { - effective_defs.insert(def.to_owned()); - } - } else if preprocess1::import_start.parse(input_new(&line)).is_ok() { - let mut import_lines = String::default(); - let mut open_count = 0; - let initial_offset = offset; - - loop { - // PERF: Ideally we don't do multiple `match_indices` passes over `line` - // in addition to the final pass for the import parse - open_count += line.match_indices('{').count(); - open_count = open_count.saturating_sub(line.match_indices('}').count()); - - // PERF: it's bad that we allocate here. ideally we would use something like - // let import_lines = &shader_str[initial_offset..offset] - // but we need the comments removed, and the iterator approach doesn't make that easy - import_lines.push_str(&line); - import_lines.push('\n'); - - if open_count == 0 || lines.peek().is_none() { - break; - } - - // output spaces for removed lines to keep spans consistent (errors report against substituted_source, which is not preprocessed) - offset += line.len() + 1; - - line = lines.next().unwrap().0; - } - - parse_imports(import_lines.as_str(), &mut declared_imports).map_err( - |(err, line_offset)| { - ComposerErrorInner::ImportParseError( - err.to_owned(), - initial_offset + line_offset, - ) - }, - )?; - } else if let Some(cap) = { - let a = preprocess1::define_import_path.parse(input_new(&line)).ok(); - a - } { - name = Some(line[cap.path.unwrap()].to_string()); - } else if let Some(cap) = { - let a = preprocess1::define_shader_def.parse(input_new(&line)).ok(); - a - } { - if allow_defines { - let name = line[cap.name.unwrap()].to_string(); - - let value = if let Some(val) = cap.value.map(|v| &line[v]) { - if let Ok(val) = val.parse::() { - ShaderDefValue::UInt(val) - } else if let Ok(val) = val.parse::() { - ShaderDefValue::Int(val) - } else if let Ok(val) = val.parse::() { - ShaderDefValue::Bool(val) - } else { - ShaderDefValue::Bool(false) // this error will get picked up when we fully preprocess the module - } - } else { - ShaderDefValue::Bool(true) - }; - - defines.insert(name, value); - } else { - return Err(ComposerErrorInner::DefineInModule(offset)); - } - } else { - for cap in self - .def_regex - .captures_iter(&line) - .chain(self.def_regex_delimited.captures_iter(&line)) - { - effective_defs.insert(cap.get(1).unwrap().as_str().to_owned()); - } +#[derive(Debug, Clone)] +pub struct VersionDirective { + version: Option>, +} +impl VersionDirective { + pub fn version_number(&self, input: &str) -> Option { + self.version + .as_ref() + .and_then(|v| (&input[v.clone()]).parse::().ok()) + } +} - substitute_identifiers(&line, offset, &declared_imports, &mut used_imports, true) - .unwrap(); - } +pub fn version(input: &mut Input<'_>) -> PResult { + seq! {VersionDirective{ + _: "#version".span(), + _: spaces_single_line.resume_after(empty), + version: take_while(1.., |c:char| c.is_ascii_digit()).span().resume_after(empty), + _: spaces_until_new_line + }} + .parse_next(input) +} - offset += line.len() + 1; - } +/// Note: We're disallowing spaces between the `#` and the `ifdef`. +/// `#ifdef {name}` or `#else ifdef {name}` or `#ifndef {name}` or `#else ifndef {name` +#[derive(Debug, Clone)] +pub struct IfDefDirective { + pub is_else_if: bool, + pub is_not: bool, + pub name: Option>, +} - Ok(PreprocessorMetaData { - name, - imports: used_imports.into_values().collect(), - defines, - effective_defs, - }) +impl IfDefDirective { + pub fn name<'a>(&'a self, input: &'a str) -> Option<&'a str> { + self.name.as_ref().map(|v| &input[v.clone()]) } } -#[cfg(test)] -mod test { - use super::*; - - #[rustfmt::skip] - const WGSL_ELSE_IFDEF: &str = r" -struct View { - view_proj: mat4x4, - world_position: vec3, -}; -@group(0) @binding(0) -var view: View; - -#ifdef TEXTURE -// Main texture -@group(1) @binding(0) -var sprite_texture: texture_2d; -#else ifdef SECOND_TEXTURE -// Second texture -@group(1) @binding(0) -var sprite_texture: texture_2d; -#else ifdef THIRD_TEXTURE -// Third texture -@group(1) @binding(0) -var sprite_texture: texture_2d; -#else -@group(1) @binding(0) -var sprite_texture: texture_2d_array; -#endif - -struct VertexOutput { - @location(0) uv: vec2, - @builtin(position) position: vec4, -}; - -@vertex -fn vertex( - @location(0) vertex_position: vec3, - @location(1) vertex_uv: vec2 -) -> VertexOutput { - var out: VertexOutput; - out.uv = vertex_uv; - out.position = view.view_proj * vec4(vertex_position, 1.0); - return out; +/// `#ifop {name} {op} {value}` or `#else ifop {name} {op} {value}` +#[derive(Debug, Clone)] +pub struct IfOpDirective { + pub is_else_if: bool, + pub name: Option>, + pub op: Option>, + pub value: Option>, } -"; - //preprocessor tests - #[test] - fn process_shader_def_unknown_operator() { - #[rustfmt::skip] - const WGSL: &str = r" -struct View { - view_proj: mat4x4, - world_position: vec3, -}; -@group(0) @binding(0) -var view: View; -#if TEXTURE !! true -@group(1) @binding(0) -var sprite_texture: texture_2d; -#endif -struct VertexOutput { - @location(0) uv: vec2, - @builtin(position) position: vec4, -}; -@vertex -fn vertex( - @location(0) vertex_position: vec3, - @location(1) vertex_uv: vec2 -) -> VertexOutput { - var out: VertexOutput; - out.uv = vertex_uv; - out.position = view.view_proj * vec4(vertex_position, 1.0); - return out; +pub enum IfDirective { + If(IfDefDirective), + IfOp(IfOpDirective), + Else(ElseDirective), } -"; - let processor = Preprocessor::default(); +pub fn if_directive(input: &mut Input<'_>) -> PResult { + #[derive(PartialEq, Eq)] + enum Start { + IfDef, + IfNotDef, + IfOp, + Else, + } + let (start, is_else) = alt(( + "#ifop".map(|_| (Start::IfOp, false)), + "#ifdef".map(|_| (Start::IfDef, false)), + "#ifndef".map(|_| (Start::IfNotDef, false)), + ( + "#else", + spaces_single_line.resume_after(empty), + alt(( + "ifdef".map(|_| Start::IfDef), + "ifndef".map(|_| Start::IfNotDef), + "ifop".map(|_| Start::IfOp), + spaces_until_new_line.map(|_| Start::Else), + )), + ) + .map(|(_, _, next)| (next, true)), + )) + .parse_next(input)?; + + match start { + Start::IfDef | Start::IfNotDef => { + let _ = spaces_single_line.resume_after(empty).parse_next(input)?; + let name = shader_def_name.resume_after(empty).parse_next(input)?; + let _ = spaces_until_new_line.parse_next(input)?; + Ok(IfDirective::If(IfDefDirective { + is_else_if: is_else, + is_not: start == Start::IfNotDef, + name, + })) + } + Start::IfOp => { + let _ = spaces_single_line.resume_after(empty).parse_next(input)?; + let name = shader_def_name.resume_after(empty).parse_next(input)?; + let _ = opt(spaces_single_line).parse_next(input)?; + let op = alt(("==", "!=", "<", "<=", ">", ">=")) + .span() + .resume_after(empty) + .parse_next(input)?; + let _ = opt(spaces_single_line).parse_next(input)?; + let value = shader_def_value.resume_after(empty).parse_next(input)?; + let _ = spaces_until_new_line.parse_next(input)?; + Ok(IfDirective::IfOp(IfOpDirective { + is_else_if: is_else, + name, + op, + value, + })) + } + Start::Else => Ok(IfDirective::Else(ElseDirective)), + } +} - let result_missing = processor.preprocess( - WGSL, - &[("TEXTURE".to_owned(), ShaderDefValue::Bool(true))].into(), - true, - ); +/// `#else` +#[derive(Debug, Clone)] +pub struct ElseDirective; + +/// `#endif` +#[derive(Debug, Clone)] +pub struct EndIfDirective; + +pub fn end_if_directive(input: &mut Input<'_>) -> PResult { + seq! {EndIfDirective{ + _: "#endif", + _: spaces_single_line.resume_after(empty), + _: spaces_until_new_line + }} + .parse_next(input) +} - let expected: Result = - Err(ComposerErrorInner::UnknownShaderDefOperator { - pos: 124, - operator: "!!".to_string(), - }); +/// Note: We're disallowing the previous `#ANYTHING` syntax, since it's rarely used and error prone +/// (a misspelled `#inport` would get mistaken for a `#ANYTHING``). +/// `#{name of defined value}`` +#[derive(Debug, Clone)] +pub struct UseDefineDirective { + pub name: Option>, +} - assert_eq!(format!("{result_missing:?}"), format!("{expected:?}"),); +impl UseDefineDirective { + pub fn name<'a>(&'a self, input: &'a str) -> Option<&'a str> { + self.name.as_ref().map(|v| &input[v.clone()]) } - #[test] - fn process_shader_def_equal_int() { - #[rustfmt::skip] - const WGSL: &str = r" -struct View { - view_proj: mat4x4, - world_position: vec3, -}; -@group(0) @binding(0) -var view: View; -#if TEXTURE == 3 -@group(1) @binding(0) -var sprite_texture: texture_2d; -#endif -struct VertexOutput { - @location(0) uv: vec2, - @builtin(position) position: vec4, -}; -@vertex -fn vertex( - @location(0) vertex_position: vec3, - @location(1) vertex_uv: vec2 -) -> VertexOutput { - var out: VertexOutput; - out.uv = vertex_uv; - out.position = view.view_proj * vec4(vertex_position, 1.0); - return out; } -"; - #[rustfmt::skip] - const EXPECTED_EQ: &str = r" -struct View { - view_proj: mat4x4, - world_position: vec3, -}; -@group(0) @binding(0) -var view: View; - -@group(1) @binding(0) -var sprite_texture: texture_2d; - -struct VertexOutput { - @location(0) uv: vec2, - @builtin(position) position: vec4, -}; -@vertex -fn vertex( - @location(0) vertex_position: vec3, - @location(1) vertex_uv: vec2 -) -> VertexOutput { - var out: VertexOutput; - out.uv = vertex_uv; - out.position = view.view_proj * vec4(vertex_position, 1.0); - return out; +/// Remember that this one doesn't need to be on its own line +pub fn use_define_directive(input: &mut Input<'_>) -> PResult { + seq! {UseDefineDirective{ + _: "#{", + _: spaces_single_line.resume_after(empty), + name: shader_def_name.resume_after(empty), + _: spaces_single_line.resume_after(empty), + _: "}".resume_after(empty) + }} + .parse_next(input) } -"; - #[rustfmt::skip] - const EXPECTED_NEQ: &str = r" -struct View { - view_proj: mat4x4, - world_position: vec3, -}; -@group(0) @binding(0) -var view: View; - - - - -struct VertexOutput { - @location(0) uv: vec2, - @builtin(position) position: vec4, -}; -@vertex -fn vertex( - @location(0) vertex_position: vec3, - @location(1) vertex_uv: vec2 -) -> VertexOutput { - var out: VertexOutput; - out.uv = vertex_uv; - out.position = view.view_proj * vec4(vertex_position, 1.0); - return out; +/// `#define {name} {value}`, except it can only be used with other preprocessor macros. +/// Unlike its C cousin, it doesn't aggressively replace text. +#[derive(Debug, Clone)] +pub struct DefineShaderDef { + pub name: Option>, + pub value: Option>, } -"; - let processor = Preprocessor::default(); - let result_eq = processor - .preprocess( - WGSL, - &[("TEXTURE".to_string(), ShaderDefValue::Int(3))].into(), - true, - ) - .unwrap(); - assert_eq!(result_eq.preprocessed_source, EXPECTED_EQ); - - let result_neq = processor - .preprocess( - WGSL, - &[("TEXTURE".to_string(), ShaderDefValue::Int(7))].into(), - true, - ) - .unwrap(); - assert_eq!(result_neq.preprocessed_source, EXPECTED_NEQ); - - let result_missing = processor.preprocess(WGSL, &Default::default(), true); - - let expected_err: Result< - (Option, String, Vec), - ComposerErrorInner, - > = Err(ComposerErrorInner::UnknownShaderDef { - pos: 124, - shader_def_name: "TEXTURE".to_string(), - }); - assert_eq!(format!("{result_missing:?}"), format!("{expected_err:?}"),); - - let result_wrong_type = processor.preprocess( - WGSL, - &[("TEXTURE".to_string(), ShaderDefValue::Bool(true))].into(), - true, - ); - let expected_err: Result< - (Option, String, Vec), - ComposerErrorInner, - > = Err(ComposerErrorInner::InvalidShaderDefComparisonValue { - pos: 124, - shader_def_name: "TEXTURE".to_string(), - expected: "bool".to_string(), - value: "3".to_string(), - }); +pub fn define_shader_def(input: &mut Input<'_>) -> PResult { + // Technically I'm changing the #define behaviour + // I'm no longer allowing redefining numbers, like #define 3 a|b + seq! {DefineShaderDef{ + _: "#define", + _: spaces_single_line.resume_after(empty), + name: shader_def_name.resume_after(empty), + _: spaces_single_line.resume_after(empty), + value: opt(shader_def_value), + _: spaces_until_new_line + }} + .parse_next(input) +} - assert_eq!( - format!("{result_wrong_type:?}"), - format!("{expected_err:?}") - ); - } +fn shader_def_name(input: &mut Input<'_>) -> PResult> { + ( + one_of(|c: char| c.is_ascii_alphabetic() || c == '_'), + take_while(0.., |c: char| c.is_ascii_alphanumeric() || c == '_'), + ) + .span() + .parse_next(input) +} - #[test] - fn process_shader_def_equal_bool() { - #[rustfmt::skip] - const WGSL: &str = r" -struct View { - view_proj: mat4x4, - world_position: vec3, -}; -@group(0) @binding(0) -var view: View; -#if TEXTURE == true -@group(1) @binding(0) -var sprite_texture: texture_2d; -#endif -struct VertexOutput { - @location(0) uv: vec2, - @builtin(position) position: vec4, -}; -@vertex -fn vertex( - @location(0) vertex_position: vec3, - @location(1) vertex_uv: vec2 -) -> VertexOutput { - var out: VertexOutput; - out.uv = vertex_uv; - out.position = view.view_proj * vec4(vertex_position, 1.0); - return out; +fn shader_def_value(input: &mut Input<'_>) -> PResult> { + take_while(1.., |c: char| { + c.is_ascii_alphanumeric() || c == '_' || c == '-' + }) + .span() + .parse_next(input) } -"; - #[rustfmt::skip] - const EXPECTED_EQ: &str = r" -struct View { - view_proj: mat4x4, - world_position: vec3, -}; -@group(0) @binding(0) -var view: View; - -@group(1) @binding(0) -var sprite_texture: texture_2d; - -struct VertexOutput { - @location(0) uv: vec2, - @builtin(position) position: vec4, -}; -@vertex -fn vertex( - @location(0) vertex_position: vec3, - @location(1) vertex_uv: vec2 -) -> VertexOutput { - var out: VertexOutput; - out.uv = vertex_uv; - out.position = view.view_proj * vec4(vertex_position, 1.0); - return out; +#[derive(Debug, Clone)] +pub struct DefineImportPath { + pub path: Option>, } -"; - #[rustfmt::skip] - const EXPECTED_NEQ: &str = r" -struct View { - view_proj: mat4x4, - world_position: vec3, -}; -@group(0) @binding(0) -var view: View; - - - - -struct VertexOutput { - @location(0) uv: vec2, - @builtin(position) position: vec4, -}; -@vertex -fn vertex( - @location(0) vertex_position: vec3, - @location(1) vertex_uv: vec2 -) -> VertexOutput { - var out: VertexOutput; - out.uv = vertex_uv; - out.position = view.view_proj * vec4(vertex_position, 1.0); - return out; +pub fn define_import_path(input: &mut Input<'_>) -> PResult { + seq! {DefineImportPath{ + _: "#define_import_path", + _: spaces_single_line.resume_after(empty), + path: take_while(1.., |c: char| !c.is_whitespace()).span().resume_after(empty), + _: spaces_until_new_line + }} + .parse_next(input) } -"; - let processor = Preprocessor::default(); - let result_eq = processor - .preprocess( - WGSL, - &[("TEXTURE".to_string(), ShaderDefValue::Bool(true))].into(), - true, - ) - .unwrap(); - assert_eq!(result_eq.preprocessed_source, EXPECTED_EQ); - - let result_neq = processor - .preprocess( - WGSL, - &[("TEXTURE".to_string(), ShaderDefValue::Bool(false))].into(), - true, - ) - .unwrap(); - assert_eq!(result_neq.preprocessed_source, EXPECTED_NEQ); - } - #[test] - fn process_shader_def_not_equal_bool() { - #[rustfmt::skip] - const WGSL: &str = r" -struct View { - view_proj: mat4x4, - world_position: vec3, -}; -@group(0) @binding(0) -var view: View; -#if TEXTURE != false -@group(1) @binding(0) -var sprite_texture: texture_2d; -#endif -struct VertexOutput { - @location(0) uv: vec2, - @builtin(position) position: vec4, -}; -@vertex -fn vertex( - @location(0) vertex_position: vec3, - @location(1) vertex_uv: vec2 -) -> VertexOutput { - var out: VertexOutput; - out.uv = vertex_uv; - out.position = view.view_proj * vec4(vertex_position, 1.0); - return out; +/// Formal grammar +/// ```ebnf +/// ::= "#import" "::"? ";"? +/// +/// ::= ( "as" | "::" "{" "}")? +/// ::= ("," )* ","? +/// +/// ::= ( | ) ("::" ( | ) )* +/// ::= ([a-z]) ([a-z] | [0-9])* +/// ::= "\"" + "\"" +/// ::= [a-z] +/// +/// ::= " "+ +/// ``` +/// +/// Can be tested on https://bnfplayground.pauliankline.com/ +/// +/// Except that +/// - `` should be Unicode aware +/// - `` should use the XID rules instead of only allowing lowercase letters +/// - `` should be a string with at least one character, and follow the usual "quotes and \\ backslash for escaping" rules +/// - spaces are allowed between every token +/// ``` +#[derive(Debug, Clone)] +pub struct ImportDirective { + pub root_specifier: Option>, + pub tree: Option<(ImportTree, Range)>, } -"; - #[rustfmt::skip] - const EXPECTED_EQ: &str = r" -struct View { - view_proj: mat4x4, - world_position: vec3, -}; -@group(0) @binding(0) -var view: View; - -@group(1) @binding(0) -var sprite_texture: texture_2d; - -struct VertexOutput { - @location(0) uv: vec2, - @builtin(position) position: vec4, -}; -@vertex -fn vertex( - @location(0) vertex_position: vec3, - @location(1) vertex_uv: vec2 -) -> VertexOutput { - var out: VertexOutput; - out.uv = vertex_uv; - out.position = view.view_proj * vec4(vertex_position, 1.0); - return out; +pub fn import_directive(input: &mut Input<'_>) -> PResult { + seq! {ImportDirective { + _ : "#import", + _: spaces.resume_after(empty), + root_specifier: opt("::".span()), + tree: import_tree.with_span().resume_after(empty), + _: opt(spaces), + _: opt(";"), + }} + .parse_next(input) } -"; - #[rustfmt::skip] - const EXPECTED_NEQ: &str = r" -struct View { - view_proj: mat4x4, - world_position: vec3, -}; -@group(0) @binding(0) -var view: View; - - - - -struct VertexOutput { - @location(0) uv: vec2, - @builtin(position) position: vec4, -}; -@vertex -fn vertex( - @location(0) vertex_position: vec3, - @location(1) vertex_uv: vec2 -) -> VertexOutput { - var out: VertexOutput; - out.uv = vertex_uv; - out.position = view.view_proj * vec4(vertex_position, 1.0); - return out; +#[derive(Debug, Clone)] +pub enum ImportTree { + Path(Vec>), + Alias { + path: Vec>, + alias: Option>, + }, + Children { + path: Vec>, + children: Vec, + }, } -"; - let processor = Preprocessor::default(); - let result_eq = processor - .preprocess( - WGSL, - &[("TEXTURE".to_string(), ShaderDefValue::Bool(true))].into(), - true, - ) - .unwrap(); - assert_eq!(result_eq.preprocessed_source, EXPECTED_EQ); - - let result_neq = processor - .preprocess( - WGSL, - &[("TEXTURE".to_string(), ShaderDefValue::Bool(false))].into(), - true, - ) - .unwrap(); - assert_eq!(result_neq.preprocessed_source, EXPECTED_NEQ); - - let result_missing = processor.preprocess(WGSL, &[].into(), true); - let expected_err: Result< - (Option, String, Vec), - ComposerErrorInner, - > = Err(ComposerErrorInner::UnknownShaderDef { - pos: 124, - shader_def_name: "TEXTURE".to_string(), - }); - assert_eq!(format!("{result_missing:?}"), format!("{expected_err:?}"),); - - let result_wrong_type = processor.preprocess( - WGSL, - &[("TEXTURE".to_string(), ShaderDefValue::Int(7))].into(), - true, - ); - let expected_err: Result< - (Option, String, Vec), - ComposerErrorInner, - > = Err(ComposerErrorInner::InvalidShaderDefComparisonValue { - pos: 124, - shader_def_name: "TEXTURE".to_string(), - expected: "int".to_string(), - value: "false".to_string(), - }); - assert_eq!( - format!("{result_wrong_type:?}"), - format!("{expected_err:?}"), - ); - } +fn import_tree(input: &mut Input<'_>) -> PResult { + let path = path.parse_next(input)?; + let s = opt(spaces).parse_next(input)?; - #[test] - fn process_shader_def_replace() { - #[rustfmt::skip] - const WGSL: &str = r" -struct View { - view_proj: mat4x4, - world_position: vec3, -}; -@group(0) @binding(0) -var view: View; -struct VertexOutput { - @location(0) uv: vec2, - @builtin(position) position: vec4, -}; -@vertex -fn vertex( - @location(0) vertex_position: vec3, - @location(1) vertex_uv: vec2 -) -> VertexOutput { - var out: VertexOutput; - out.uv = vertex_uv; - var a: i32 = #FIRST_VALUE; - var b: i32 = #FIRST_VALUE * #SECOND_VALUE; - var c: i32 = #MISSING_VALUE; - var d: bool = #BOOL_VALUE; - out.position = view.view_proj * vec4(vertex_position, 1.0); - return out; -} -"; + if s.is_some() { + let as_token = opt("as").parse_next(input)?; + if as_token.is_some() { + let _ = spaces.resume_after(empty).parse_next(input)?; + let alias = identifier.resume_after(empty).parse_next(input)?; + return Ok(ImportTree::Alias { path, alias }); + } + } - #[rustfmt::skip] - const EXPECTED_REPLACED: &str = r" -struct View { - view_proj: mat4x4, - world_position: vec3, -}; -@group(0) @binding(0) -var view: View; -struct VertexOutput { - @location(0) uv: vec2, - @builtin(position) position: vec4, -}; -@vertex -fn vertex( - @location(0) vertex_position: vec3, - @location(1) vertex_uv: vec2 -) -> VertexOutput { - var out: VertexOutput; - out.uv = vertex_uv; - var a: i32 = 5; - var b: i32 = 5 * 3; - var c: i32 = #MISSING_VALUE; - var d: bool = true; - out.position = view.view_proj * vec4(vertex_position, 1.0); - return out; -} -"; - let processor = Preprocessor::default(); - let result = processor - .preprocess( - WGSL, - &[ - ("BOOL_VALUE".to_string(), ShaderDefValue::Bool(true)), - ("FIRST_VALUE".to_string(), ShaderDefValue::Int(5)), - ("SECOND_VALUE".to_string(), ShaderDefValue::Int(3)), - ] - .into(), - true, + if let Some(_) = opt("::{").parse_next(input)? { + let _ = opt(spaces).parse_next(input)?; + let children = import_trees + .retry_after( + // TODO: This recovery can explode and eat the whole file + (take_till(0.., (',', '}')), ",").map(|_| ()), ) - .unwrap(); - assert_eq!(result.preprocessed_source, EXPECTED_REPLACED); + // TODO: This recovery can explode and eat the whole file + .resume_after((take_till(0.., '}'), '}').map(|_| ())) + .parse_next(input)? + .unwrap_or_default(); + + let _ = opt(spaces).parse_next(input)?; + let _ = "}".resume_after(empty).parse_next(input)?; + return Ok(ImportTree::Children { path, children }); } - #[test] - fn process_shader_define_in_shader() { - #[rustfmt::skip] - const WGSL: &str = r" -#define NOW_DEFINED -#ifdef NOW_DEFINED -defined -#endif -"; + Ok(ImportTree::Path(path)) +} - #[rustfmt::skip] - const EXPECTED: &str = r" - - -defined - -"; - let processor = Preprocessor::default(); - let PreprocessorMetaData { - defines: shader_defs, - .. - } = processor.get_preprocessor_metadata(&WGSL, true).unwrap(); - println!("defines: {:?}", shader_defs); - let result = processor.preprocess(&WGSL, &shader_defs, true).unwrap(); - assert_eq!(result.preprocessed_source, EXPECTED); - } +fn import_trees(input: &mut Input<'_>) -> PResult> { + terminated( + separated(1.., import_tree, (opt(spaces), ",", opt(spaces))), + (opt(spaces), ","), + ) + .parse_next(input) +} - #[test] - fn process_shader_define_in_shader_with_value() { - #[rustfmt::skip] - const WGSL: &str = r" -#define DEFUINT 1 -#define DEFINT -1 -#define DEFBOOL false -#if DEFUINT == 1 -uint: #DEFUINT -#endif -#if DEFINT == -1 -int: #DEFINT -#endif -#if DEFBOOL == false -bool: #DEFBOOL -#endif -"; +fn path(input: &mut Input<'_>) -> PResult>> { + separated( + 1.., + alt((nonempty_string, identifier)), + (opt(spaces), "::", opt(spaces)), + ) + .parse_next(input) +} - #[rustfmt::skip] - const EXPECTED: &str = r" - - - - -uint: 1 - - -int: -1 - - -bool: false - -"; - let processor = Preprocessor::default(); - let PreprocessorMetaData { - defines: shader_defs, - .. - } = processor.get_preprocessor_metadata(&WGSL, true).unwrap(); - println!("defines: {:?}", shader_defs); - let result = processor.preprocess(&WGSL, &shader_defs, true).unwrap(); - assert_eq!(result.preprocessed_source, EXPECTED); - } +fn identifier(input: &mut Input<'_>) -> PResult> { + ( + one_of(|c: char| unicode_ident::is_xid_start(c)), + take_while(0.., |c: char| unicode_ident::is_xid_continue(c)), + ) + .span() + .parse_next(input) +} - #[test] - fn process_shader_def_else_ifdef_ends_up_in_else() { - #[rustfmt::skip] - const EXPECTED: &str = r" -struct View { - view_proj: mat4x4, - world_position: vec3, -}; -@group(0) @binding(0) -var view: View; -@group(1) @binding(0) -var sprite_texture: texture_2d_array; -struct VertexOutput { - @location(0) uv: vec2, - @builtin(position) position: vec4, -}; -@vertex -fn vertex( - @location(0) vertex_position: vec3, - @location(1) vertex_uv: vec2 -) -> VertexOutput { - var out: VertexOutput; - out.uv = vertex_uv; - out.position = view.view_proj * vec4(vertex_position, 1.0); - return out; +fn nonempty_string(input: &mut Input<'_>) -> PResult> { + quoted_string.verify(|s| s.len() > 2).parse_next(input) } -"; - let processor = Preprocessor::default(); - let result = processor - .preprocess(&WGSL_ELSE_IFDEF, &[].into(), true) - .unwrap(); - assert_eq!( - result - .preprocessed_source - .replace(" ", "") - .replace("\n", "") - .replace("\r", ""), - EXPECTED - .replace(" ", "") - .replace("\n", "") - .replace("\r", "") - ); + +fn quoted_string(input: &mut Input<'_>) -> PResult> { + // See https://docs.rs/winnow/latest/winnow/_topic/json/index.html + preceded( + '\"', + terminated( + repeat(0.., string_character).fold(|| (), |a, _| a), + '\"'.resume_after(empty), + ), + ) + .span() + .parse_next(input) +} +fn string_character(input: &mut Input<'_>) -> PResult<()> { + let c = none_of('\"').parse_next(input)?; + if c == '\\' { + let _ = any.parse_next(input)?; } + Ok(()) +} - #[test] - fn process_shader_def_else_ifdef_no_match_and_no_fallback_else() { - #[rustfmt::skip] - const WGSL_ELSE_IFDEF_NO_ELSE_FALLBACK: &str = r" -struct View { - view_proj: mat4x4, - world_position: vec3, -}; -@group(0) @binding(0) -var view: View; - -#ifdef TEXTURE -// Main texture -@group(1) @binding(0) -var sprite_texture: texture_2d; -#else ifdef OTHER_TEXTURE -// Other texture -@group(1) @binding(0) -var sprite_texture: texture_2d; -#endif - -struct VertexOutput { - @location(0) uv: vec2, - @builtin(position) position: vec4, -}; +pub struct Spaces { + /// Comments that are in this "spaces" block + pub comments: Vec>, +} -@vertex -fn vertex( - @location(0) vertex_position: vec3, - @location(1) vertex_uv: vec2 -) -> VertexOutput { - var out: VertexOutput; - out.uv = vertex_uv; - out.position = view.view_proj * vec4(vertex_position, 1.0); - return out; +/// Parses at least one whitespace or comment. +fn spaces(input: &mut Input<'_>) -> PResult { + repeat( + 1.., + alt(( + take_while(1.., |c: char| c.is_whitespace()).map(|_| None), + single_line_comment.span().map(|c| Some(c)), + multi_line_comment.span().map(|c| Some(c)), + )), + ) + .fold( + || Vec::new(), + |mut comments, comment| { + if let Some(comment) = comment { + comments.push(comment); + } + comments + }, + ) + .map(|comments| Spaces { comments }) + .parse_next(input) } -"; - #[rustfmt::skip] - const EXPECTED: &str = r" -struct View { - view_proj: mat4x4, - world_position: vec3, -}; -@group(0) @binding(0) -var view: View; -struct VertexOutput { - @location(0) uv: vec2, - @builtin(position) position: vec4, -}; -@vertex -fn vertex( - @location(0) vertex_position: vec3, - @location(1) vertex_uv: vec2 -) -> VertexOutput { - var out: VertexOutput; - out.uv = vertex_uv; - out.position = view.view_proj * vec4(vertex_position, 1.0); - return out; +/// Parses at least one non-newline whitespace or comment. +/// Preprocessor directives are always on their own line. So they need a slightly different spaces parser. +fn spaces_single_line(input: &mut Input<'_>) -> PResult { + repeat( + 1.., + alt(( + take_while(1.., |c: char| c.is_whitespace() && !is_newline_start(c)).map(|_| None), + single_line_comment.span().map(|c| Some(c)), + multi_line_comment.span().map(|c| Some(c)), + )), + ) + .fold( + || Vec::new(), + |mut comments, comment| { + if let Some(comment) = comment { + comments.push(comment); + } + comments + }, + ) + .map(|comments| Spaces { comments }) + .parse_next(input) } -"; - let processor = Preprocessor::default(); - let result = processor - .preprocess(&WGSL_ELSE_IFDEF_NO_ELSE_FALLBACK, &[].into(), true) - .unwrap(); - assert_eq!( - result - .preprocessed_source - .replace(" ", "") - .replace("\n", "") - .replace("\r", ""), - EXPECTED - .replace(" ", "") - .replace("\n", "") - .replace("\r", "") - ); - } - #[test] - fn process_shader_def_else_ifdef_ends_up_in_first_clause() { - #[rustfmt::skip] - const EXPECTED: &str = r" -struct View { - view_proj: mat4x4, - world_position: vec3, -}; -@group(0) @binding(0) -var view: View; - -// Main texture -@group(1) @binding(0) -var sprite_texture: texture_2d; - -struct VertexOutput { - @location(0) uv: vec2, - @builtin(position) position: vec4, -}; +/// Checks if it's part of a Unicode line break, according to https://www.w3.org/TR/WGSL/#line-break +fn is_newline_start(c: char) -> bool { + c == '\u{000A}' + || c == '\u{000B}' + || c == '\u{000C}' + || c == '\u{000D}' + || c == '\u{0085}' + || c == '\u{2028}' + || c == '\u{2029}' +} -@vertex -fn vertex( - @location(0) vertex_position: vec3, - @location(1) vertex_uv: vec2 -) -> VertexOutput { - var out: VertexOutput; - out.uv = vertex_uv; - out.position = view.view_proj * vec4(vertex_position, 1.0); - return out; +fn spaces_until_new_line(input: &mut Input<'_>) -> PResult { + let spaces = opt(spaces_single_line).parse_next(input)?; + let _newline = new_line + .retry_after(take_till(0.., is_newline_start).map(|_| ())) + .parse_next(input)?; + Ok(spaces.unwrap_or(Spaces { + comments: Vec::new(), + })) } -"; - let processor = Preprocessor::default(); - let result = processor - .preprocess( - &WGSL_ELSE_IFDEF, - &[("TEXTURE".to_string(), ShaderDefValue::Bool(true))].into(), - true, - ) - .unwrap(); - assert_eq!( - result - .preprocessed_source - .replace(" ", "") - .replace("\n", "") - .replace("\r", ""), - EXPECTED - .replace(" ", "") - .replace("\n", "") - .replace("\r", "") - ); - } - #[test] - fn process_shader_def_else_ifdef_ends_up_in_second_clause() { - #[rustfmt::skip] - const EXPECTED: &str = r" -struct View { - view_proj: mat4x4, - world_position: vec3, -}; -@group(0) @binding(0) -var view: View; -// Second texture -@group(1) @binding(0) -var sprite_texture: texture_2d; -struct VertexOutput { - @location(0) uv: vec2, - @builtin(position) position: vec4, -}; -@vertex -fn vertex( - @location(0) vertex_position: vec3, - @location(1) vertex_uv: vec2 -) -> VertexOutput { - var out: VertexOutput; - out.uv = vertex_uv; - out.position = view.view_proj * vec4(vertex_position, 1.0); - return out; +fn new_line(input: &mut Input<'_>) -> PResult<()> { + alt(( + "\u{000D}\u{000A}".map(|_| ()), + one_of(is_newline_start).map(|_| ()), + eof.map(|_| ()), + )) + .parse_next(input) } -"; - let processor = Preprocessor::default(); - let result = processor - .preprocess( - &WGSL_ELSE_IFDEF, - &[("SECOND_TEXTURE".to_string(), ShaderDefValue::Bool(true))].into(), - true, - ) - .unwrap(); - assert_eq!( - result - .preprocessed_source - .replace(" ", "") - .replace("\n", "") - .replace("\r", ""), - EXPECTED - .replace(" ", "") - .replace("\n", "") - .replace("\r", "") - ); - } - #[test] - fn process_shader_def_else_ifdef_ends_up_in_third_clause() { - #[rustfmt::skip] - const EXPECTED: &str = r" -struct View { - view_proj: mat4x4, - world_position: vec3, -}; -@group(0) @binding(0) -var view: View; -// Third texture -@group(1) @binding(0) -var sprite_texture: texture_2d; -struct VertexOutput { - @location(0) uv: vec2, - @builtin(position) position: vec4, -}; -@vertex -fn vertex( - @location(0) vertex_position: vec3, - @location(1) vertex_uv: vec2 -) -> VertexOutput { - var out: VertexOutput; - out.uv = vertex_uv; - out.position = view.view_proj * vec4(vertex_position, 1.0); - return out; +fn single_line_comment(input: &mut Input<'_>) -> PResult<()> { + let _start = "//".parse_next(input)?; + let _text = take_till(0.., is_newline_start).parse_next(input)?; + let _newline = new_line.parse_next(input)?; + Ok(()) } -"; - let processor = Preprocessor::default(); - let result = processor - .preprocess( - &WGSL_ELSE_IFDEF, - &[("THIRD_TEXTURE".to_string(), ShaderDefValue::Bool(true))].into(), - true, - ) - .unwrap(); - assert_eq!( - result - .preprocessed_source - .replace(" ", "") - .replace("\n", "") - .replace("\r", ""), - EXPECTED - .replace(" ", "") - .replace("\n", "") - .replace("\r", "") - ); - } - #[test] - fn process_shader_def_else_ifdef_only_accepts_one_valid_else_ifdef() { - #[rustfmt::skip] - const EXPECTED: &str = r" -struct View { - view_proj: mat4x4, - world_position: vec3, -}; -@group(0) @binding(0) -var view: View; -// Second texture -@group(1) @binding(0) -var sprite_texture: texture_2d; -struct VertexOutput { - @location(0) uv: vec2, - @builtin(position) position: vec4, -}; -@vertex -fn vertex( - @location(0) vertex_position: vec3, - @location(1) vertex_uv: vec2 -) -> VertexOutput { - var out: VertexOutput; - out.uv = vertex_uv; - out.position = view.view_proj * vec4(vertex_position, 1.0); - return out; +fn multi_line_comment(input: &mut Input<'_>) -> PResult<()> { + let _start = "/*".parse_next(input)?; + loop { + if let Some(_end) = opt("*/").parse_next(input)? { + return Ok(()); + } else if let Some(_) = opt(multi_line_comment).parse_next(input)? { + // We found a nested comment, skip it + } else { + // Skip any other character + // TODO: Eof error recovery + let _ = take_till(1.., ('*', '/')).parse_next(input)?; + } + } } -"; - let processor = Preprocessor::default(); - let result = processor - .preprocess( - &WGSL_ELSE_IFDEF, - &[ - ("SECOND_TEXTURE".to_string(), ShaderDefValue::Bool(true)), - ("THIRD_TEXTURE".to_string(), ShaderDefValue::Bool(true)), - ] - .into(), - true, - ) - .unwrap(); - assert_eq!( - result - .preprocessed_source - .replace(" ", "") - .replace("\n", "") - .replace("\r", ""), - EXPECTED - .replace(" ", "") - .replace("\n", "") - .replace("\r", "") - ); + +#[cfg(test)] +mod tests { + use super::*; + + fn replace_comments(input: &str) -> String { + let mut input = input_new(input); + let mut output = String::new(); + loop { + if let Some(span) = opt(single_line_comment.span()) + .parse_next(&mut input) + .unwrap() + { + output.push_str(&" ".repeat(span.len())); + } else if let Some(span) = opt(multi_line_comment.span()) + .parse_next(&mut input) + .unwrap() + { + output.push_str(&" ".repeat(span.len())); + } else if let Some(v) = opt(any::<_, ContextError>).parse_next(&mut input).unwrap() { + output.push(v); + } else { + let _ = eof::<_, ContextError>.parse_next(&mut input).unwrap(); + break; + } + } + output } #[test] - fn process_shader_def_else_ifdef_complicated_nesting() { - // Test some nesting including #else ifdef statements - // 1. Enter an #else ifdef - // 2. Then enter an #else - // 3. Then enter another #else ifdef - - #[rustfmt::skip] - const WGSL_COMPLICATED_ELSE_IFDEF: &str = r" -#ifdef NOT_DEFINED -// not defined -#else ifdef IS_DEFINED -// defined 1 -#ifdef NOT_DEFINED -// not defined -#else -// should be here -#ifdef NOT_DEFINED -// not defined -#else ifdef ALSO_NOT_DEFINED -// not defined -#else ifdef IS_DEFINED -// defined 2 -#endif -#endif -#endif -"; - - #[rustfmt::skip] - const EXPECTED: &str = r" -// defined 1 -// should be here -// defined 2 -"; - let processor = Preprocessor::default(); - let result = processor - .preprocess( - &WGSL_COMPLICATED_ELSE_IFDEF, - &[("IS_DEFINED".to_string(), ShaderDefValue::Bool(true))].into(), - true, - ) - .unwrap(); + fn test_path_simple() { + let input = "a::b::c"; assert_eq!( - result - .preprocessed_source - .replace(" ", "") - .replace("\n", "") - .replace("\r", ""), - EXPECTED - .replace(" ", "") - .replace("\n", "") - .replace("\r", "") + path.parse(input_new(input)).ok(), + Some(vec![0..1, 3..4, 6..7]) ); } - #[test] - fn process_shader_def_else_ifndef() { - #[rustfmt::skip] - const INPUT: &str = r" -#ifdef NOT_DEFINED -fail 1 -#else ifdef ALSO_NOT_DEFINED -fail 2 -#else ifndef ALSO_ALSO_NOT_DEFINED -ok -#else -fail 3 -#endif -"; - - const EXPECTED: &str = r"ok"; - let processor = Preprocessor::default(); - let result = processor.preprocess(&INPUT, &[].into(), true).unwrap(); + fn test_path_trailing() { + // It shouldn't eat the trailing character + let input = "a::b::c::"; + assert_eq!(path.parse(input_new(input)).ok(), None); + let mut inp = input_new(input); assert_eq!( - result - .preprocessed_source - .replace(" ", "") - .replace("\n", "") - .replace("\r", ""), - EXPECTED - .replace(" ", "") - .replace("\n", "") - .replace("\r", "") + path.with_span().parse_next(&mut inp).ok(), + Some((vec![0..1, 3..4, 6..7], 0..7,)) ); + // assert_eq!("::".parse_next(&mut inp).ok(), Some("::")); } #[test] - fn process_shader_def_else_if() { - #[rustfmt::skip] - const INPUT: &str = r" -#ifdef NOT_DEFINED -fail 1 -#else if x == 1 -fail 2 -#else if x == 2 -ok -#else -fail 3 -#endif + fn comment_test() { + let input = r" +not commented +// line commented +not commented +/* block commented on a line */ +not commented +// line comment with a /* block comment unterminated +not commented +/* block comment + spanning lines */ +not commented +/* block comment + spanning lines and with // line comments + even with a // line commented terminator */ +not commented "; - const EXPECTED: &str = r"ok"; - let processor = Preprocessor::default(); - let result = processor - .preprocess( - &INPUT, - &[("x".to_owned(), ShaderDefValue::Int(2))].into(), - true, - ) - .unwrap(); + let replaced = replace_comments(input); + assert_eq!(replaced.len(), input.len()); assert_eq!( - result - .preprocessed_source - .replace(" ", "") - .replace("\n", "") - .replace("\r", ""), - EXPECTED - .replace(" ", "") - .replace("\n", "") - .replace("\r", "") + replaced + .lines() + .zip(input.lines()) + .find(|(line, original)| { + (*line != "not commented" && !line.chars().all(|c| c == ' ')) + || line.len() != original.len() + }), + None ); + + let partial_tests = [ + ( + "1.0 /* block comment with a partial line comment on the end *// 2.0", + "1.0 / 2.0", + ), + ( + "1.0 /* block comment with a partial block comment on the end */* 2.0", + "1.0 * 2.0", + ), + ( + "1.0 /* block comment 1 *//* block comment 2 */ * 2.0", + "1.0 * 2.0", + ), + ( + "1.0 /* block comment with real line comment after */// line comment", + "1.0 ", + ), + ]; + + for &(input, expected) in partial_tests.iter() { + assert_eq!(&replace_comments(input), expected); + } } } diff --git a/src/compose/preprocess1.rs b/src/compose/preprocess1.rs deleted file mode 100644 index a61f1d5..0000000 --- a/src/compose/preprocess1.rs +++ /dev/null @@ -1,722 +0,0 @@ -use std::{collections::HashSet, ops::Range}; - -use winnow::{ - combinator::{alt, empty, eof, fail, opt, peek, preceded, repeat, separated, seq, terminated}, - error::{ContextError, StrContext}, - stream::Recoverable, - token::{any, none_of, one_of, take_till, take_while}, - Located, PResult, Parser, -}; - -/** - * The abstract syntax trees do not include spaces or comments. They are implicity there between adjacent tokens. - * It is also missing a lot of filler tokens, like semicolons, commas, and braces. - * The syntax tree only has ranges, and needs the original source code to extract the actual text. - * - * If we ever want to have a full concrete syntax tree, we should look into https://github.com/domenicquirl/cstree - */ - -pub type Input<'a> = Recoverable, ContextError>; -pub fn input_new(input: &str) -> Input { - Recoverable::new(Located::new(input)) -} - -pub struct Preprocessed { - pub parts: Vec, -} -impl Preprocessed { - pub fn get_module_names<'a, 'b>(&'b self, input: &'a str) -> impl Iterator + 'b - where - 'a: 'b, - { - self.parts.iter().filter_map(|v| match v { - PreprocessorPart::DefineImportPath(DefineImportPath { path }) => { - path.as_ref().map(|v| &input[v.clone()]) - } - _ => None, - }) - } - pub fn get_imports(&self, input: &str) -> Vec { - fn to_stack<'a>(input: &'a str, ranges: &[Range]) -> Vec<&'a str> { - ranges - .iter() - .map(|range| &input[range.clone()]) - .collect::>() - } - - fn walk_import_tree<'a>( - input: &'a str, - tree: &ImportTree, - stack: &[&'a str], - ) -> Vec { - let (name_range, path_ranges) = match tree { - ImportTree::Path(path_ranges) => (path_ranges.last().unwrap().clone(), path_ranges), - ImportTree::Alias { - path: path_ranges, - alias: alias_range, - } => ( - alias_range - .clone() - .unwrap_or_else(|| path_ranges.last().unwrap().clone()), - path_ranges, - ), - ImportTree::Children { path, children } => { - let extended_stack = [stack, &to_stack(input, path)].concat(); - return children - .iter() - .flat_map(|child| walk_import_tree(input, child, &extended_stack)) - .collect(); - } - }; - - let offset = name_range.start; - let name = input[name_range].to_string(); - let path = [stack, &to_stack(input, &path_ranges)].concat().join("::"); - vec![ResolvedImport { name, path, offset }] - } - - self.parts - .iter() - .filter_map(move |v| match v { - PreprocessorPart::Import(v) => Some(v), - _ => None, - }) - .filter_map(|v| match &v.tree { - Some((tree, _)) => Some(walk_import_tree(input, tree, &[])), - None => None, - }) - .flatten() - .collect::>() - } - - pub fn get_used_defs(&self, input: &str) -> HashSet { - self.parts - .iter() - .filter_map(|v| match v { - PreprocessorPart::If(v) => v.name.as_ref(), - PreprocessorPart::IfOp(v) => v.name.as_ref(), - PreprocessorPart::UseDefine(v) => v.name.as_ref(), - _ => None, - }) - .map(|v| input[v.clone()].to_owned()) - .collect() - } -} -pub struct ResolvedImport { - pub offset: usize, - pub name: String, - pub path: String, -} - -pub enum PreprocessorPart { - Version(VersionDirective), - If(IfDefDirective), - IfOp(IfOpDirective), - Else(ElseDirective), - EndIf(EndIfDirective), - UseDefine(UseDefineDirective), - DefineShaderDef(DefineShaderDef), - DefineImportPath(DefineImportPath), - Import(ImportDirective), - UnkownDirective(Range), - /// Normal shader code - Text(Range), -} - -// Note: This is a public API that lower level tools may use. It's a recoverable parser. -pub fn preprocess(input: &mut Input<'_>) -> PResult { - // All of the directives start with a #. - // And most of the directives have to be on their own line. - let mut parts = Vec::new(); - let mut start_text = empty.span().parse_next(input)?.start; - loop { - // I'm at the start of a line. Let's try parsing a preprocessor directive. - if let Some(_) = opt(spaces_single_line).parse_next(input)? { - if let Some(_) = opt(peek('#').span()).parse_next(input)? { - // It's a preprocessor directive - let (part, span): (Option<_>, _) = alt(( - version.map(PreprocessorPart::Version), - if_directive.map(|v| match v { - IfDirective::If(v) => PreprocessorPart::If(v), - IfDirective::IfOp(v) => PreprocessorPart::IfOp(v), - IfDirective::Else(v) => PreprocessorPart::Else(v), - }), - end_if_directive.map(PreprocessorPart::EndIf), - use_define_directive.map(PreprocessorPart::UseDefine), - define_import_path.map(PreprocessorPart::DefineImportPath), - define_shader_def.map(PreprocessorPart::DefineShaderDef), - import_directive.map(PreprocessorPart::Import), - fail.context(StrContext::Label("Unknown directive")), - )) - .resume_after(take_till(0.., is_newline_start).map(|_| ())) - .with_span() - .parse_next(input)?; - parts.push(PreprocessorPart::Text(start_text..span.start)); - start_text = span.end; - parts.push(part.unwrap_or_else(move || PreprocessorPart::UnkownDirective(span))); - continue; - } - } - - // Normal line - loop { - let text = take_till(1.., |c: char| is_newline_start(c) || c == '#') - .span() - .parse_next(input)?; - - if let Some(_) = opt(new_line).parse_next(input)? { - // Nice, we finished a line - break; - } else if let Some((use_define, span)) = - opt(use_define_directive.with_span()).parse_next(input)? - { - parts.push(PreprocessorPart::Text(start_text..span.start)); - start_text = span.end; - parts.push(PreprocessorPart::UseDefine(use_define)); - // Continue parsing the line - } else if let Some(_) = opt(eof).parse_next(input)? { - // We reached the end of the file - parts.push(PreprocessorPart::Text(start_text..text.end)); - return Ok(Preprocessed { parts }); - } else { - // It's a # that we don't care about - // Skip it and continue parsing the line - let _ = any.parse_next(input)?; - } - } - } -} - -pub struct VersionDirective { - version: Option>, -} -impl VersionDirective { - pub fn version_number(&self, input: &str) -> Option { - self.version - .as_ref() - .and_then(|v| (&input[v.clone()]).parse::().ok()) - } -} - -pub fn version(input: &mut Input<'_>) -> PResult { - seq! {VersionDirective{ - _: "#version".span(), - _: spaces_single_line.resume_after(empty), - version: take_while(1.., |c:char| c.is_ascii_digit()).span().resume_after(empty), - _: spaces_until_new_line - }} - .parse_next(input) -} - -/// Note: We're disallowing spaces between the `#` and the `ifdef`. -/// `#ifdef {name}` or `#else ifdef {name}` or `#ifndef {name}` or `#else ifndef {name` -pub struct IfDefDirective { - pub is_else_if: bool, - pub is_not: bool, - pub name: Option>, -} - -/// `#ifop {name} {op} {value}` or `#else ifop {name} {op} {value}` -pub struct IfOpDirective { - pub is_else_if: bool, - pub name: Option>, - pub op: Option>, - pub value: Option>, -} - -pub enum IfDirective { - If(IfDefDirective), - IfOp(IfOpDirective), - Else(ElseDirective), -} - -pub fn if_directive(input: &mut Input<'_>) -> PResult { - #[derive(PartialEq, Eq)] - enum Start { - IfDef, - IfNotDef, - IfOp, - Else, - } - let (start, is_else) = alt(( - "#ifop".map(|_| (Start::IfOp, false)), - "#ifdef".map(|_| (Start::IfDef, false)), - "#ifndef".map(|_| (Start::IfNotDef, false)), - ( - "#else", - spaces_single_line.resume_after(empty), - alt(( - "ifdef".map(|_| Start::IfDef), - "ifndef".map(|_| Start::IfNotDef), - "ifop".map(|_| Start::IfOp), - spaces_until_new_line.map(|_| Start::Else), - )), - ) - .map(|(_, _, next)| (next, true)), - )) - .parse_next(input)?; - - match start { - Start::IfDef | Start::IfNotDef => { - let _ = spaces_single_line.resume_after(empty).parse_next(input)?; - let name = shader_def_name.resume_after(empty).parse_next(input)?; - let _ = spaces_until_new_line.parse_next(input)?; - Ok(IfDirective::If(IfDefDirective { - is_else_if: is_else, - is_not: start == Start::IfNotDef, - name, - })) - } - Start::IfOp => { - let _ = spaces_single_line.resume_after(empty).parse_next(input)?; - let name = shader_def_name.resume_after(empty).parse_next(input)?; - let _ = opt(spaces_single_line).parse_next(input)?; - let op = alt(("==", "!=", "<", "<=", ">", ">=")) - .span() - .resume_after(empty) - .parse_next(input)?; - let _ = opt(spaces_single_line).parse_next(input)?; - let value = shader_def_value.resume_after(empty).parse_next(input)?; - let _ = spaces_until_new_line.parse_next(input)?; - Ok(IfDirective::IfOp(IfOpDirective { - is_else_if: is_else, - name, - op, - value, - })) - } - Start::Else => Ok(IfDirective::Else(ElseDirective)), - } -} - -/// `#else` -pub struct ElseDirective; - -/// `#endif` -pub struct EndIfDirective; - -pub fn end_if_directive(input: &mut Input<'_>) -> PResult { - seq! {EndIfDirective{ - _: "#endif", - _: spaces_single_line.resume_after(empty), - _: spaces_until_new_line - }} - .parse_next(input) -} - -/// Note: We're disallowing the previous `#ANYTHING` syntax, since it's rarely used and error prone -/// (a misspelled `#inport` would get mistaken for a `#ANYTHING``). -/// `#{name of defined value}`` -pub struct UseDefineDirective { - pub name: Option>, -} - -/// Remember that this one doesn't need to be on its own line -pub fn use_define_directive(input: &mut Input<'_>) -> PResult { - seq! {UseDefineDirective{ - _: "#{", - _: spaces_single_line.resume_after(empty), - name: shader_def_name.resume_after(empty), - _: spaces_single_line.resume_after(empty), - _: "}".resume_after(empty) - }} - .parse_next(input) -} - -/// `#define {name} {value}`, except it can only be used with other preprocessor macros. -/// Unlike its C cousin, it doesn't aggressively replace text. -pub struct DefineShaderDef { - pub name: Option>, - pub value: Option>, -} - -pub fn define_shader_def(input: &mut Input<'_>) -> PResult { - // Technically I'm changing the #define behaviour - // I'm no longer allowing redefining numbers, like #define 3 a|b - seq! {DefineShaderDef{ - _: "#define", - _: spaces_single_line.resume_after(empty), - name: shader_def_name.resume_after(empty), - _: spaces_single_line.resume_after(empty), - value: opt(shader_def_value), - _: spaces_until_new_line - }} - .parse_next(input) -} - -fn shader_def_name(input: &mut Input<'_>) -> PResult> { - ( - one_of(|c: char| c.is_ascii_alphabetic() || c == '_'), - take_while(0.., |c: char| c.is_ascii_alphanumeric() || c == '_'), - ) - .span() - .parse_next(input) -} - -fn shader_def_value(input: &mut Input<'_>) -> PResult> { - take_while(1.., |c: char| { - c.is_ascii_alphanumeric() || c == '_' || c == '-' - }) - .span() - .parse_next(input) -} - -pub struct DefineImportPath { - pub path: Option>, -} - -pub fn define_import_path(input: &mut Input<'_>) -> PResult { - seq! {DefineImportPath{ - _: "#define_import_path", - _: spaces_single_line.resume_after(empty), - path: take_while(1.., |c: char| !c.is_whitespace()).span().resume_after(empty), - _: spaces_until_new_line - }} - .parse_next(input) -} - -/// Formal grammar -/// ```ebnf -/// ::= "#import" "::"? ";"? -/// -/// ::= ( "as" | "::" "{" "}")? -/// ::= ("," )* ","? -/// -/// ::= ( | ) ("::" ( | ) )* -/// ::= ([a-z]) ([a-z] | [0-9])* -/// ::= "\"" + "\"" -/// ::= [a-z] -/// -/// ::= " "+ -/// ``` -/// -/// Can be tested on https://bnfplayground.pauliankline.com/ -/// -/// Except that -/// - `` should be Unicode aware -/// - `` should use the XID rules instead of only allowing lowercase letters -/// - `` should be a string with at least one character, and follow the usual "quotes and \\ backslash for escaping" rules -/// - spaces are allowed between every token -/// ``` -pub struct ImportDirective { - pub root_specifier: Option>, - pub tree: Option<(ImportTree, Range)>, -} - -pub fn import_directive(input: &mut Input<'_>) -> PResult { - seq! {ImportDirective { - _ : "#import", - _: spaces.resume_after(empty), - root_specifier: opt("::".span()), - tree: import_tree.with_span().resume_after(empty), - _: opt(spaces), - _: opt(";"), - }} - .parse_next(input) -} - -#[derive(Debug, Clone)] -pub enum ImportTree { - Path(Vec>), - Alias { - path: Vec>, - alias: Option>, - }, - Children { - path: Vec>, - children: Vec, - }, -} - -fn import_tree(input: &mut Input<'_>) -> PResult { - let path = path.parse_next(input)?; - let s = opt(spaces).parse_next(input)?; - - if s.is_some() { - let as_token = opt("as").parse_next(input)?; - if as_token.is_some() { - let _ = spaces.resume_after(empty).parse_next(input)?; - let alias = identifier.resume_after(empty).parse_next(input)?; - return Ok(ImportTree::Alias { path, alias }); - } - } - - if let Some(_) = opt("::{").parse_next(input)? { - let _ = opt(spaces).parse_next(input)?; - let children = import_trees - .retry_after( - // TODO: This recovery can explode and eat the whole file - (take_till(0.., (',', '}')), ",").map(|_| ()), - ) - // TODO: This recovery can explode and eat the whole file - .resume_after((take_till(0.., '}'), '}').map(|_| ())) - .parse_next(input)? - .unwrap_or_default(); - - let _ = opt(spaces).parse_next(input)?; - let _ = "}".resume_after(empty).parse_next(input)?; - return Ok(ImportTree::Children { path, children }); - } - - Ok(ImportTree::Path(path)) -} - -fn import_trees(input: &mut Input<'_>) -> PResult> { - terminated( - separated(1.., import_tree, (opt(spaces), ",", opt(spaces))), - (opt(spaces), ","), - ) - .parse_next(input) -} - -fn path(input: &mut Input<'_>) -> PResult>> { - separated( - 1.., - alt((nonempty_string, identifier)), - (opt(spaces), "::", opt(spaces)), - ) - .parse_next(input) -} - -fn identifier(input: &mut Input<'_>) -> PResult> { - ( - one_of(|c: char| unicode_ident::is_xid_start(c)), - take_while(0.., |c: char| unicode_ident::is_xid_continue(c)), - ) - .span() - .parse_next(input) -} - -fn nonempty_string(input: &mut Input<'_>) -> PResult> { - quoted_string.verify(|s| s.len() > 2).parse_next(input) -} - -fn quoted_string(input: &mut Input<'_>) -> PResult> { - // See https://docs.rs/winnow/latest/winnow/_topic/json/index.html - preceded( - '\"', - terminated( - repeat(0.., string_character).fold(|| (), |a, _| a), - '\"'.resume_after(empty), - ), - ) - .span() - .parse_next(input) -} -fn string_character(input: &mut Input<'_>) -> PResult<()> { - let c = none_of('\"').parse_next(input)?; - if c == '\\' { - let _ = any.parse_next(input)?; - } - Ok(()) -} - -pub struct Spaces { - /// Comments that are in this "spaces" block - pub comments: Vec>, -} - -/// Parses at least one whitespace or comment. -fn spaces(input: &mut Input<'_>) -> PResult { - repeat( - 1.., - alt(( - take_while(1.., |c: char| c.is_whitespace()).map(|_| None), - single_line_comment.span().map(|c| Some(c)), - multi_line_comment.span().map(|c| Some(c)), - )), - ) - .fold( - || Vec::new(), - |mut comments, comment| { - if let Some(comment) = comment { - comments.push(comment); - } - comments - }, - ) - .map(|comments| Spaces { comments }) - .parse_next(input) -} - -/// Parses at least one non-newline whitespace or comment. -/// Preprocessor directives are always on their own line. So they need a slightly different spaces parser. -fn spaces_single_line(input: &mut Input<'_>) -> PResult { - repeat( - 1.., - alt(( - take_while(1.., |c: char| c.is_whitespace() && !is_newline_start(c)).map(|_| None), - single_line_comment.span().map(|c| Some(c)), - multi_line_comment.span().map(|c| Some(c)), - )), - ) - .fold( - || Vec::new(), - |mut comments, comment| { - if let Some(comment) = comment { - comments.push(comment); - } - comments - }, - ) - .map(|comments| Spaces { comments }) - .parse_next(input) -} - -/// Checks if it's part of a Unicode line break, according to https://www.w3.org/TR/WGSL/#line-break -fn is_newline_start(c: char) -> bool { - c == '\u{000A}' - || c == '\u{000B}' - || c == '\u{000C}' - || c == '\u{000D}' - || c == '\u{0085}' - || c == '\u{2028}' - || c == '\u{2029}' -} - -fn spaces_until_new_line(input: &mut Input<'_>) -> PResult { - let spaces = opt(spaces_single_line).parse_next(input)?; - let _newline = new_line - .retry_after(take_till(0.., is_newline_start).map(|_| ())) - .parse_next(input)?; - Ok(spaces.unwrap_or(Spaces { - comments: Vec::new(), - })) -} - -fn new_line(input: &mut Input<'_>) -> PResult<()> { - alt(( - "\u{000D}\u{000A}".map(|_| ()), - one_of(is_newline_start).map(|_| ()), - eof.map(|_| ()), - )) - .parse_next(input) -} - -fn single_line_comment(input: &mut Input<'_>) -> PResult<()> { - let _start = "//".parse_next(input)?; - let _text = take_till(0.., is_newline_start).parse_next(input)?; - let _newline = new_line.parse_next(input)?; - Ok(()) -} - -fn multi_line_comment(input: &mut Input<'_>) -> PResult<()> { - let _start = "/*".parse_next(input)?; - loop { - if let Some(_end) = opt("*/").parse_next(input)? { - return Ok(()); - } else if let Some(_) = opt(multi_line_comment).parse_next(input)? { - // We found a nested comment, skip it - } else { - // Skip any other character - // TODO: Eof error recovery - let _ = take_till(1.., ('*', '/')).parse_next(input)?; - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn replace_comments(input: &str) -> String { - let mut input = input_new(input); - let mut output = String::new(); - loop { - if let Some(span) = opt(single_line_comment.span()) - .parse_next(&mut input) - .unwrap() - { - output.push_str(&" ".repeat(span.len())); - } else if let Some(span) = opt(multi_line_comment.span()) - .parse_next(&mut input) - .unwrap() - { - output.push_str(&" ".repeat(span.len())); - } else if let Some(v) = opt(any::<_, ContextError>).parse_next(&mut input).unwrap() { - output.push(v); - } else { - let _ = eof::<_, ContextError>.parse_next(&mut input).unwrap(); - break; - } - } - output - } - - #[test] - fn test_path_simple() { - let input = "a::b::c"; - assert_eq!( - path.parse(input_new(input)).ok(), - Some(vec![0..1, 3..4, 6..7]) - ); - } - #[test] - fn test_path_trailing() { - // It shouldn't eat the trailing character - let input = "a::b::c::"; - assert_eq!(path.parse(input_new(input)).ok(), None); - let mut inp = input_new(input); - assert_eq!( - path.with_span().parse_next(&mut inp).ok(), - Some((vec![0..1, 3..4, 6..7], 0..7,)) - ); - // assert_eq!("::".parse_next(&mut inp).ok(), Some("::")); - } - - #[test] - fn comment_test() { - let input = r" -not commented -// line commented -not commented -/* block commented on a line */ -not commented -// line comment with a /* block comment unterminated -not commented -/* block comment - spanning lines */ -not commented -/* block comment - spanning lines and with // line comments - even with a // line commented terminator */ -not commented -"; - - let replaced = replace_comments(input); - assert_eq!(replaced.len(), input.len()); - assert_eq!( - replaced - .lines() - .zip(input.lines()) - .find(|(line, original)| { - (*line != "not commented" && !line.chars().all(|c| c == ' ')) - || line.len() != original.len() - }), - None - ); - - let partial_tests = [ - ( - "1.0 /* block comment with a partial line comment on the end *// 2.0", - "1.0 / 2.0", - ), - ( - "1.0 /* block comment with a partial block comment on the end */* 2.0", - "1.0 * 2.0", - ), - ( - "1.0 /* block comment 1 *//* block comment 2 */ * 2.0", - "1.0 * 2.0", - ), - ( - "1.0 /* block comment with real line comment after */// line comment", - "1.0 ", - ), - ]; - - for &(input, expected) in partial_tests.iter() { - assert_eq!(&replace_comments(input), expected); - } - } -} diff --git a/src/compose/test.rs b/src/compose/test.rs index 846ac71..e8084f9 100644 --- a/src/compose/test.rs +++ b/src/compose/test.rs @@ -12,8 +12,8 @@ mod test { }; use crate::compose::{ - get_preprocessor_data, ComposableModuleDescriptor, Composer, ImportDefinition, - NagaModuleDescriptor, ShaderDefValue, ShaderLanguage, ShaderType, + ComposableModuleDescriptor, Composer, NagaModuleDescriptor, ShaderDefValue, ShaderLanguage, + ShaderType, }; macro_rules! output_eq { @@ -520,6 +520,8 @@ mod test { #[cfg(feature = "test_shader")] #[test] fn additional_import() { + use crate::compose::{AdditionalImport, ModuleName}; + let mut composer = Composer::default(); composer .add_composable_module(ComposableModuleDescriptor { @@ -542,9 +544,9 @@ mod test { .make_naga_module(NagaModuleDescriptor { source: include_str!("tests/add_imports/top.wgsl"), file_path: "tests/add_imports/top.wgsl", - additional_imports: &[ImportDefinition { - import: "plugin".to_owned(), - ..Default::default() + additional_imports: &[AdditionalImport { + module: ModuleName::new("plugin"), + items: Default::default(), }], ..Default::default() }) @@ -575,9 +577,9 @@ mod test { source: include_str!("tests/add_imports/top.wgsl"), file_path: "tests/add_imports/top.wgsl", as_name: Some("test_module".to_owned()), - additional_imports: &[ImportDefinition { - import: "plugin".to_owned(), - ..Default::default() + additional_imports: &[AdditionalImport { + module: ModuleName::new("plugin"), + items: Default::default(), }], ..Default::default() }) @@ -1126,6 +1128,7 @@ mod test { assert_eq!(test_shader(&mut composer), 36.0); } + /* #[test] fn test_bevy_path_imports() { let (_, mut imports, _) = @@ -1133,36 +1136,36 @@ mod test { imports.iter_mut().for_each(|import| { import.items.sort(); }); - imports.sort_by(|a, b| a.import.cmp(&b.import)); + imports.sort_by(|a, b| a.module.cmp(&b.module)); assert_eq!( imports, vec![ ImportDefinition { - import: "\"shaders/skills/hit.wgsl\"".to_owned(), + module: "\"shaders/skills/hit.wgsl\"".to_owned(), items: vec!["frag".to_owned(), "vert".to_owned(),], }, ImportDefinition { - import: "\"shaders/skills/lightning.wgsl\"".to_owned(), + module: "\"shaders/skills/lightning.wgsl\"".to_owned(), items: vec!["frag".to_owned(), "vert".to_owned(),], }, ImportDefinition { - import: "\"shaders/skills/lightning_ring.wgsl\"".to_owned(), + module: "\"shaders/skills/lightning_ring.wgsl\"".to_owned(), items: vec!["frag".to_owned(), "vert".to_owned(),], }, ImportDefinition { - import: "\"shaders/skills/magic_arrow.wgsl\"".to_owned(), + module: "\"shaders/skills/magic_arrow.wgsl\"".to_owned(), items: vec!["frag".to_owned(), "vert".to_owned(),], }, ImportDefinition { - import: "\"shaders/skills/orb.wgsl\"".to_owned(), + module: "\"shaders/skills/orb.wgsl\"".to_owned(), items: vec!["frag".to_owned(), "vert".to_owned(),], }, ImportDefinition { - import: "\"shaders/skills/railgun_trail.wgsl\"".to_owned(), + module: "\"shaders/skills/railgun_trail.wgsl\"".to_owned(), items: vec!["frag".to_owned(), "vert".to_owned(),], }, ImportDefinition { - import: "\"shaders/skills/shared.wgsl\"".to_owned(), + module: "\"shaders/skills/shared.wgsl\"".to_owned(), items: vec![ "Vertex".to_owned(), "VertexOutput".to_owned(), @@ -1170,16 +1173,16 @@ mod test { ], }, ImportDefinition { - import: "\"shaders/skills/slash.wgsl\"".to_owned(), + module: "\"shaders/skills/slash.wgsl\"".to_owned(), items: vec!["frag".to_owned(), "vert".to_owned(),], }, ImportDefinition { - import: "\"shaders/skills/sound.wgsl\"".to_owned(), + module: "\"shaders/skills/sound.wgsl\"".to_owned(), items: vec!["frag".to_owned(), "vert".to_owned(),], }, ] ); - } + } */ #[test] fn test_quoted_import_dup_name() {