diff --git a/Cargo.toml b/Cargo.toml index 39d5f21..b2a34e2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ default = ["test_shader"] test_shader = [] prune = [] override_any = [] +allow_deprecated = [] [dependencies] naga = { version = "0.13", features = ["wgsl-in", "wgsl-out", "glsl-in", "glsl-out", "clone", "span"] } diff --git a/readme.md b/readme.md index 2a2ef0c..1ec57f6 100644 --- a/readme.md +++ b/readme.md @@ -32,15 +32,17 @@ fn my_func() -> f32 { } ``` -shaders can then import the module with an `#import` directive (with an optional `as` name). at point of use, imported items must be qualified: +alternatively the module name can be specified as an argument to `Composer::add_composable_module`. + +shaders can then import the module with an `#import` directive (with an optional `as` name) : ```wgsl -#import my_module -#import my_other_module as Mod2 +#import my_module; +#import my_other_module as mod2; fn main() -> f32 { let x = my_module::my_func(); - let y = Mod2::my_other_func(); + let y = mod2::my_other_func(); return x*y; } ``` @@ -48,18 +50,33 @@ fn main() -> f32 { or import a comma-separated list of individual items : ```wgsl -#import my_module my_func, my_const +#import my_module::{my_func, my_const} fn main() -> f32 { return my_func(my_const); } ``` +Some rust-style import syntax is supported, and items can be directly imported using the fully qualified item name : + +```wgsl +#import my_package::{ + first_module::{item_one as item, item_two}, + second_module::submodule, +} + +fn main() -> f32 { + return item + item_two + submodule::subitem + my_package::third_module::item; +} +``` + +`module::self` and `module::*` are not currently supported. + imports can be nested - modules may import other modules, but not recursively. when a new module is added, all its `#import`s must already have been added. the same module can be imported multiple times by different modules in the import tree. there is no overlap of namespaces, so the same function names (or type, constant, or variable names) may be used in different modules. -note: when importing an item with the `#import module item` directive, the final shader will include the required dependencies (bindings, globals, consts, other functions) of the imported item, but will not include the rest of the imported module. it will however still include all of any modules imported by the imported module. this is probably not desired in general and may be fixed in a future version. currently for a more complete culling of unused dependencies the `prune` module can be used. +note: the final shader will include the required dependencies (bindings, globals, consts, other functions) of any imported items that are used, but will not include the rest of the imported module. ## overriding functions @@ -79,6 +96,8 @@ override fn Lighting::point_light (world_position: vec3) -> vec3 { } ``` +overrides must either be declared in the top-level shader, or the module containing the override must be imported as an `additional_import` in a `Composer::add_composable_module` or `Composer::make_naga_module` call. using `#import` to import a module with overrides will not work due to tree-shaking. + override function definitions cause *all* calls to the original function in the entire shader scope to be replaced by calls to the new function, with the exception of calls within the override function itself. the function signature of the override must match the base function. diff --git a/src/compose/error.rs b/src/compose/error.rs index ec06ae7..e6b86d2 100644 --- a/src/compose/error.rs +++ b/src/compose/error.rs @@ -8,10 +8,7 @@ use codespan_reporting::{ use thiserror::Error; use tracing::trace; -use super::{ - preprocess::{PreprocessOutput, PreprocessorMetaData}, - Composer, ShaderDefValue, -}; +use super::{preprocess::PreprocessOutput, Composer, ShaderDefValue}; use crate::{compose::SPAN_SHIFT, redirect::RedirectError}; #[derive(Debug)] @@ -42,7 +39,7 @@ impl ErrSource { let raw_source = &composer.module_sets.get(name).unwrap().sanitized_source; let Ok(PreprocessOutput { preprocessed_source: source, - meta: PreprocessorMetaData { imports, .. }, + .. }) = composer .preprocessor .preprocess(raw_source, defs, composer.validate) @@ -50,10 +47,6 @@ impl ErrSource { return Default::default(); }; - let Ok(source) = composer.substitute_shader_string(&source, &imports) else { - return Default::default(); - }; - Cow::Owned(source) } ErrSource::Constructing { source, .. } => Cow::Borrowed(source), @@ -77,6 +70,8 @@ pub struct ComposerError { #[derive(Debug, Error)] pub enum ComposerErrorInner { + #[error("{0}")] + ImportParseError(String, usize), #[error("required import '{0}' not found")] ImportNotFound(String, usize), #[error("{0}")] @@ -215,6 +210,10 @@ impl ComposerError { vec![Label::primary((), *pos..*pos)], vec![format!("missing import '{msg}'")], ), + ComposerErrorInner::ImportParseError(msg, pos) => ( + vec![Label::primary((), *pos..*pos)], + vec![format!("invalid import spec: '{msg}'")], + ), ComposerErrorInner::WgslParseError(e) => ( e.labels() .map(|(range, msg)| { diff --git a/src/compose/mod.rs b/src/compose/mod.rs index b679aff..36a2070 100644 --- a/src/compose/mod.rs +++ b/src/compose/mod.rs @@ -141,8 +141,10 @@ use self::preprocess::Preprocessor; pub mod comment_strip_iter; pub mod error; +pub mod parse_imports; pub mod preprocess; mod test; +pub mod tokenizer; pub mod util; #[derive(Hash, PartialEq, Eq, Clone, Copy, Debug, Default)] @@ -263,6 +265,8 @@ pub struct ComposableModuleDefinition { modules: HashMap, // used in spans when this module is included module_index: usize, + // preprocessor meta data + // metadata: PreprocessorMetaData, } impl ComposableModuleDefinition { @@ -289,17 +293,10 @@ impl ComposableModuleDefinition { } } -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct ImportDefinition { pub import: String, - pub as_name: Option, - pub items: Option>, -} - -impl ImportDefinition { - fn as_name(&self) -> &str { - self.as_name.as_deref().unwrap_or(self.import.as_str()) - } + pub items: Vec, } #[derive(Debug, Clone)] @@ -341,25 +338,37 @@ impl Default for Composer { module_sets: Default::default(), module_index: Default::default(), preprocessor: Preprocessor::default(), - check_decoration_regex: Regex::new(format!("({}|{})", regex_syntax::escape(DECORATION_PRE), regex_syntax::escape(DECORATION_OVERRIDE_PRE)).as_str()).unwrap(), + check_decoration_regex: Regex::new( + format!( + "({}|{})", + regex_syntax::escape(DECORATION_PRE), + regex_syntax::escape(DECORATION_OVERRIDE_PRE) + ) + .as_str(), + ) + .unwrap(), undecorate_regex: Regex::new( format!( - "{}([A-Z0-9]*){}", + r"([\w\d_]+){}([A-Z0-9]*){}", regex_syntax::escape(DECORATION_PRE), regex_syntax::escape(DECORATION_POST) ) .as_str(), ) .unwrap(), - virtual_fn_regex: Regex::new(r"(?P[\s]*virtual\s+fn\s+)(?P[^\s]+)(?P\s*)\(").unwrap(), + virtual_fn_regex: Regex::new( + r"(?P[\s]*virtual\s+fn\s+)(?P[^\s]+)(?P\s*)\(", + ) + .unwrap(), override_fn_regex: Regex::new( format!( - r"(?P[\s]*override\s+fn\s*){}(?P[^\s]+){}(?P[^\s]+)(?P\s*)\(", + r"(override\s+fn\s+)([^\s]+){}([\w\d]+){}(\s*)\(", regex_syntax::escape(DECORATION_PRE), regex_syntax::escape(DECORATION_POST) ) - .as_str() - ).unwrap(), + .as_str(), + ) + .unwrap(), undecorate_override_regex: Regex::new( format!( "{}([A-Z0-9]*){}", @@ -375,11 +384,11 @@ impl Default for Composer { } } -const DECORATION_PRE: &str = "_naga_oil_mod_"; -const DECORATION_POST: &str = "_member"; +const DECORATION_PRE: &str = "X_naga_oil_mod_X"; +const DECORATION_POST: &str = "X"; // must be same length as DECORATION_PRE for spans to work -const DECORATION_OVERRIDE_PRE: &str = "_naga_oil_vrt_"; +const DECORATION_OVERRIDE_PRE: &str = "X_naga_oil_vrt_X"; struct IrBuildResult { module: naga::Module, @@ -390,14 +399,14 @@ struct IrBuildResult { impl Composer { pub fn decorated_name(module_name: Option<&str>, item_name: &str) -> String { match module_name { - Some(module_name) => format!("{}{}", Self::decorate(module_name), item_name), + Some(module_name) => format!("{}{}", item_name, Self::decorate(module_name)), None => item_name.to_owned(), } } - fn decorate(as_name: &str) -> String { - let as_name = data_encoding::BASE32_NOPAD.encode(as_name.as_bytes()); - format!("{DECORATION_PRE}{as_name}{DECORATION_POST}") + fn decorate(module: &str) -> String { + let encoded = data_encoding::BASE32_NOPAD.encode(module.as_bytes()); + format!("{DECORATION_PRE}{encoded}{DECORATION_POST}") } fn decode(from: &str) -> String { @@ -408,7 +417,11 @@ impl Composer { let undecor = self .undecorate_regex .replace_all(string, |caps: ®ex::Captures| { - format!("{}::", Self::decode(caps.get(1).unwrap().as_str())) + format!( + "{}::{}", + Self::decode(caps.get(2).unwrap().as_str()), + caps.get(1).unwrap().as_str() + ) }); let undecor = @@ -451,99 +464,6 @@ impl Composer { substituted_source.into_owned() } - fn substitute_shader_string( - &self, - source: &str, - imports: &[ImportDefWithOffset], - ) -> Result { - // sort imports by decreasing length so we don't accidentally replace substrings of a longer import - let mut imports = imports.to_vec(); - imports.sort_by_key(|import| usize::MAX - import.definition.as_name().len()); - - let mut imported_items = HashMap::new(); - let mut substituted_source = source.to_owned(); - - for import in imports { - match import.definition.items { - Some(items) => { - // gather individual imported items - for item in &items { - imported_items.insert( - item.clone(), - format!("{}{}", Self::decorate(&import.definition.import), item), - ); - } - } - None => { - // replace the module name directly - substituted_source = substituted_source.replace( - format!("{}::", import.definition.as_name()).as_str(), - &Self::decorate(&import.definition.import), - ); - } - } - } - - // map individually imported items - let mut item_substituted_source = String::default(); - let mut current_word = String::default(); - let mut line_is_directive = None; - let mut is_valid_import_substitution_point = true; - - for char in substituted_source.chars() { - if !current_word.is_empty() { - if unicode_ident::is_xid_continue(char) { - current_word.push(char); - continue; - } - - let mut output = ¤t_word; - - // substitute current word if we are not writing a directive (e.g. `#import xyz`) - if is_valid_import_substitution_point { - if let Some(replacement) = imported_items.get(¤t_word) { - output = replacement; - } - } - - item_substituted_source += output; - current_word.clear(); - } - - // set current directive state - if char == '\r' || char == '\n' { - // new line -> could be - line_is_directive = None; - } else if line_is_directive.is_none() { - line_is_directive = match char { - // first non-ws char is a #, this is a directive - '#' => Some(true), - // whitespace, still unknown - ' ' | '\t' => None, - // non-whitespace and not a '#', not a directive - _ => Some(false), - } - } - - if unicode_ident::is_xid_start(char) { - current_word.push(char); - } else { - item_substituted_source.push(char); - // we should only substitute global names - // '.' -> avoid substituting members with name == import item - // '@' -> avoid substituting annotations - if char == '.' || char == '@' { - is_valid_import_substitution_point = false; - } else { - is_valid_import_substitution_point = !line_is_directive.unwrap_or(false); - } - } - } - substituted_source = item_substituted_source; - - Ok(substituted_source) - } - fn naga_to_string( &self, naga_module: &mut naga::Module, @@ -801,7 +721,7 @@ impl Composer { .collect(); for (h, ty) in source_ir.types.iter() { if let Some(name) = &ty.name { - let decorated_type_name = format!("{module_decoration}{name}"); + let decorated_type_name = format!("{name}{module_decoration}"); if !owned_types.contains(&decorated_type_name) { continue; } @@ -842,12 +762,11 @@ impl Composer { .constants .iter() .flat_map(|(_, c)| c.name.as_deref()) - .filter(|name| name.starts_with(module_decoration)) + .filter(|name| name.ends_with(module_decoration)) .collect(); for (h, c) in source_ir.constants.iter() { if let Some(name) = &c.name { - if name.starts_with(module_decoration) && !recompiled_consts.contains(name.as_str()) - { + if name.ends_with(module_decoration) && !recompiled_consts.contains(name.as_str()) { return Err(ComposerErrorInner::InvalidIdentifier { original: name.clone(), at: source_ir.constants.get_span(h), @@ -860,12 +779,11 @@ impl Composer { .global_variables .iter() .flat_map(|(_, c)| c.name.as_deref()) - .filter(|name| name.starts_with(module_decoration)) + .filter(|name| name.ends_with(module_decoration)) .collect(); for (h, gv) in source_ir.global_variables.iter() { if let Some(name) = &gv.name { - if name.starts_with(module_decoration) - && !recompiled_globals.contains(name.as_str()) + if name.ends_with(module_decoration) && !recompiled_globals.contains(name.as_str()) { return Err(ComposerErrorInner::InvalidIdentifier { original: name.clone(), @@ -879,11 +797,11 @@ impl Composer { .functions .iter() .flat_map(|(_, c)| c.name.as_deref()) - .filter(|name| name.starts_with(module_decoration)) + .filter(|name| name.ends_with(module_decoration)) .collect(); for (h, f) in source_ir.functions.iter() { if let Some(name) = &f.name { - if name.starts_with(module_decoration) && !recompiled_fns.contains(name.as_str()) { + if name.ends_with(module_decoration) && !recompiled_fns.contains(name.as_str()) { return Err(ComposerErrorInner::InvalidIdentifier { original: name.clone(), at: source_ir.functions.get_span(h), @@ -899,6 +817,7 @@ impl Composer { // - build the naga IR (against headers) // - record any types/vars/constants/functions that are defined within this module // - build headers for each supported language + #[allow(clippy::too_many_arguments)] fn create_composable_module( &mut self, module_definition: &ComposableModuleDefinition, @@ -906,34 +825,9 @@ impl Composer { shader_defs: &HashMap, create_headers: bool, demote_entrypoints: bool, + source: &str, + imports: Vec, ) -> Result { - let wrap_err = |inner: ComposerErrorInner| -> ComposerError { - ComposerError { - inner, - source: ErrSource::Module { - name: module_definition.name.to_owned(), - offset: 0, - defs: shader_defs.clone(), - }, - } - }; - - let PreprocessOutput { - preprocessed_source: source, - meta: PreprocessorMetaData { imports, .. }, - } = self - .preprocessor - .preprocess( - &module_definition.sanitized_source, - shader_defs, - self.validate, - ) - .map_err(wrap_err)?; - - let source = self - .substitute_shader_string(&source, &imports) - .map_err(wrap_err)?; - let mut imports: Vec<_> = imports .into_iter() .map(|import_with_offset| import_with_offset.definition) @@ -950,7 +844,7 @@ impl Composer { let mut virtual_functions: HashSet = Default::default(); let source = self .virtual_fn_regex - .replace_all(&source, |cap: ®ex::Captures| { + .replace_all(source, |cap: ®ex::Captures| { let target_function = cap.get(2).unwrap().as_str().to_owned(); let replacement_str = format!( @@ -974,18 +868,30 @@ impl Composer { let source = self.override_fn_regex .replace_all(&source, |cap: ®ex::Captures| { - let target_module = cap.get(2).unwrap().as_str().to_owned(); - let target_function = cap.get(3).unwrap().as_str().to_owned(); + let target_module = cap.get(3).unwrap().as_str().to_owned(); + let target_function = cap.get(2).unwrap().as_str().to_owned(); #[cfg(not(feature = "override_any"))] { + let wrap_err = |inner: ComposerErrorInner| -> ComposerError { + ComposerError { + inner, + source: ErrSource::Module { + name: module_definition.name.to_owned(), + offset: 0, + defs: shader_defs.clone(), + }, + } + }; + // ensure overrides are applied to virtual functions let raw_module_name = Self::decode(&target_module); let module_set = self.module_sets.get(&raw_module_name); match module_set { None => { - let pos = cap.get(2).unwrap().start(); + // TODO this should be unreachable? + let pos = cap.get(3).unwrap().start(); override_error = Some(wrap_err( ComposerErrorInner::ImportNotFound(raw_module_name, pos), )); @@ -993,7 +899,7 @@ impl Composer { Some(module_set) => { let module = module_set.get_module(shader_defs).unwrap(); if !module.virtual_functions.contains(&target_function) { - let pos = cap.get(3).unwrap().start(); + let pos = cap.get(2).unwrap().start(); override_error = Some(wrap_err(ComposerErrorInner::OverrideNotVirtual { name: target_function.clone(), @@ -1006,17 +912,17 @@ impl Composer { let base_name = format!( "{}{}{}{}", + target_function.as_str(), DECORATION_PRE, target_module.as_str(), DECORATION_POST, - target_function.as_str() ); let rename = format!( "{}{}{}{}", + target_function.as_str(), DECORATION_OVERRIDE_PRE, target_module.as_str(), DECORATION_POST, - target_function.as_str() ); let replacement_str = format!( @@ -1073,7 +979,7 @@ impl Composer { override_functions .entry(base_name.clone()) .or_default() - .push(format!("{module_decoration}{rename}")); + .push(format!("{rename}{module_decoration}")); } // rename and record owned items (except types which can't be mutably accessed) @@ -1081,7 +987,7 @@ impl Composer { for (h, c) in source_ir.constants.iter_mut() { if let Some(name) = c.name.as_mut() { if !name.contains(DECORATION_PRE) { - *name = format!("{module_decoration}{name}"); + *name = format!("{name}{module_decoration}"); owned_constants.insert(name.clone(), h); } } @@ -1091,7 +997,7 @@ impl Composer { for (h, gv) in source_ir.global_variables.iter_mut() { if let Some(name) = gv.name.as_mut() { if !name.contains(DECORATION_PRE) { - *name = format!("{module_decoration}{name}"); + *name = format!("{name}{module_decoration}"); owned_vars.insert(name.clone(), h); } @@ -1102,7 +1008,7 @@ impl Composer { for (h_f, f) in source_ir.functions.iter_mut() { if let Some(name) = f.name.as_mut() { if !name.contains(DECORATION_PRE) { - *name = format!("{module_decoration}{name}"); + *name = format!("{name}{module_decoration}"); // create dummy header function let header_function = naga::Function { @@ -1126,8 +1032,8 @@ impl Composer { for ep in &mut source_ir.entry_points { ep.function.name = Some(format!( "{}{}", + ep.function.name.as_deref().unwrap_or("main"), module_decoration, - ep.function.name.as_deref().unwrap_or("main") )); let header_function = naga::Function { name: ep.function.name.clone(), @@ -1167,7 +1073,7 @@ impl Composer { // we need to exclude autogenerated struct names, i.e. those that begin with "__" // "__" is a reserved prefix for naga so user variables cannot use it. if !name.contains(DECORATION_PRE) && !name.starts_with("__") { - let name = format!("{module_decoration}{name}"); + let name = format!("{name}{module_decoration}"); owned_types.insert(name.clone()); // copy and rename types module_builder.rename_type(&h, Some(name.clone())); @@ -1274,7 +1180,7 @@ impl Composer { let items: Option> = items.map(|items| { items .iter() - .map(|item| format!("{}{}", composable.decorated_name, item)) + .map(|item| format!("{}{}", item, composable.decorated_name)) .collect() }); let items = items.as_ref(); @@ -1319,7 +1225,11 @@ impl Composer { for (h_f, f) in source_ir.functions.iter() { if let Some(name) = &f.name { if composable.owned_functions.contains(name) - && items.map_or(true, |items| items.contains(name)) + && (items.map_or(true, |items| items.contains(name)) + || composable + .override_functions + .values() + .any(|v| v.contains(name))) { let span = composable.module_ir.functions.get_span(h_f); derived.import_function_if_new(f, span); @@ -1344,10 +1254,6 @@ impl Composer { return; } - if import.items.is_none() { - already_added.insert(import.import.clone()); - } - let import_module_set = self.module_sets.get(&import.import).unwrap(); let module = import_module_set.get_module(shader_defs).unwrap(); @@ -1358,7 +1264,7 @@ impl Composer { Self::add_composable_data( derived, module, - import.items.as_ref(), + Some(&import.items), import_module_set.module_index << SPAN_SHIFT, header, ); @@ -1369,7 +1275,10 @@ impl Composer { module_set: &ComposableModuleDefinition, shader_defs: &HashMap, ) -> Result { - let imports = self + let PreprocessOutput { + preprocessed_source, + imports, + } = self .preprocessor .preprocess(&module_set.sanitized_source, shader_defs, self.validate) .map_err(|inner| ComposerError { @@ -1379,9 +1288,7 @@ impl Composer { offset: 0, defs: shader_defs.clone(), }, - })? - .meta - .imports; + })?; self.ensure_imports(imports.iter().map(|import| &import.definition), shader_defs)?; self.ensure_imports(&module_set.additional_imports, shader_defs)?; @@ -1392,6 +1299,8 @@ impl Composer { shader_defs, true, true, + &preprocessed_source, + imports, ) } @@ -1504,13 +1413,12 @@ impl Composer { let substituted_source = self.sanitize_and_set_auto_bindings(source); - let ( - PreprocessorMetaData { - name: module_name, - mut imports, - }, - _, - ) = self + let PreprocessorMetaData { + name: module_name, + mut imports, + mut effective_defs, + .. + } = self .preprocessor .get_preprocessor_metadata(&substituted_source, false) .map_err(|inner| ComposerError { @@ -1551,7 +1459,6 @@ impl Composer { }), ); - let mut effective_defs = HashSet::new(); for import in &imports { // we require modules already added so that we can capture the shader_defs that may impact us by impacting our dependencies let module_set = self @@ -1577,9 +1484,6 @@ impl Composer { ); } - // record our explicit effective shader_defs - effective_defs.extend(self.preprocessor.effective_defs(source)); - // remove defs that are already specified through our imports effective_defs.retain(|name| !shader_defs.contains_key(name)); @@ -1644,7 +1548,12 @@ impl Composer { let sanitized_source = self.sanitize_and_set_auto_bindings(source); - let (_, defines) = self + let PreprocessorMetaData { + name, + defines, + imports, + .. + } = self .preprocessor .get_preprocessor_metadata(&sanitized_source, true) .map_err(|inner| ComposerError { @@ -1657,32 +1566,7 @@ impl Composer { })?; shader_defs.extend(defines); - let PreprocessOutput { - preprocessed_source, - meta: PreprocessorMetaData { name, imports }, - } = self - .preprocessor - .preprocess(&sanitized_source, &shader_defs, false) - .map_err(|inner| ComposerError { - inner, - source: ErrSource::Constructing { - path: file_path.to_owned(), - source: sanitized_source.to_owned(), - offset: 0, - }, - })?; - let name = name.unwrap_or_default(); - let substituted_source = self - .substitute_shader_string(&sanitized_source, &imports) - .map_err(|inner| ComposerError { - inner, - source: ErrSource::Constructing { - path: file_path.to_owned(), - source: sanitized_source.to_owned(), - offset: 0, - }, - })?; // make sure imports have been added // and gather additional defs specified at module level @@ -1727,7 +1611,7 @@ impl Composer { let definition = ComposableModuleDefinition { name, - sanitized_source: substituted_source, + sanitized_source: sanitized_source.clone(), language: shader_type.into(), file_path: file_path.to_owned(), module_index: 0, @@ -1739,13 +1623,36 @@ impl Composer { modules: Default::default(), }; + let PreprocessOutput { + preprocessed_source, + imports, + } = self + .preprocessor + .preprocess(&sanitized_source, &shader_defs, self.validate) + .map_err(|inner| ComposerError { + inner, + source: ErrSource::Constructing { + path: file_path.to_owned(), + source: sanitized_source, + offset: 0, + }, + })?; + let composable = self - .create_composable_module(&definition, String::from(""), &shader_defs, false, false) + .create_composable_module( + &definition, + String::from(""), + &shader_defs, + false, + false, + &preprocessed_source, + imports, + ) .map_err(|e| ComposerError { inner: e.inner, source: ErrSource::Constructing { path: definition.file_path.to_owned(), - source: definition.sanitized_source.to_owned(), + source: preprocessed_source.clone(), offset: e.source.offset(), }, })?; @@ -1838,7 +1745,7 @@ impl Composer { match module_index { 0 => ErrSource::Constructing { path: file_path.to_owned(), - source: definition.sanitized_source, + source: preprocessed_source.clone(), offset: composable.start_offset, }, _ => { @@ -1861,7 +1768,7 @@ impl Composer { } None => ErrSource::Constructing { path: file_path.to_owned(), - source: preprocessed_source, + source: preprocessed_source.clone(), offset: composable.start_offset, }, }; @@ -1889,15 +1796,23 @@ pub fn get_preprocessor_data( Vec, HashMap, ) { - let (PreprocessorMetaData { name, imports }, defines) = PREPROCESSOR - .get_preprocessor_metadata(source, true) - .unwrap(); - ( + if let Ok(PreprocessorMetaData { name, - imports - .into_iter() - .map(|import_with_offset| import_with_offset.definition) - .collect(), + imports, defines, - ) + .. + }) = PREPROCESSOR.get_preprocessor_metadata(source, true) + { + ( + name, + imports + .into_iter() + .map(|import_with_offset| import_with_offset.definition) + .collect(), + defines, + ) + } else { + // if errors occur we return nothing; the actual error will be displayed when the caller attempts to use the shader + Default::default() + } } diff --git a/src/compose/parse_imports.rs b/src/compose/parse_imports.rs new file mode 100644 index 0000000..50d9384 --- /dev/null +++ b/src/compose/parse_imports.rs @@ -0,0 +1,370 @@ +use std::collections::HashMap; + +use super::{ + tokenizer::{Token, Tokenizer}, + Composer, ImportDefWithOffset, ImportDefinition, +}; + +pub fn parse_imports<'a>( + input: &'a str, + declared_imports: &mut HashMap>, +) -> Result<(), (&'a str, usize)> { + let mut tokens = Tokenizer::new(input, false).peekable(); + + match tokens.next() { + Some(Token::Other('#', _)) => (), + Some(other) => return Err(("expected `#import`", other.pos())), + None => return Err(("expected #import", input.len())), + }; + match tokens.next() { + Some(Token::Identifier("import", _)) => (), + Some(other) => return Err(("expected `#import`", other.pos())), + None => return Err(("expected `#import`", input.len())), + }; + + let mut stack = Vec::default(); + let mut current = String::default(); + let mut as_name = None; + let mut is_deprecated_itemlist = false; + + loop { + match tokens.peek() { + Some(Token::Identifier(ident, _)) => { + current.push_str(ident); + tokens.next(); + + if tokens.peek().and_then(Token::identifier) == Some("as") { + let pos = tokens.next().unwrap().pos(); + let Some(Token::Identifier(name, _)) = tokens.next() else { + return Err(("expected identifier after `as`", pos)); + }; + + as_name = Some(name); + } + + // support deprecated #import mod item + if let Some(Token::Identifier(..)) = tokens.peek() { + #[cfg(not(feature = "allow_deprecated"))] + tracing::warn!("item list imports are deprecated, please use `rust::style::item_imports` (or use feature `allow_deprecated`)`\n| {}", input); + + is_deprecated_itemlist = true; + stack.push(format!("{}::", current)); + current = String::default(); + as_name = None; + } + + continue; + } + Some(Token::Other('{', pos)) => { + if !current.ends_with("::") { + return Err(("open brace must follow `::`", *pos)); + } + stack.push(current); + current = String::default(); + as_name = None; + } + Some(Token::Other(',', _)) + | Some(Token::Other('}', _)) + | Some(Token::Other('\n', _)) + | None => { + if !current.is_empty() { + let used_name = as_name.map(ToString::to_string).unwrap_or_else(|| { + current + .rsplit_once("::") + .map(|(_, name)| name.to_owned()) + .unwrap_or(current.clone()) + }); + declared_imports.entry(used_name).or_default().push(format!( + "{}{}", + stack.join(""), + current + )); + current = String::default(); + as_name = None; + } + + if let Some(Token::Other('}', pos)) = tokens.peek() { + if stack.pop().is_none() { + return Err(("close brace without open", *pos)); + } + } + + if tokens.peek().is_none() { + break; + } + } + Some(Token::Other(';', _)) => { + tokens.next(); + if let Some(token) = tokens.peek() { + return Err(("unexpected token after ';'", token.pos())); + } + } + Some(Token::Other(_, pos)) => return Err(("unexpected token", *pos)), + Some(Token::Whitespace(..)) => unreachable!(), + } + + tokens.next(); + } + + if !(stack.is_empty() || is_deprecated_itemlist && stack.len() == 1) { + return Err(("missing close brace", input.len())); + } + + Ok(()) +} + +pub fn substitute_identifiers( + input: &str, + offset: usize, + declared_imports: &HashMap>, + used_imports: &mut HashMap, + allow_ambiguous: bool, +) -> Result { + let tokens = Tokenizer::new(input, true); + let mut output = String::with_capacity(input.len()); + let mut in_substitution_position = true; + + for token in tokens { + match token { + Token::Identifier(ident, token_pos) => { + if in_substitution_position { + let (first, residual) = ident.split_once("::").unwrap_or((ident, "")); + let full_paths = declared_imports + .get(first) + .cloned() + .unwrap_or(vec![first.to_owned()]); + + if !allow_ambiguous && full_paths.len() > 1 { + return Err(offset + token_pos); + } + + for mut full_path in full_paths { + if !residual.is_empty() { + full_path.push_str("::"); + full_path.push_str(residual); + } + + if let Some((module, item)) = full_path.rsplit_once("::") { + used_imports + .entry(module.to_owned()) + .or_insert_with(|| ImportDefWithOffset { + definition: ImportDefinition { + import: module.to_owned(), + ..Default::default() + }, + offset: offset + token_pos, + }) + .definition + .items + .push(item.to_owned()); + output.push_str(item); + output.push_str(&Composer::decorate(module)); + } else if full_path.find('"').is_some() { + // we don't want to replace local variables that shadow quoted module imports with the + // quoted name as that won't compile. + // since quoted items always refer to modules, we can just emit the original ident + // in this case + output.push_str(ident); + } else { + // if there are no quotes we do the replacement. this means that individually imported + // items can be used, and any shadowing local variables get harmlessly renamed. + // TODO: it can lead to weird errors, but such is life + output.push_str(&full_path); + } + } + } else { + output.push_str(ident); + } + } + Token::Other(other, _) => { + output.push(other); + if other == '.' || other == '@' { + in_substitution_position = false; + continue; + } + } + Token::Whitespace(ws, _) => output.push_str(ws), + } + + in_substitution_position = true; + } + + Ok(output) +} + +#[cfg(test)] +fn test_parse(input: &str) -> Result>, (&str, usize)> { + let mut declared_imports = HashMap::default(); + parse_imports(input, &mut declared_imports)?; + Ok(declared_imports) +} + +#[test] +fn import_tokens() { + let input = r" + #import a::b + "; + assert_eq!( + test_parse(input), + Ok(HashMap::from_iter([( + "b".to_owned(), + vec!("a::b".to_owned()) + )])) + ); + + let input = r" + #import a::{b, c} + "; + assert_eq!( + test_parse(input), + Ok(HashMap::from_iter([ + ("b".to_owned(), vec!("a::b".to_owned())), + ("c".to_owned(), vec!("a::c".to_owned())), + ])) + ); + + let input = r" + #import a::{b as d, c} + "; + assert_eq!( + test_parse(input), + Ok(HashMap::from_iter([ + ("d".to_owned(), vec!("a::b".to_owned())), + ("c".to_owned(), vec!("a::c".to_owned())), + ])) + ); + + let input = r" + #import a::{b::{c, d}, e} + "; + assert_eq!( + test_parse(input), + Ok(HashMap::from_iter([ + ("c".to_owned(), vec!("a::b::c".to_owned())), + ("d".to_owned(), vec!("a::b::d".to_owned())), + ("e".to_owned(), vec!("a::e".to_owned())), + ])) + ); + + let input = r" + #import a::b::{c, d}, e + "; + assert_eq!( + test_parse(input), + Ok(HashMap::from_iter([ + ("c".to_owned(), vec!("a::b::c".to_owned())), + ("d".to_owned(), vec!("a::b::d".to_owned())), + ("e".to_owned(), vec!("e".to_owned())), + ])) + ); + + let input = r" + #import a, b + "; + assert_eq!( + test_parse(input), + Ok(HashMap::from_iter([ + ("a".to_owned(), vec!("a".to_owned())), + ("b".to_owned(), vec!("b".to_owned())), + ])) + ); + + let input = r" + #import a::b c, d + "; + assert_eq!( + test_parse(input), + Ok(HashMap::from_iter([ + ("c".to_owned(), vec!("a::b::c".to_owned())), + ("d".to_owned(), vec!("a::b::d".to_owned())), + ])) + ); + + let input = r" + #import a::b c + "; + assert_eq!( + test_parse(input), + Ok(HashMap::from_iter([( + "c".to_owned(), + vec!("a::b::c".to_owned()) + ),])) + ); + + let input = r" + #import a::b::{c::{d, e}, f, g::{h as i, j}} + "; + assert_eq!( + test_parse(input), + Ok(HashMap::from_iter([ + ("d".to_owned(), vec!("a::b::c::d".to_owned())), + ("e".to_owned(), vec!("a::b::c::e".to_owned())), + ("f".to_owned(), vec!("a::b::f".to_owned())), + ("i".to_owned(), vec!("a::b::g::h".to_owned())), + ("j".to_owned(), vec!("a::b::g::j".to_owned())), + ])) + ); + + let input = r" + #import a::b::{ + c::{d, e}, + f, + g::{ + h as i, + j::k::l as m, + } + } + "; + assert_eq!( + test_parse(input), + Ok(HashMap::from_iter([ + ("d".to_owned(), vec!("a::b::c::d".to_owned())), + ("e".to_owned(), vec!("a::b::c::e".to_owned())), + ("f".to_owned(), vec!("a::b::f".to_owned())), + ("i".to_owned(), vec!("a::b::g::h".to_owned())), + ("m".to_owned(), vec!("a::b::g::j::k::l".to_owned())), + ])) + ); + + let input = r#" + #import "path//with\ all sorts of .stuff"::{a, b} + "#; + assert_eq!( + test_parse(input), + Ok(HashMap::from_iter([ + ( + "a".to_owned(), + vec!(r#""path//with\ all sorts of .stuff"::a"#.to_owned()) + ), + ( + "b".to_owned(), + vec!(r#""path//with\ all sorts of .stuff"::b"#.to_owned()) + ), + ])) + ); + + let input = r" + #import a::b::{ + "; + assert!(test_parse(input).is_err()); + + let input = r" + #import a::b::{{c} + "; + assert!(test_parse(input).is_err()); + + let input = r" + #import a::b::{c}} + "; + assert!(test_parse(input).is_err()); + + let input = r" + #import a::b{{c,d}} + "; + assert!(test_parse(input).is_err()); + + let input = r" + #import a:b + "; + assert!(test_parse(input).is_err()); +} diff --git a/src/compose/preprocess.rs b/src/compose/preprocess.rs index c1e6c32..e805b05 100644 --- a/src/compose/preprocess.rs +++ b/src/compose/preprocess.rs @@ -1,11 +1,11 @@ use std::collections::{HashMap, HashSet}; use regex::Regex; -use tracing::warn; use super::{ - comment_strip_iter::CommentReplaceExt, ComposerErrorInner, ImportDefWithOffset, - ImportDefinition, ShaderDefValue, + comment_strip_iter::CommentReplaceExt, + parse_imports::{parse_imports, substitute_identifiers}, + ComposerErrorInner, ImportDefWithOffset, ShaderDefValue, }; #[derive(Debug)] @@ -18,10 +18,7 @@ pub struct Preprocessor { endif_regex: Regex, def_regex: Regex, def_regex_delimited: Regex, - import_custom_path_as_regex: Regex, - import_custom_path_regex: Regex, - import_items_regex: Regex, - identifier_regex: Regex, + import_regex: Regex, define_import_path_regex: Regex, define_shader_def_regex: Regex, } @@ -40,14 +37,7 @@ impl Default for Preprocessor { endif_regex: Regex::new(r"^\s*#\s*endif").unwrap(), def_regex: Regex::new(r"#\s*([\w|\d|_]+)").unwrap(), def_regex_delimited: Regex::new(r"#\s*\{([\w|\d|_]+)\}").unwrap(), - import_custom_path_as_regex: Regex::new(r"^\s*#\s*import\s+([^\s]+)\s+as\s+([^\s]+)") - .unwrap(), - import_custom_path_regex: Regex::new(r"^\s*#\s*import\s+([^\s]+)").unwrap(), - import_items_regex: Regex::new( - r"^\s*#\s*import\s+([^\s]+)\s+((?:[\w|\d|_]+)(?:\s*,\s*[\w|\d|_]+)*)", - ) - .unwrap(), - identifier_regex: Regex::new(r"([\w|\d|_]+)").unwrap(), + import_regex: Regex::new(r"^\s*#\s*import\s").unwrap(), define_import_path_regex: Regex::new(r"^\s*#\s*define_import_path\s+([^\s]+)").unwrap(), define_shader_def_regex: Regex::new(r"^\s*#\s*define\s+([\w|\d|_]+)\s*([-\w|\d]+)?") .unwrap(), @@ -59,6 +49,8 @@ impl Default for Preprocessor { pub struct PreprocessorMetaData { pub name: Option, pub imports: Vec, + pub defines: HashMap, + pub effective_defs: HashSet, } enum ScopeLevel { @@ -136,228 +128,233 @@ impl Scope { #[derive(Debug)] pub struct PreprocessOutput { pub preprocessed_source: String, - pub meta: PreprocessorMetaData, + pub imports: Vec, } impl Preprocessor { + fn check_scope<'a>( + &self, + shader_defs: &HashMap, + line: &'a str, + scope: Option<&mut Scope>, + offset: usize, + ) -> Result<(bool, Option<&'a str>), ComposerErrorInner> { + if let Some(cap) = self.ifdef_regex.captures(line) { + let is_else = cap.get(1).is_some(); + let def = cap.get(2).unwrap().as_str(); + let cond = shader_defs.contains_key(def); + scope.map_or(Ok(()), |scope| scope.branch(is_else, cond, offset))?; + return Ok((true, Some(def))); + } else if let Some(cap) = self.ifndef_regex.captures(line) { + let is_else = cap.get(1).is_some(); + let def = cap.get(2).unwrap().as_str(); + let cond = !shader_defs.contains_key(def); + scope.map_or(Ok(()), |scope| scope.branch(is_else, cond, offset))?; + return Ok((true, Some(def))); + } else if let Some(cap) = self.ifop_regex.captures(line) { + let is_else = cap.get(1).is_some(); + let def = cap.get(2).unwrap().as_str(); + let op = cap.get(3).unwrap(); + let val = cap.get(4).unwrap(); + + if scope.is_none() { + // don't try to evaluate if we don't have a scope + return Ok((true, Some(def))); + } + + fn act_on( + a: T, + b: T, + op: &str, + pos: usize, + ) -> Result { + match op { + "==" => Ok(a == b), + "!=" => Ok(a != b), + ">" => Ok(a > b), + ">=" => Ok(a >= b), + "<" => Ok(a < b), + "<=" => Ok(a <= b), + _ => Err(ComposerErrorInner::UnknownShaderDefOperator { + pos, + operator: op.to_string(), + }), + } + } + + let def_value = shader_defs + .get(def) + .ok_or(ComposerErrorInner::UnknownShaderDef { + pos: offset, + shader_def_name: def.to_string(), + })?; + + let invalid_def = |ty: &str| ComposerErrorInner::InvalidShaderDefComparisonValue { + pos: offset, + shader_def_name: def.to_string(), + value: val.as_str().to_string(), + expected: ty.to_string(), + }; + + let new_scope = match def_value { + ShaderDefValue::Bool(def_value) => { + let val = val.as_str().parse().map_err(|_| invalid_def("bool"))?; + act_on(*def_value, val, op.as_str(), offset)? + } + ShaderDefValue::Int(def_value) => { + let val = val.as_str().parse().map_err(|_| invalid_def("int"))?; + act_on(*def_value, val, op.as_str(), offset)? + } + ShaderDefValue::UInt(def_value) => { + let val = val.as_str().parse().map_err(|_| invalid_def("uint"))?; + act_on(*def_value, val, op.as_str(), offset)? + } + }; + + scope.map_or(Ok(()), |scope| scope.branch(is_else, new_scope, offset))?; + return Ok((true, Some(def))); + } else if self.else_regex.is_match(line) { + scope.map_or(Ok(()), |scope| scope.branch(true, true, offset))?; + return Ok((true, None)); + } else if self.endif_regex.is_match(line) { + scope.map_or(Ok(()), |scope| scope.pop(offset))?; + return Ok((true, None)); + } + + Ok((false, None)) + } + // process #if[(n)?def]? / #else / #endif preprocessor directives, // strip module name and imports // also strip "#version xxx" + // replace items with resolved decorated names pub fn preprocess( &self, shader_str: &str, shader_defs: &HashMap, - mut validate_len: bool, + validate_len: bool, ) -> Result { - let mut imports = Vec::new(); - + let mut declared_imports = HashMap::new(); + let mut used_imports = HashMap::new(); let mut scope = Scope::new(); let mut final_string = String::new(); - let mut name = None; let mut offset = 0; - let mut at_start = true; - #[cfg(debug)] let len = shader_str.len(); // this code broadly stolen from bevy_render::ShaderProcessor - for (line, original_line) in shader_str - .lines() - .replace_comments() - .zip(shader_str.lines()) - { - let line = &line; + let mut lines = shader_str.lines(); + let mut lines = lines.replace_comments().zip(shader_str.lines()).peekable(); + while let Some((mut line, original_line)) = lines.next() { let mut output = false; - let mut still_at_start = false; - if line.is_empty() || line.chars().all(|c| c.is_ascii_whitespace()) { - still_at_start = true; - } - if let Some(cap) = self.version_regex.captures(line) { + if let Some(cap) = self.version_regex.captures(&line) { let v = cap.get(1).unwrap().as_str(); if v != "440" && v != "450" { return Err(ComposerErrorInner::GlslInvalidVersion(offset)); } - } else if let Some(cap) = self.ifdef_regex.captures(line) { - let is_else = cap.get(1).is_some(); - let cond = shader_defs.contains_key(cap.get(2).unwrap().as_str()); - scope.branch(is_else, cond, offset)?; - } else if let Some(cap) = self.ifndef_regex.captures(line) { - let is_else = cap.get(1).is_some(); - let cond = !shader_defs.contains_key(cap.get(2).unwrap().as_str()); - scope.branch(is_else, cond, offset)?; - } else if let Some(cap) = self.ifop_regex.captures(line) { - let is_else = cap.get(1).is_some(); - let def = cap.get(2).unwrap(); - let op = cap.get(3).unwrap(); - let val = cap.get(4).unwrap(); - - fn act_on( - a: T, - b: T, - op: &str, - pos: usize, - ) -> Result { - match op { - "==" => Ok(a == b), - "!=" => Ok(a != b), - ">" => Ok(a > b), - ">=" => Ok(a >= b), - "<" => Ok(a < b), - "<=" => Ok(a <= b), - _ => Err(ComposerErrorInner::UnknownShaderDefOperator { - pos, - operator: op.to_string(), - }), - } - } + } else if self + .check_scope(shader_defs, &line, Some(&mut scope), offset)? + .0 + || self.define_import_path_regex.captures(&line).is_some() + || self.define_shader_def_regex.captures(&line).is_some() + { + // ignore + } else if scope.active() { + if self.import_regex.is_match(&line) { + let mut import_lines = String::default(); + let mut open_count = 0; + let initial_offset = offset; + + loop { + // output spaces for removed lines to keep spans consistent (errors report against substituted_source, which is not preprocessed) + final_string.extend(std::iter::repeat(" ").take(line.len())); + offset += line.len() + 1; + + // PERF: Ideally we don't do multiple `match_indices` passes over `line` + // in addition to the final pass for the import parse + open_count += line.match_indices('{').count(); + open_count = open_count.saturating_sub(line.match_indices('}').count()); + + // PERF: it's bad that we allocate here. ideally we would use something like + // let import_lines = &shader_str[initial_offset..offset] + // but we need the comments removed, and the iterator approach doesn't make that easy + import_lines.push_str(&line); + import_lines.push('\n'); + + if open_count == 0 || lines.peek().is_none() { + break; + } - let def_value = - shader_defs - .get(def.as_str()) - .ok_or(ComposerErrorInner::UnknownShaderDef { - pos: offset, - shader_def_name: def.as_str().to_string(), - })?; - let new_scope = match def_value { - ShaderDefValue::Bool(def_value) => { - let val = val.as_str().parse().map_err(|_| { - ComposerErrorInner::InvalidShaderDefComparisonValue { - pos: offset, - shader_def_name: def.as_str().to_string(), - value: val.as_str().to_string(), - expected: "bool".to_string(), - } - })?; - act_on(*def_value, val, op.as_str(), offset)? + line = lines.next().unwrap().0; } - ShaderDefValue::Int(def_value) => { - let val = val.as_str().parse().map_err(|_| { - ComposerErrorInner::InvalidShaderDefComparisonValue { - pos: offset, - shader_def_name: def.as_str().to_string(), - value: val.as_str().to_string(), - expected: "int".to_string(), + + parse_imports(import_lines.as_str(), &mut declared_imports).map_err( + |(err, line_offset)| { + ComposerErrorInner::ImportParseError( + err.to_owned(), + initial_offset + line_offset, + ) + }, + )?; + output = true; + } else { + let replaced_lines = [original_line, &line].map(|input| { + let mut output = input.to_string(); + for capture in self.def_regex.captures_iter(input) { + let def = capture.get(1).unwrap(); + if let Some(def) = shader_defs.get(def.as_str()) { + output = self + .def_regex + .replace(&output, def.value_as_string()) + .to_string(); } - })?; - act_on(*def_value, val, op.as_str(), offset)? - } - ShaderDefValue::UInt(def_value) => { - let val = val.as_str().parse().map_err(|_| { - ComposerErrorInner::InvalidShaderDefComparisonValue { - pos: offset, - shader_def_name: def.as_str().to_string(), - value: val.as_str().to_string(), - expected: "int".to_string(), + } + for capture in self.def_regex_delimited.captures_iter(input) { + let def = capture.get(1).unwrap(); + if let Some(def) = shader_defs.get(def.as_str()) { + output = self + .def_regex_delimited + .replace(&output, def.value_as_string()) + .to_string(); } - })?; - act_on(*def_value, val, op.as_str(), offset)? - } - }; - - scope.branch(is_else, new_scope, offset)?; - } else if self.else_regex.is_match(line) { - scope.branch(true, true, offset)?; - } else if self.endif_regex.is_match(line) { - scope.pop(offset)?; - } else if let Some(cap) = self.define_import_path_regex.captures(line) { - name = Some(cap.get(1).unwrap().as_str().to_string()); - } else if let Some(cap) = self.define_shader_def_regex.captures(line) { - if at_start { - still_at_start = true; - - let def = cap.get(1).unwrap(); - let name = def.as_str().to_string(); - - let value = if let Some(val) = cap.get(2) { - if let Ok(val) = val.as_str().parse::() { - ShaderDefValue::UInt(val) - } else if let Ok(val) = val.as_str().parse::() { - ShaderDefValue::Int(val) - } else if let Ok(val) = val.as_str().parse::() { - ShaderDefValue::Bool(val) - } else { - return Err(ComposerErrorInner::InvalidShaderDefDefinitionValue { - name, - value: val.as_str().to_string(), - pos: offset, - }); } - } else { - ShaderDefValue::Bool(true) - }; - - match shader_defs.get(name.as_str()) { - Some(current_value) if *current_value == value => (), - _ => return Err(ComposerErrorInner::DefineInModule(offset)), - } - } else { - return Err(ComposerErrorInner::DefineInModule(offset)); - } - } else if scope.active() { - if let Some(cap) = self.import_custom_path_as_regex.captures(line) { - imports.push(ImportDefWithOffset { - definition: ImportDefinition { - import: cap.get(1).unwrap().as_str().to_string(), - as_name: Some(cap.get(2).unwrap().as_str().to_string()), - items: Default::default(), - }, - offset, + output }); - } else if let Some(cap) = self.import_items_regex.captures(line) { - imports.push(ImportDefWithOffset { - definition: ImportDefinition { - import: cap.get(1).unwrap().as_str().to_string(), - as_name: None, - items: Some( - self.identifier_regex - .captures_iter(cap.get(2).unwrap().as_str()) - .map(|ident_cap| ident_cap.get(1).unwrap().as_str().to_owned()) - .collect(), - ), - }, + + let original_line = &replaced_lines[0]; + let decommented_line = &replaced_lines[1]; + + // we don't want to capture imports from comments so we run using a dummy used_imports, and disregard any errors + let item_replaced_line = substitute_identifiers( + original_line, offset, - }); - } else if let Some(cap) = self.import_custom_path_regex.captures(line) { - imports.push(ImportDefWithOffset { - definition: ImportDefinition { - import: cap.get(1).unwrap().as_str().to_string(), - as_name: None, - items: Default::default(), - }, + &declared_imports, + &mut Default::default(), + true, + ) + .unwrap(); + // we also run against the de-commented line to replace real imports, and throw an error if appropriate + let _ = substitute_identifiers( + decommented_line, offset, - }); - } else { - let mut line_with_defs = original_line.to_string(); - for capture in self.def_regex.captures_iter(line) { - let def = capture.get(1).unwrap(); - if let Some(def) = shader_defs.get(def.as_str()) { - line_with_defs = self - .def_regex - .replace(&line_with_defs, def.value_as_string()) - .to_string(); - } - } - for capture in self.def_regex_delimited.captures_iter(line) { - let def = capture.get(1).unwrap(); - if let Some(def) = shader_defs.get(def.as_str()) { - line_with_defs = self - .def_regex_delimited - .replace(&line_with_defs, def.value_as_string()) - .to_string(); - } - } - final_string.push_str(&line_with_defs); - let diff = line.len() as i32 - line_with_defs.len() as i32; - if diff > 0 { - final_string.extend(std::iter::repeat(" ").take(diff as usize)); - } else if diff < 0 && validate_len { - // this sucks - warn!("source code map requires shader_def values to be no longer than the corresponding shader_def name, error reporting may not be correct:\noriginal: {}\nreplaced: {}", line, line_with_defs); - validate_len = false; - } + &declared_imports, + &mut used_imports, + false, + ) + .map_err(|pos| { + ComposerErrorInner::ImportParseError( + "Ambiguous import path for item".to_owned(), + pos, + ) + })?; + + final_string.push_str(&item_replaced_line); + let diff = line.len().saturating_sub(item_replaced_line.len()); + final_string.extend(std::iter::repeat(" ").take(diff)); + offset += original_line.len() + 1; output = true; } } @@ -365,11 +362,9 @@ impl Preprocessor { if !output { // output spaces for removed lines to keep spans consistent (errors report against substituted_source, which is not preprocessed) final_string.extend(std::iter::repeat(" ").take(line.len())); + offset += line.len() + 1; } final_string.push('\n'); - offset += line.len() + 1; - - at_start &= still_at_start; } scope.finish(offset)?; @@ -379,10 +374,12 @@ impl Preprocessor { let revised_len = final_string.len(); assert_eq!(len, revised_len); } + #[cfg(not(debug))] + let _ = validate_len; Ok(PreprocessOutput { preprocessed_source: final_string, - meta: PreprocessorMetaData { name, imports }, + imports: used_imports.into_values().collect(), }) } @@ -391,49 +388,62 @@ impl Preprocessor { &self, shader_str: &str, allow_defines: bool, - ) -> Result<(PreprocessorMetaData, HashMap), ComposerErrorInner> { - let mut imports = Vec::new(); + ) -> Result { + let mut declared_imports = HashMap::default(); + let mut used_imports = HashMap::default(); let mut name = None; let mut offset = 0; let mut defines = HashMap::default(); + let mut effective_defs = HashSet::default(); - for line in shader_str.lines().replace_comments() { - let line = &line; - if let Some(cap) = self.import_custom_path_as_regex.captures(line) { - imports.push(ImportDefWithOffset { - definition: ImportDefinition { - import: cap.get(1).unwrap().as_str().to_string(), - as_name: Some(cap.get(2).unwrap().as_str().to_string()), - items: Default::default(), - }, - offset, - }); - } else if let Some(cap) = self.import_items_regex.captures(line) { - imports.push(ImportDefWithOffset { - definition: ImportDefinition { - import: cap.get(1).unwrap().as_str().to_string(), - as_name: None, - items: Some( - self.identifier_regex - .captures_iter(cap.get(2).unwrap().as_str()) - .map(|ident_cap| ident_cap.get(1).unwrap().as_str().to_owned()) - .collect(), - ), - }, - offset, - }); - } else if let Some(cap) = self.import_custom_path_regex.captures(line) { - imports.push(ImportDefWithOffset { - definition: ImportDefinition { - import: cap.get(1).unwrap().as_str().to_string(), - as_name: None, - items: Default::default(), + let mut lines = shader_str.lines(); + let mut lines = lines.replace_comments().peekable(); + + while let Some(mut line) = lines.next() { + let (is_scope, def) = self.check_scope(&HashMap::default(), &line, None, offset)?; + + if is_scope { + if let Some(def) = def { + effective_defs.insert(def.to_owned()); + } + } else if self.import_regex.is_match(&line) { + let mut import_lines = String::default(); + let mut open_count = 0; + let initial_offset = offset; + + loop { + // PERF: Ideally we don't do multiple `match_indices` passes over `line` + // in addition to the final pass for the import parse + open_count += line.match_indices('{').count(); + open_count = open_count.saturating_sub(line.match_indices('}').count()); + + // PERF: it's bad that we allocate here. ideally we would use something like + // let import_lines = &shader_str[initial_offset..offset] + // but we need the comments removed, and the iterator approach doesn't make that easy + import_lines.push_str(&line); + import_lines.push('\n'); + + if open_count == 0 || lines.peek().is_none() { + break; + } + + // output spaces for removed lines to keep spans consistent (errors report against substituted_source, which is not preprocessed) + offset += line.len() + 1; + + line = lines.next().unwrap(); + } + + parse_imports(import_lines.as_str(), &mut declared_imports).map_err( + |(err, line_offset)| { + ComposerErrorInner::ImportParseError( + err.to_owned(), + initial_offset + line_offset, + ) }, - offset, - }); - } else if let Some(cap) = self.define_import_path_regex.captures(line) { + )?; + } else if let Some(cap) = self.define_import_path_regex.captures(&line) { name = Some(cap.get(1).unwrap().as_str().to_string()); - } else if let Some(cap) = self.define_shader_def_regex.captures(line) { + } else if let Some(cap) = self.define_shader_def_regex.captures(&line) { if allow_defines { let def = cap.get(1).unwrap(); let name = def.as_str().to_string(); @@ -456,33 +466,20 @@ impl Preprocessor { } else { return Err(ComposerErrorInner::DefineInModule(offset)); } + } else { + substitute_identifiers(&line, offset, &declared_imports, &mut used_imports, true) + .unwrap(); } offset += line.len() + 1; } - Ok((PreprocessorMetaData { name, imports }, defines)) - } - - pub fn effective_defs(&self, source: &str) -> HashSet { - let mut effective_defs = HashSet::default(); - - for line in source.lines().replace_comments() { - if let Some(cap) = self.ifdef_regex.captures(&line) { - let def = cap.get(2).unwrap(); - effective_defs.insert(def.as_str().to_owned()); - } - if let Some(cap) = self.ifndef_regex.captures(&line) { - let def = cap.get(2).unwrap(); - effective_defs.insert(def.as_str().to_owned()); - } - if let Some(cap) = self.ifop_regex.captures(&line) { - let def = cap.get(2).unwrap(); - effective_defs.insert(def.as_str().to_owned()); - } - } - - effective_defs + Ok(PreprocessorMetaData { + name, + imports: used_imports.into_values().collect(), + defines, + effective_defs, + }) } } @@ -1048,9 +1045,12 @@ defined "; let processor = Preprocessor::default(); - let (_, defines) = processor.get_preprocessor_metadata(&WGSL, true).unwrap(); - println!("defines: {:?}", defines); - let result = processor.preprocess(&WGSL, &defines, true).unwrap(); + let PreprocessorMetaData { + defines: shader_defs, + .. + } = processor.get_preprocessor_metadata(&WGSL, true).unwrap(); + println!("defines: {:?}", shader_defs); + let result = processor.preprocess(&WGSL, &shader_defs, true).unwrap(); assert_eq!(result.preprocessed_source, EXPECTED); } @@ -1088,9 +1088,12 @@ bool: false "; let processor = Preprocessor::default(); - let (_, defines) = processor.get_preprocessor_metadata(&WGSL, true).unwrap(); - println!("defines: {:?}", defines); - let result = processor.preprocess(&WGSL, &defines, true).unwrap(); + let PreprocessorMetaData { + defines: shader_defs, + .. + } = processor.get_preprocessor_metadata(&WGSL, true).unwrap(); + println!("defines: {:?}", shader_defs); + let result = processor.preprocess(&WGSL, &shader_defs, true).unwrap(); assert_eq!(result.preprocessed_source, EXPECTED); } diff --git a/src/compose/test.rs b/src/compose/test.rs index bed0462..a90102e 100644 --- a/src/compose/test.rs +++ b/src/compose/test.rs @@ -12,7 +12,7 @@ mod test { }; use crate::compose::{ - ComposableModuleDescriptor, Composer, ComposerErrorInner, ImportDefinition, + get_preprocessor_data, ComposableModuleDescriptor, Composer, ImportDefinition, NagaModuleDescriptor, ShaderDefValue, ShaderLanguage, ShaderType, }; @@ -156,6 +156,9 @@ mod test { naga::back::wgsl::WriterFlags::EXPLICIT_TYPES, ) .unwrap(); + let mut wgsl: Vec<_> = wgsl.lines().collect(); + wgsl.sort(); + let wgsl = wgsl.join("\n"); // println!("{}", wgsl); // let mut f = std::fs::File::create("dup_import.txt").unwrap(); @@ -510,7 +513,18 @@ mod test { }) .unwrap(); - assert_eq!(test_shader(&mut composer), 3.0); + // this test doesn't work any more. + // overrides only work if the composer realises the module is required. + // not we can't just blindly import any `#import`ed items because that would break: + // #import a::b + // a::b::c::d(); + // the path would be interpreted as a module when it may actually + // be only a fragment of a path to a module. + // so either i need to add another directive (#import_overrides) + // or we just limit overrides to modules included via the additional_modules + // in `Composer::make_naga_module` and `Composer::add_composable_module` + + // assert_eq!(test_shader(&mut composer), 3.0); } #[cfg(feature = "test_shader")] @@ -848,23 +862,6 @@ mod test { fn bad_identifiers() { let mut composer = Composer::default(); - let check_err = |composer: &mut Composer, name: &str| -> bool { - let result = composer.make_naga_module(NagaModuleDescriptor { - source: &format!("#import {name}"), - file_path: name, - ..Default::default() - }); - - if let Err(err) = &result { - if let ComposerErrorInner::InvalidIdentifier { original, .. } = &err.inner { - return original.ends_with("bad_"); - } - } - - println!("{result:?}"); - false - }; - composer .add_composable_module(ComposableModuleDescriptor { source: include_str!("tests/invalid_identifiers/const.wgsl"), @@ -872,8 +869,6 @@ mod test { ..Default::default() }) .unwrap(); - assert!(check_err(&mut composer, "consts")); - composer .add_composable_module(ComposableModuleDescriptor { source: include_str!("tests/invalid_identifiers/fn.wgsl"), @@ -881,8 +876,6 @@ mod test { ..Default::default() }) .unwrap(); - assert!(check_err(&mut composer, "fns")); - composer .add_composable_module(ComposableModuleDescriptor { source: include_str!("tests/invalid_identifiers/global.wgsl"), @@ -890,8 +883,6 @@ mod test { ..Default::default() }) .unwrap(); - assert!(check_err(&mut composer, "globals")); - composer .add_composable_module(ComposableModuleDescriptor { source: include_str!("tests/invalid_identifiers/struct_member.wgsl"), @@ -899,8 +890,6 @@ mod test { ..Default::default() }) .unwrap(); - assert!(check_err(&mut composer, "struct_members")); - composer .add_composable_module(ComposableModuleDescriptor { source: include_str!("tests/invalid_identifiers/struct.wgsl"), @@ -908,7 +897,44 @@ mod test { ..Default::default() }) .unwrap(); - assert!(check_err(&mut composer, "structs")); + let module = composer + .make_naga_module(NagaModuleDescriptor { + source: include_str!("tests/invalid_identifiers/top_valid.wgsl"), + file_path: "tests/invalid_identifiers/top_valid.wgsl", + ..Default::default() + }) + .unwrap(); + + let info = naga::valid::Validator::new( + naga::valid::ValidationFlags::all(), + naga::valid::Capabilities::default(), + ) + .validate(&module) + .unwrap(); + let wgsl = naga::back::wgsl::write_string( + &module, + &info, + naga::back::wgsl::WriterFlags::EXPLICIT_TYPES, + ) + .unwrap(); + let mut wgsl: Vec<_> = wgsl.lines().collect(); + wgsl.sort(); + let wgsl = wgsl.join("\n"); + + // let mut f = std::fs::File::create("bad_identifiers.txt").unwrap(); + // f.write_all(wgsl.as_bytes()).unwrap(); + // drop(f); + + output_eq!(wgsl, "tests/expected/bad_identifiers.txt"); + + composer + .make_naga_module(NagaModuleDescriptor { + source: include_str!("tests/invalid_identifiers/top_invalid.wgsl"), + file_path: "tests/invalid_identifiers/top_invalid.wgsl", + ..Default::default() + }) + .err() + .unwrap(); } #[test] @@ -960,6 +986,9 @@ mod test { naga::back::wgsl::WriterFlags::EXPLICIT_TYPES, ) .unwrap(); + let mut wgsl: Vec<_> = wgsl.lines().collect(); + wgsl.sort(); + let wgsl = wgsl.join("\n"); // let mut f = std::fs::File::create("dup_struct_import.txt").unwrap(); // f.write_all(wgsl.as_bytes()).unwrap(); @@ -1083,6 +1112,133 @@ mod test { output_eq!(wgsl, "tests/expected/conditional_import_b.txt"); } + #[cfg(feature = "test_shader")] + #[test] + fn rusty_imports() { + let mut composer = Composer::default(); + + composer + .add_composable_module(ComposableModuleDescriptor { + source: include_str!("tests/rusty_imports/mod_a_b_c.wgsl"), + file_path: "tests/rusty_imports/mod_a_b_c.wgsl", + ..Default::default() + }) + .unwrap(); + + composer + .add_composable_module(ComposableModuleDescriptor { + source: include_str!("tests/rusty_imports/mod_a_x.wgsl"), + file_path: "tests/rusty_imports/mod_a_x.wgsl", + ..Default::default() + }) + .unwrap(); + + composer + .add_composable_module(ComposableModuleDescriptor { + source: include_str!("tests/rusty_imports/top.wgsl"), + file_path: "tests/rusty_imports/top.wgsl", + ..Default::default() + }) + .unwrap(); + + assert_eq!(test_shader(&mut composer), 36.0); + } + + #[test] + fn test_bevy_path_imports() { + let (_, mut imports, _) = + get_preprocessor_data(include_str!("tests/bevy_path_imports/skill.wgsl")); + imports.iter_mut().for_each(|import| { + import.items.sort(); + }); + imports.sort_by(|a, b| a.import.cmp(&b.import)); + assert_eq!( + imports, + vec![ + ImportDefinition { + import: "\"shaders/skills/hit.wgsl\"".to_owned(), + items: vec!["frag".to_owned(), "vert".to_owned(),], + }, + ImportDefinition { + import: "\"shaders/skills/lightning.wgsl\"".to_owned(), + items: vec!["frag".to_owned(), "vert".to_owned(),], + }, + ImportDefinition { + import: "\"shaders/skills/lightning_ring.wgsl\"".to_owned(), + items: vec!["frag".to_owned(), "vert".to_owned(),], + }, + ImportDefinition { + import: "\"shaders/skills/magic_arrow.wgsl\"".to_owned(), + items: vec!["frag".to_owned(), "vert".to_owned(),], + }, + ImportDefinition { + import: "\"shaders/skills/orb.wgsl\"".to_owned(), + items: vec!["frag".to_owned(), "vert".to_owned(),], + }, + ImportDefinition { + import: "\"shaders/skills/railgun_trail.wgsl\"".to_owned(), + items: vec!["frag".to_owned(), "vert".to_owned(),], + }, + ImportDefinition { + import: "\"shaders/skills/shared.wgsl\"".to_owned(), + items: vec![ + "Vertex".to_owned(), + "VertexOutput".to_owned(), + "VertexOutput".to_owned(), + ], + }, + ImportDefinition { + import: "\"shaders/skills/slash.wgsl\"".to_owned(), + items: vec!["frag".to_owned(), "vert".to_owned(),], + }, + ImportDefinition { + import: "\"shaders/skills/sound.wgsl\"".to_owned(), + items: vec!["frag".to_owned(), "vert".to_owned(),], + }, + ] + ); + } + + #[test] + fn test_quoted_import_dup_name() { + let mut composer = Composer::default(); + + composer + .add_composable_module(ComposableModuleDescriptor { + source: include_str!("tests/quoted_dup/mod.wgsl"), + file_path: "tests/quoted_dup/mod.wgsl", + ..Default::default() + }) + .unwrap(); + + let module = composer + .make_naga_module(NagaModuleDescriptor { + source: include_str!("tests/quoted_dup/top.wgsl"), + file_path: "tests/quoted_dup/top.wgsl", + ..Default::default() + }) + .unwrap(); + + let info = naga::valid::Validator::new( + naga::valid::ValidationFlags::all(), + naga::valid::Capabilities::default(), + ) + .validate(&module) + .unwrap(); + let wgsl = naga::back::wgsl::write_string( + &module, + &info, + naga::back::wgsl::WriterFlags::EXPLICIT_TYPES, + ) + .unwrap(); + + // let mut f = std::fs::File::create("test_quoted_import_dup_name.txt").unwrap(); + // f.write_all(wgsl.as_bytes()).unwrap(); + // drop(f); + + output_eq!(wgsl, "tests/expected/test_quoted_import_dup_name.txt"); + } + #[test] fn use_shared_global() { let mut composer = Composer::default(); diff --git a/src/compose/tests/bevy_path_imports/skill.wgsl b/src/compose/tests/bevy_path_imports/skill.wgsl new file mode 100644 index 0000000..04a9139 --- /dev/null +++ b/src/compose/tests/bevy_path_imports/skill.wgsl @@ -0,0 +1,31 @@ +#import "shaders/skills/shared.wgsl" Vertex, VertexOutput + +#if EFFECT_ID == 0 + #import "shaders/skills/sound.wgsl" frag, vert +#else if EFFECT_ID == 1 + #import "shaders/skills/orb.wgsl" frag, vert +#else if EFFECT_ID == 2 + #import "shaders/skills/slash.wgsl" frag, vert +#else if EFFECT_ID == 3 + #import "shaders/skills/railgun_trail.wgsl" frag, vert +#else if EFFECT_ID == 4 + #import "shaders/skills/magic_arrow.wgsl" frag, vert +#else if EFFECT_ID == 5 + #import "shaders/skills/hit.wgsl" frag, vert +#else if EFFECT_ID == 6 + #import "shaders/skills/lightning_ring.wgsl" frag, vert +#else if EFFECT_ID == 7 + #import "shaders/skills/lightning.wgsl" frag, vert +#endif + +#import something_unused + +@fragment +fn fragment(in: VertexOutput) -> @location(0) vec4 { + return frag(in); +} + +@vertex +fn vertex(vertex: Vertex) -> VertexOutput { + return vert(vertex); +} diff --git a/src/compose/tests/call_entrypoint/include.wgsl b/src/compose/tests/call_entrypoint/include.wgsl index ccf932b..c44c757 100644 --- a/src/compose/tests/call_entrypoint/include.wgsl +++ b/src/compose/tests/call_entrypoint/include.wgsl @@ -8,5 +8,5 @@ fn non_ep(f: f32) -> f32 { fn fragment( @builtin(position) frag_coord: vec4, ) -> @location(0) vec4 { - return vec4(1.0); + return vec4(1.5 * frag_coord); } \ No newline at end of file diff --git a/src/compose/tests/dup_struct_import/a.wgsl b/src/compose/tests/dup_struct_import/a.wgsl index e0b484d..cba7a1a 100644 --- a/src/compose/tests/dup_struct_import/a.wgsl +++ b/src/compose/tests/dup_struct_import/a.wgsl @@ -2,7 +2,7 @@ #import struct fn a() -> struct::MyStruct { - var s: struct::MyStruct; - s.value = 1.0; - return s; + var s_a: struct::MyStruct; + s_a.value = 1.0; + return s_a; } \ No newline at end of file diff --git a/src/compose/tests/dup_struct_import/b.wgsl b/src/compose/tests/dup_struct_import/b.wgsl index b8d5630..aad40b1 100644 --- a/src/compose/tests/dup_struct_import/b.wgsl +++ b/src/compose/tests/dup_struct_import/b.wgsl @@ -2,7 +2,7 @@ #import struct fn b() -> struct::MyStruct { - var s: struct::MyStruct; - s.value = 2.0; - return s; + var s_b: struct::MyStruct; + s_b.value = 2.0; + return s_b; } \ No newline at end of file diff --git a/src/compose/tests/dup_struct_import/top.wgsl b/src/compose/tests/dup_struct_import/top.wgsl index a4a835e..a5fa48f 100644 --- a/src/compose/tests/dup_struct_import/top.wgsl +++ b/src/compose/tests/dup_struct_import/top.wgsl @@ -4,5 +4,5 @@ fn main() -> f32 { let a = a::a(); let b = b::b(); - return a.value + b.value; + return a.value / b.value; } \ No newline at end of file diff --git a/src/compose/tests/error_test/include.wgsl b/src/compose/tests/error_test/include.wgsl index 056cdd4..406ba51 100644 --- a/src/compose/tests/error_test/include.wgsl +++ b/src/compose/tests/error_test/include.wgsl @@ -6,4 +6,7 @@ or here, just moving lines around a bit #import missing -fn sub() {} +fn sub() { + // have to use something for it to be declared missing + let x = missing::y(); +} diff --git a/src/compose/tests/error_test/wgsl_parse_wrap.wgsl b/src/compose/tests/error_test/wgsl_parse_wrap.wgsl index 5f8f861..aecc9d7 100644 --- a/src/compose/tests/error_test/wgsl_parse_wrap.wgsl +++ b/src/compose/tests/error_test/wgsl_parse_wrap.wgsl @@ -1,3 +1,3 @@ -#import wgsl_parse_err - -fn ok() {} \ No newline at end of file +fn ok() { + wgsl_parse_err::woops(); +} \ No newline at end of file diff --git a/src/compose/tests/error_test/wgsl_valid_wrap.wgsl b/src/compose/tests/error_test/wgsl_valid_wrap.wgsl index 60e25ef..5260f9f 100644 --- a/src/compose/tests/error_test/wgsl_valid_wrap.wgsl +++ b/src/compose/tests/error_test/wgsl_valid_wrap.wgsl @@ -1,3 +1,3 @@ -#import valid_inc - -fn whatever() {} \ No newline at end of file +fn whatever() { + valid_inc::main(); +} \ No newline at end of file diff --git a/src/compose/tests/expected/additional_import.txt b/src/compose/tests/expected/additional_import.txt index 59f8b91..f57a22d 100644 --- a/src/compose/tests/expected/additional_import.txt +++ b/src/compose/tests/expected/additional_import.txt @@ -1,14 +1,14 @@ -fn _naga_oil_mod_N53GK4TSNFSGCYTMMU_memberfunc() -> f32 { +fn funcX_naga_oil_mod_XN53GK4TSNFSGCYTMMUX() -> f32 { return 1.0; } -fn _naga_oil_mod_OBWHKZ3JNY_member_naga_oil_vrt_N53GK4TSNFSGCYTMMU_memberfunc() -> f32 { - let _e0: f32 = _naga_oil_mod_N53GK4TSNFSGCYTMMU_memberfunc(); +fn funcX_naga_oil_vrt_XN53GK4TSNFSGCYTMMUXX_naga_oil_mod_XOBWHKZ3JNYX() -> f32 { + let _e0: f32 = funcX_naga_oil_mod_XN53GK4TSNFSGCYTMMUX(); return (_e0 + 1.0); } fn entry_point() -> f32 { - let _e0: f32 = _naga_oil_mod_OBWHKZ3JNY_member_naga_oil_vrt_N53GK4TSNFSGCYTMMU_memberfunc(); + let _e0: f32 = funcX_naga_oil_vrt_XN53GK4TSNFSGCYTMMUXX_naga_oil_mod_XOBWHKZ3JNYX(); return _e0; } diff --git a/src/compose/tests/expected/atomics.txt b/src/compose/tests/expected/atomics.txt index 051f2e4..760d5f0 100644 --- a/src/compose/tests/expected/atomics.txt +++ b/src/compose/tests/expected/atomics.txt @@ -3,31 +3,31 @@ struct gen___atomic_compare_exchange_resultUint4_ { exchanged: bool, } -var _naga_oil_mod_ORSXG5C7NVXWI5LMMU_memberatom: atomic; +var atomX_naga_oil_mod_XORSXG5C7NVXWI5LMMUX: atomic; -fn _naga_oil_mod_ORSXG5C7NVXWI5LMMU_memberentry_point() -> f32 { +fn entry_pointX_naga_oil_mod_XORSXG5C7NVXWI5LMMUX() -> f32 { var y: u32; - atomicStore((&_naga_oil_mod_ORSXG5C7NVXWI5LMMU_memberatom), 1u); - let _e3: u32 = atomicLoad((&_naga_oil_mod_ORSXG5C7NVXWI5LMMU_memberatom)); + atomicStore((&atomX_naga_oil_mod_XORSXG5C7NVXWI5LMMUX), 1u); + let _e3: u32 = atomicLoad((&atomX_naga_oil_mod_XORSXG5C7NVXWI5LMMUX)); y = _e3; - let _e7: u32 = atomicAdd((&_naga_oil_mod_ORSXG5C7NVXWI5LMMU_memberatom), 2u); + let _e7: u32 = atomicAdd((&atomX_naga_oil_mod_XORSXG5C7NVXWI5LMMUX), 2u); let _e8: u32 = y; y = (_e8 + _e7); - let _e12: u32 = atomicSub((&_naga_oil_mod_ORSXG5C7NVXWI5LMMU_memberatom), 1u); + let _e12: u32 = atomicSub((&atomX_naga_oil_mod_XORSXG5C7NVXWI5LMMUX), 1u); let _e13: u32 = y; y = (_e13 + _e12); - let _e17: u32 = atomicMax((&_naga_oil_mod_ORSXG5C7NVXWI5LMMU_memberatom), 5u); + let _e17: u32 = atomicMax((&atomX_naga_oil_mod_XORSXG5C7NVXWI5LMMUX), 5u); let _e18: u32 = y; y = (_e18 + _e17); - let _e22: u32 = atomicMin((&_naga_oil_mod_ORSXG5C7NVXWI5LMMU_memberatom), 4u); + let _e22: u32 = atomicMin((&atomX_naga_oil_mod_XORSXG5C7NVXWI5LMMUX), 4u); let _e23: u32 = y; y = (_e23 + _e22); let _e25: u32 = y; - let _e27: u32 = atomicExchange((&_naga_oil_mod_ORSXG5C7NVXWI5LMMU_memberatom), _e25); + let _e27: u32 = atomicExchange((&atomX_naga_oil_mod_XORSXG5C7NVXWI5LMMUX), _e25); let _e28: u32 = y; y = (_e28 + _e27); - let _e33: gen___atomic_compare_exchange_resultUint4_ = atomicCompareExchangeWeak((&_naga_oil_mod_ORSXG5C7NVXWI5LMMU_memberatom), 12u, 0u); + let _e33: gen___atomic_compare_exchange_resultUint4_ = atomicCompareExchangeWeak((&atomX_naga_oil_mod_XORSXG5C7NVXWI5LMMUX), 12u, 0u); if _e33.exchanged { let _e36: u32 = y; y = (_e36 + _e33.old_value); @@ -37,7 +37,7 @@ fn _naga_oil_mod_ORSXG5C7NVXWI5LMMU_memberentry_point() -> f32 { } fn main() -> f32 { - let _e0: f32 = _naga_oil_mod_ORSXG5C7NVXWI5LMMU_memberentry_point(); + let _e0: f32 = entry_pointX_naga_oil_mod_XORSXG5C7NVXWI5LMMUX(); return _e0; } diff --git a/src/compose/tests/expected/bad_identifiers.txt b/src/compose/tests/expected/bad_identifiers.txt new file mode 100644 index 0000000..a67db37 --- /dev/null +++ b/src/compose/tests/expected/bad_identifiers.txt @@ -0,0 +1,40 @@ + + + + + + + + + d.fine = 3.0; + e.fine_member = 4.0; + fine: f32, + fine_member: f32, + let _e11: f32 = bad_X_naga_oil_mod_XM5WG6YTBNRZQX; + let _e22: f32 = d.fine; + let _e25: f32 = e.fine_member; + let _e4: f32 = fineX_naga_oil_mod_XMZXHGX(1.0); + let _e6: f32 = bad_X_naga_oil_mod_XMZXHGX(2.0); + let _e9: f32 = fineX_naga_oil_mod_XM5WG6YTBNRZQX; + let a: f32 = (fineX_naga_oil_mod_XMNXW443UOMX + bad_X_naga_oil_mod_XMNXW443UOMX); + let b: f32 = (_e4 + _e6); + let c: f32 = (_e9 + _e11); + return ((((a + b) + c) + _e22) + _e25); + return in; + return in_1; + var d: IsFineX_naga_oil_mod_XON2HE5LDORZQX; + var e: Isbad_X_naga_oil_mod_XON2HE5LDORZQX; +const bad_X_naga_oil_mod_XMNXW443UOMX: f32 = 1.0; +const fineX_naga_oil_mod_XMNXW443UOMX: f32 = 1.0; +fn bad_X_naga_oil_mod_XMZXHGX(in_1: f32) -> f32 { +fn fineX_naga_oil_mod_XMZXHGX(in: f32) -> f32 { +fn main() -> f32 { +struct IsFineX_naga_oil_mod_XON2HE5LDORZQX { +struct Isbad_X_naga_oil_mod_XON2HE5LDORZQX { +var bad_X_naga_oil_mod_XM5WG6YTBNRZQX: f32 = 1.0; +var fineX_naga_oil_mod_XM5WG6YTBNRZQX: f32 = 1.0; +} +} +} +} +} \ No newline at end of file diff --git a/src/compose/tests/expected/big_shaderdefs.txt b/src/compose/tests/expected/big_shaderdefs.txt index ae8b6d6..45535e0 100644 --- a/src/compose/tests/expected/big_shaderdefs.txt +++ b/src/compose/tests/expected/big_shaderdefs.txt @@ -1,4 +1,4 @@ -fn _naga_oil_mod_NVXWI_memberf() -> f32 { +fn fX_naga_oil_mod_XNVXWIX() -> f32 { var x: f32; x = 0.0; @@ -8,7 +8,7 @@ fn _naga_oil_mod_NVXWI_memberf() -> f32 { } fn main() -> f32 { - let _e0: f32 = _naga_oil_mod_NVXWI_memberf(); + let _e0: f32 = fX_naga_oil_mod_XNVXWIX(); return _e0; } diff --git a/src/compose/tests/expected/conditional_import_a.txt b/src/compose/tests/expected/conditional_import_a.txt index 2f41734..6e9f4c1 100644 --- a/src/compose/tests/expected/conditional_import_a.txt +++ b/src/compose/tests/expected/conditional_import_a.txt @@ -1,6 +1,6 @@ -const _naga_oil_mod_ME_memberC: u32 = 1u; +const CX_naga_oil_mod_XMEX: u32 = 1u; fn main() -> u32 { - return _naga_oil_mod_ME_memberC; + return CX_naga_oil_mod_XMEX; } diff --git a/src/compose/tests/expected/conditional_import_b.txt b/src/compose/tests/expected/conditional_import_b.txt index 31a7a56..1284aa5 100644 --- a/src/compose/tests/expected/conditional_import_b.txt +++ b/src/compose/tests/expected/conditional_import_b.txt @@ -1,6 +1,6 @@ -const _naga_oil_mod_MI_memberC: u32 = 2u; +const CX_naga_oil_mod_XMIX: u32 = 2u; fn main() -> u32 { - return _naga_oil_mod_MI_memberC; + return CX_naga_oil_mod_XMIX; } diff --git a/src/compose/tests/expected/dup_import.txt b/src/compose/tests/expected/dup_import.txt index 855c630..822ce70 100644 --- a/src/compose/tests/expected/dup_import.txt +++ b/src/compose/tests/expected/dup_import.txt @@ -1,20 +1,16 @@ -const _naga_oil_mod_MNXW443UOM_memberPI: f32 = 3.1; -fn _naga_oil_mod_ME_memberf() -> f32 { - return (_naga_oil_mod_MNXW443UOM_memberPI * 1.0); -} -fn _naga_oil_mod_MI_memberf() -> f32 { - return (_naga_oil_mod_MNXW443UOM_memberPI * 2.0); -} -fn _naga_oil_mod_MI_memberg() -> f32 { - return (_naga_oil_mod_MNXW443UOM_memberPI * 2.0); -} -fn main() -> f32 { - let _e0: f32 = _naga_oil_mod_ME_memberf(); - let _e1: f32 = _naga_oil_mod_MI_memberf(); + let _e0: f32 = fX_naga_oil_mod_XMEX(); + let _e1: f32 = fX_naga_oil_mod_XMIX(); + return (PIX_naga_oil_mod_XMNXW443UOMX * 1.0); + return (PIX_naga_oil_mod_XMNXW443UOMX * 2.0); return (_e0 * _e1); +const PIX_naga_oil_mod_XMNXW443UOMX: f32 = 3.1; +fn fX_naga_oil_mod_XMEX() -> f32 { +fn fX_naga_oil_mod_XMIX() -> f32 { +fn main() -> f32 { } - +} +} \ No newline at end of file diff --git a/src/compose/tests/expected/dup_struct_import.txt b/src/compose/tests/expected/dup_struct_import.txt index 910e5b5..e0eed15 100644 --- a/src/compose/tests/expected/dup_struct_import.txt +++ b/src/compose/tests/expected/dup_struct_import.txt @@ -1,26 +1,26 @@ -struct _naga_oil_mod_ON2HE5LDOQ_memberMyStruct { - value: f32, -} -fn _naga_oil_mod_ME_membera() -> _naga_oil_mod_ON2HE5LDOQ_memberMyStruct { - var s: _naga_oil_mod_ON2HE5LDOQ_memberMyStruct; - s.value = 1.0; - let _e3: _naga_oil_mod_ON2HE5LDOQ_memberMyStruct = s; - return _e3; -} -fn _naga_oil_mod_MI_memberb() -> _naga_oil_mod_ON2HE5LDOQ_memberMyStruct { - var s_1: _naga_oil_mod_ON2HE5LDOQ_memberMyStruct; - s_1.value = 2.0; - let _e3: _naga_oil_mod_ON2HE5LDOQ_memberMyStruct = s_1; - return _e3; -} + + let _e0: MyStructX_naga_oil_mod_XON2HE5LDOQX = aX_naga_oil_mod_XMEX(); + let _e1: MyStructX_naga_oil_mod_XON2HE5LDOQX = bX_naga_oil_mod_XMIX(); + let _e3: MyStructX_naga_oil_mod_XON2HE5LDOQX = s_a; + let _e3: MyStructX_naga_oil_mod_XON2HE5LDOQX = s_b; + return (_e0.value / _e1.value); + return _e3; + return _e3; + s_a.value = 1.0; + s_b.value = 2.0; + value: f32, + var s_a: MyStructX_naga_oil_mod_XON2HE5LDOQX; + var s_b: MyStructX_naga_oil_mod_XON2HE5LDOQX; +fn aX_naga_oil_mod_XMEX() -> MyStructX_naga_oil_mod_XON2HE5LDOQX { +fn bX_naga_oil_mod_XMIX() -> MyStructX_naga_oil_mod_XON2HE5LDOQX { fn main() -> f32 { - let _e0: _naga_oil_mod_ON2HE5LDOQ_memberMyStruct = _naga_oil_mod_ME_membera(); - let _e1: _naga_oil_mod_ON2HE5LDOQ_memberMyStruct = _naga_oil_mod_MI_memberb(); - return (_e0.value + _e1.value); +struct MyStructX_naga_oil_mod_XON2HE5LDOQX { } - +} +} +} \ No newline at end of file diff --git a/src/compose/tests/expected/err_validation_2.txt b/src/compose/tests/expected/err_validation_2.txt index e6276ba..9bd41d1 100644 --- a/src/compose/tests/expected/err_validation_2.txt +++ b/src/compose/tests/expected/err_validation_2.txt @@ -1,10 +1,10 @@ -error: failed to build a valid final module: Function [2] 'valid_inc::func' is invalid +error: failed to build a valid final module: Function [1] 'valid_inc::func' is invalid ┌─ tests/error_test/wgsl_valid_err.wgsl:7:1 │ 7 │ ╭ fn func() -> f32 { 8 │ │ return 1u; │ │ ^^ naga::Expression [1] - │ ╰──────────────^ naga::Function [2] + │ ╰──────────────^ naga::Function [1] │ = The `return` value Some([1]) does not match the function return value diff --git a/src/compose/tests/expected/glsl_call_wgsl.txt b/src/compose/tests/expected/glsl_call_wgsl.txt index 5cba301..f5caede 100644 --- a/src/compose/tests/expected/glsl_call_wgsl.txt +++ b/src/compose/tests/expected/glsl_call_wgsl.txt @@ -4,12 +4,12 @@ struct VertexOutput { var gl_Position: vec4; -fn _naga_oil_mod_O5TXG3C7NVXWI5LMMU_memberwgsl_func() -> f32 { +fn wgsl_funcX_naga_oil_mod_XO5TXG3C7NVXWI5LMMUX() -> f32 { return 53.0; } fn main_1() { - let _e0: f32 = _naga_oil_mod_O5TXG3C7NVXWI5LMMU_memberwgsl_func(); + let _e0: f32 = wgsl_funcX_naga_oil_mod_XO5TXG3C7NVXWI5LMMUX(); gl_Position = vec4(_e0); return; } diff --git a/src/compose/tests/expected/glsl_const_import.txt b/src/compose/tests/expected/glsl_const_import.txt index b39a96f..f46c308 100644 --- a/src/compose/tests/expected/glsl_const_import.txt +++ b/src/compose/tests/expected/glsl_const_import.txt @@ -2,16 +2,12 @@ struct FragmentOutput { @location(0) out_color: vec4, } -const _naga_oil_mod_MNXW23LPNY_membermy_constant: f32 = 0.5; +const my_constantX_naga_oil_mod_XMNXW23LPNYX: f32 = 0.5; var out_color: vec4; -fn _naga_oil_mod_MNXW23LPNY_membermain() { - return; -} - fn main_1() { - out_color = vec4(f32(1), _naga_oil_mod_MNXW23LPNY_membermy_constant, f32(0), f32(1)); + out_color = vec4(f32(1), my_constantX_naga_oil_mod_XMNXW23LPNYX, f32(0), f32(1)); return; } diff --git a/src/compose/tests/expected/glsl_wgsl_const_import.txt b/src/compose/tests/expected/glsl_wgsl_const_import.txt index ed543e4..637c511 100644 --- a/src/compose/tests/expected/glsl_wgsl_const_import.txt +++ b/src/compose/tests/expected/glsl_wgsl_const_import.txt @@ -1,10 +1,6 @@ -const _naga_oil_mod_MNXW23LPNY_membermy_constant: f32 = 0.5; - -fn _naga_oil_mod_MNXW23LPNY_membermain() { - return; -} +const my_constantX_naga_oil_mod_XMNXW23LPNYX: f32 = 0.5; fn main() -> vec4 { - return vec4(1.0, _naga_oil_mod_MNXW23LPNY_membermy_constant, 0.0, 1.0); + return vec4(1.0, my_constantX_naga_oil_mod_XMNXW23LPNYX, 0.0, 1.0); } diff --git a/src/compose/tests/expected/import_in_decl.txt b/src/compose/tests/expected/import_in_decl.txt index c1817c3..abc899f 100644 --- a/src/compose/tests/expected/import_in_decl.txt +++ b/src/compose/tests/expected/import_in_decl.txt @@ -1,10 +1,9 @@ -const _naga_oil_mod_MNXW443UOM_memberX: u32 = 1u; -const _naga_oil_mod_MJUW4ZA_membery: u32 = 2u; +const XX_naga_oil_mod_XMNXW443UOMX: u32 = 1u; -var _naga_oil_mod_MJUW4ZA_memberarr: array; +var arrX_naga_oil_mod_XMJUW4ZAX: array; fn main() -> f32 { - let _e2: u32 = _naga_oil_mod_MJUW4ZA_memberarr[0]; + let _e2: u32 = arrX_naga_oil_mod_XMJUW4ZAX[0]; return f32(_e2); } diff --git a/src/compose/tests/expected/invalid_override_base.txt b/src/compose/tests/expected/invalid_override_base.txt index a0ee0a4..f2eb81a 100644 --- a/src/compose/tests/expected/invalid_override_base.txt +++ b/src/compose/tests/expected/invalid_override_base.txt @@ -1,6 +1,6 @@ error: override is invalid as `outer` is not virtual (this error can be disabled with feature 'override_any') - ┌─ tests/overrides/top_invalid.wgsl:3:39 + ┌─ tests/overrides/top_invalid.wgsl:3:13 │ 3 │ override fn mod::outer() -> f32 { - │ ^ + │ ^ diff --git a/src/compose/tests/expected/item_import_test.txt b/src/compose/tests/expected/item_import_test.txt index 00d9170..2f0600e 100644 --- a/src/compose/tests/expected/item_import_test.txt +++ b/src/compose/tests/expected/item_import_test.txt @@ -2,14 +2,14 @@ - let _e1: u32 = _naga_oil_mod_MNXW443UOM_memberdouble(_naga_oil_mod_MNXW443UOM_memberX); - let _e1: u32 = _naga_oil_mod_MNXW443UOM_memberdouble(_naga_oil_mod_MNXW443UOM_memberY); + let _e1: u32 = doubleX_naga_oil_mod_XMNXW443UOMX(XX_naga_oil_mod_XMNXW443UOMX); + let _e1: u32 = doubleX_naga_oil_mod_XMNXW443UOMX(YX_naga_oil_mod_XMNXW443UOMX); return (in * 2u); return _e1; return _e1; -const _naga_oil_mod_MNXW443UOM_memberX: u32 = 1u; -const _naga_oil_mod_MNXW443UOM_memberY: u32 = 2u; -fn _naga_oil_mod_MNXW443UOM_memberdouble(in: u32) -> u32 { +const XX_naga_oil_mod_XMNXW443UOMX: u32 = 1u; +const YX_naga_oil_mod_XMNXW443UOMX: u32 = 2u; +fn doubleX_naga_oil_mod_XMNXW443UOMX(in: u32) -> u32 { fn main() -> u32 { fn other() -> u32 { } diff --git a/src/compose/tests/expected/item_sub_point.txt b/src/compose/tests/expected/item_sub_point.txt index 8b89742..63daa0f 100644 --- a/src/compose/tests/expected/item_sub_point.txt +++ b/src/compose/tests/expected/item_sub_point.txt @@ -1,17 +1,17 @@ -struct _naga_oil_mod_NVXWI_memberFrag { +struct FragX_naga_oil_mod_XNVXWIX { fragment: f32, } -fn _naga_oil_mod_NVXWI_memberfragment(f_1: _naga_oil_mod_NVXWI_memberFrag) -> f32 { +fn fragmentX_naga_oil_mod_XNVXWIX(f_1: FragX_naga_oil_mod_XNVXWIX) -> f32 { return (f_1.fragment * 2.0); } @fragment fn main() -> @location(0) f32 { - var f: _naga_oil_mod_NVXWI_memberFrag; + var f: FragX_naga_oil_mod_XNVXWIX; f.fragment = 3.0; - let _e3: _naga_oil_mod_NVXWI_memberFrag = f; - let _e4: f32 = _naga_oil_mod_NVXWI_memberfragment(_e3); + let _e3: FragX_naga_oil_mod_XNVXWIX = f; + let _e4: f32 = fragmentX_naga_oil_mod_XNVXWIX(_e3); return _e4; } diff --git a/src/compose/tests/expected/missing_import.txt b/src/compose/tests/expected/missing_import.txt index cf4751f..f08bccb 100644 --- a/src/compose/tests/expected/missing_import.txt +++ b/src/compose/tests/expected/missing_import.txt @@ -1,8 +1,8 @@ error: required import 'missing' not found - ┌─ tests/error_test/include.wgsl:7:1 - │ -7 │ #import missing - │ ^ - │ - = missing import 'missing' + ┌─ tests/error_test/include.wgsl:11:13 + │ +11 │ let x = missing::y(); + │ ^ + │ + = missing import 'missing' diff --git a/src/compose/tests/expected/simple_compose.txt b/src/compose/tests/expected/simple_compose.txt index 3c97a9f..d8aacaa 100644 --- a/src/compose/tests/expected/simple_compose.txt +++ b/src/compose/tests/expected/simple_compose.txt @@ -1,9 +1,9 @@ -fn _naga_oil_mod_NFXGG_memberhello() -> f32 { +fn helloX_naga_oil_mod_XNFXGGX() -> f32 { return 1.0; } fn main() -> f32 { - let _e0: f32 = _naga_oil_mod_NFXGG_memberhello(); + let _e0: f32 = helloX_naga_oil_mod_XNFXGGX(); return _e0; } diff --git a/src/compose/tests/expected/test_quoted_import_dup_name.txt b/src/compose/tests/expected/test_quoted_import_dup_name.txt new file mode 100644 index 0000000..ce3b573 --- /dev/null +++ b/src/compose/tests/expected/test_quoted_import_dup_name.txt @@ -0,0 +1,14 @@ +fn fooX_naga_oil_mod_XEJYXK33UMVSF63LPMR2WYZJCX() -> f32 { + return 3.0; +} + +fn myfunc(foo: u32) -> f32 { + return (f32(foo) * 2.0); +} + +fn main() -> f32 { + let _e1: f32 = myfunc(1u); + let _e2: f32 = fooX_naga_oil_mod_XEJYXK33UMVSF63LPMR2WYZJCX(); + return (_e1 + _e2); +} + diff --git a/src/compose/tests/expected/use_shared_global.txt b/src/compose/tests/expected/use_shared_global.txt index 54d95c9..f4a1263 100644 --- a/src/compose/tests/expected/use_shared_global.txt +++ b/src/compose/tests/expected/use_shared_global.txt @@ -1,15 +1,15 @@ -var _naga_oil_mod_NVXWI_membera: f32 = 0.0; +var aX_naga_oil_mod_XNVXWIX: f32 = 0.0; fn add() { - let _e2: f32 = _naga_oil_mod_NVXWI_membera; - _naga_oil_mod_NVXWI_membera = (_e2 + 1.0); + let _e2: f32 = aX_naga_oil_mod_XNVXWIX; + aX_naga_oil_mod_XNVXWIX = (_e2 + 1.0); return; } fn main() -> f32 { add(); add(); - let _e1: f32 = _naga_oil_mod_NVXWI_membera; + let _e1: f32 = aX_naga_oil_mod_XNVXWIX; return _e1; } diff --git a/src/compose/tests/expected/wgsl_call_entrypoint.txt b/src/compose/tests/expected/wgsl_call_entrypoint.txt index 801fec1..8df9240 100644 --- a/src/compose/tests/expected/wgsl_call_entrypoint.txt +++ b/src/compose/tests/expected/wgsl_call_entrypoint.txt @@ -1,13 +1,9 @@ -fn _naga_oil_mod_NFXGG3DVMRSQ_membernon_ep(f: f32) -> f32 { - return (f * 2.0); -} - -fn _naga_oil_mod_NFXGG3DVMRSQ_memberfragment(frag_coord_1: vec4) -> vec4 { - return vec4(1.0); +fn fragmentX_naga_oil_mod_XNFXGG3DVMRSQX(frag_coord_1: vec4) -> vec4 { + return vec4((1.5 * frag_coord_1)); } @fragment fn fragment(@builtin(position) frag_coord: vec4) -> @location(0) vec4 { - let _e1: vec4 = _naga_oil_mod_NFXGG3DVMRSQ_memberfragment(frag_coord); + let _e1: vec4 = fragmentX_naga_oil_mod_XNFXGG3DVMRSQX(frag_coord); return _e1; } diff --git a/src/compose/tests/expected/wgsl_glsl_const_import.txt b/src/compose/tests/expected/wgsl_glsl_const_import.txt index d9fcd57..f46c308 100644 --- a/src/compose/tests/expected/wgsl_glsl_const_import.txt +++ b/src/compose/tests/expected/wgsl_glsl_const_import.txt @@ -2,12 +2,12 @@ struct FragmentOutput { @location(0) out_color: vec4, } -const _naga_oil_mod_MNXW23LPNY_membermy_constant: f32 = 0.5; +const my_constantX_naga_oil_mod_XMNXW23LPNYX: f32 = 0.5; var out_color: vec4; fn main_1() { - out_color = vec4(f32(1), _naga_oil_mod_MNXW23LPNY_membermy_constant, f32(0), f32(1)); + out_color = vec4(f32(1), my_constantX_naga_oil_mod_XMNXW23LPNYX, f32(0), f32(1)); return; } diff --git a/src/compose/tests/invalid_identifiers/top_invalid.wgsl b/src/compose/tests/invalid_identifiers/top_invalid.wgsl new file mode 100644 index 0000000..93c68f2 --- /dev/null +++ b/src/compose/tests/invalid_identifiers/top_invalid.wgsl @@ -0,0 +1,23 @@ +// #import consts +// #import fns +// #import globals +#import struct_members +// #import structs + +fn main() -> f32 { + // let a = consts::fine + consts::bad_; + // let b = fns::fine(1.0) + fns::bad_(2.0); + // let c = globals::fine + globals::bad_; + // var d: structs::IsFine; + // d.fine = 3.0; + // var e: structs::Isbad_; + // e.fine_member = 4.0; + var f = struct_members::FineStruct; + f.fine = 5.0; + var g = struct_members::BadStruct; + g.also_fine = 6.0; + g.bad_ = 7.0; + + // return a + b + c + d.fine + e.fine_member; + return f.fine + g.also_fine + g.bad_; +} \ No newline at end of file diff --git a/src/compose/tests/invalid_identifiers/top_valid.wgsl b/src/compose/tests/invalid_identifiers/top_valid.wgsl new file mode 100644 index 0000000..f0753b7 --- /dev/null +++ b/src/compose/tests/invalid_identifiers/top_valid.wgsl @@ -0,0 +1,22 @@ +#import consts +#import fns +#import globals +// #import struct_members +#import structs + +fn main() -> f32 { + let a = consts::fine + consts::bad_; + let b = fns::fine(1.0) + fns::bad_(2.0); + let c = globals::fine + globals::bad_; + var d: structs::IsFine; + d.fine = 3.0; + var e: structs::Isbad_; + e.fine_member = 4.0; + // var f = struct_members::FineStruct; + // f.fine = 5.0; + // var g = struct_members::BadStruct; + // g.also_fine = 6.0; + // g.bad_ = 7.0; + + return a + b + c + d.fine + e.fine_member; // + f.fine + g.also_fine + g.bad_; +} \ No newline at end of file diff --git a/src/compose/tests/quoted_dup/mod.wgsl b/src/compose/tests/quoted_dup/mod.wgsl new file mode 100644 index 0000000..d7cce94 --- /dev/null +++ b/src/compose/tests/quoted_dup/mod.wgsl @@ -0,0 +1,5 @@ +#define_import_path "quoted_module" + +fn foo() -> f32 { + return 3.0; +} \ No newline at end of file diff --git a/src/compose/tests/quoted_dup/top.wgsl b/src/compose/tests/quoted_dup/top.wgsl new file mode 100644 index 0000000..0ff6e24 --- /dev/null +++ b/src/compose/tests/quoted_dup/top.wgsl @@ -0,0 +1,9 @@ +#import "quoted_module" as foo; + +fn myfunc(foo: u32) -> f32 { + return f32(foo) * 2.0; +} + +fn main() -> f32 { + return myfunc(1u) + foo::foo(); +} \ No newline at end of file diff --git a/src/compose/tests/rusty_imports/mod_a_b_c.wgsl b/src/compose/tests/rusty_imports/mod_a_b_c.wgsl new file mode 100644 index 0000000..e8e0abc --- /dev/null +++ b/src/compose/tests/rusty_imports/mod_a_b_c.wgsl @@ -0,0 +1,7 @@ +#define_import_path a::b::c + +const C: f32 = 2.0; + +fn triple(in: f32) -> f32 { + return in * 3.0; +} \ No newline at end of file diff --git a/src/compose/tests/rusty_imports/mod_a_x.wgsl b/src/compose/tests/rusty_imports/mod_a_x.wgsl new file mode 100644 index 0000000..91a917e --- /dev/null +++ b/src/compose/tests/rusty_imports/mod_a_x.wgsl @@ -0,0 +1,5 @@ +#define_import_path a::x + +fn square(in: f32) -> f32 { + return in * in; +} \ No newline at end of file diff --git a/src/compose/tests/rusty_imports/top.wgsl b/src/compose/tests/rusty_imports/top.wgsl new file mode 100644 index 0000000..2ee148f --- /dev/null +++ b/src/compose/tests/rusty_imports/top.wgsl @@ -0,0 +1,8 @@ +#define_import_path test_module + +#import a::b as partial_path +#import a::b::c as full_path + +fn entry_point() -> f32 { + return a::x::square(partial_path::c::triple(full_path::C)); +} \ No newline at end of file diff --git a/src/compose/tokenizer.rs b/src/compose/tokenizer.rs new file mode 100644 index 0000000..b57577c --- /dev/null +++ b/src/compose/tokenizer.rs @@ -0,0 +1,127 @@ +use std::collections::VecDeque; + +#[derive(PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +pub enum Token<'a> { + Identifier(&'a str, usize), + Other(char, usize), + Whitespace(&'a str, usize), +} + +impl<'a> Token<'a> { + pub fn pos(&self) -> usize { + match self { + Token::Identifier(_, pos) | Token::Other(_, pos) | Token::Whitespace(_, pos) => *pos, + } + } + + pub fn identifier(&self) -> Option<&str> { + match self { + Token::Identifier(ident, _) => Some(ident), + _ => None, + } + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum TokenKind { + Identifier, + Whitespace, +} + +// a basic tokenizer that separates identifiers from non-identifiers, and optionally returns whitespace tokens +// unicode XID rules apply, except that additional characters '"' and '::' (sequences of two colons) are allowed in identifiers. +// quotes treat any further chars until the next quote as part of the identifier. +// note we don't support non-USV identifiers like 👩‍👩‍👧‍👧 which is apparently in XID_continue +pub struct Tokenizer<'a> { + tokens: VecDeque>, +} + +impl<'a> Tokenizer<'a> { + pub fn new(src: &'a str, emit_whitespace: bool) -> Self { + let mut tokens = VecDeque::default(); + let mut current_token_start = 0; + let mut current_token = None; + let mut quoted_token = false; + + let mut chars = src.char_indices().peekable(); + + while let Some((ix, char)) = chars.next() { + if char == '"' { + quoted_token = !quoted_token; + if !quoted_token { + continue; + } + } + + if let Some(tok) = current_token { + match tok { + TokenKind::Identifier => { + // accept anything within quotes, or XID_continues + if quoted_token || unicode_ident::is_xid_continue(char) { + continue; + } + // accept `::` + if char == ':' && chars.peek() == Some(&(ix + 1, ':')) { + chars.next(); + continue; + } + + tokens.push_back(Token::Identifier( + &src[current_token_start..ix], + current_token_start, + )); + } + TokenKind::Whitespace => { + if char.is_whitespace() { + continue; + } + tokens.push_back(Token::Whitespace( + &src[current_token_start..ix], + current_token_start, + )); + } + }; + + current_token_start = ix; + current_token = None; + } + + if quoted_token || unicode_ident::is_xid_start(char) { + current_token = Some(TokenKind::Identifier); + current_token_start = ix; + } else if !char.is_whitespace() { + tokens.push_back(Token::Other(char, ix)); + } else if char.is_whitespace() && emit_whitespace { + current_token = Some(TokenKind::Whitespace); + current_token_start = ix; + } + } + + if let Some(tok) = current_token { + match tok { + TokenKind::Identifier => { + tokens.push_back(Token::Identifier( + &src[current_token_start..src.len()], + current_token_start, + )); + } + TokenKind::Whitespace => { + tokens.push_back(Token::Whitespace( + &src[current_token_start..src.len()], + current_token_start, + )); + } + }; + } + + Self { tokens } + } +} + +impl<'a> Iterator for Tokenizer<'a> { + type Item = Token<'a>; + + fn next(&mut self) -> Option { + self.tokens.pop_front() + } +}