From 55d20f38e6c5e2b50648d6f174e1039dce387459 Mon Sep 17 00:00:00 2001 From: SRetip Date: Wed, 3 Apr 2024 16:57:36 +0300 Subject: [PATCH 1/8] add support for external references & distinct_defs flag --- .idea/.gitignore | 8 + .idea/modules.xml | 8 + .idea/typify.iml | 20 + .idea/vcs.xml | 6 + Cargo.lock | 331 ++++++++++++++- cargo-typify/src/lib.rs | 11 + cargo-typify/tests/outputs/help.txt | 3 + rustfmt.toml | 1 + typify-impl/Cargo.toml | 3 +- typify-impl/src/convert.rs | 13 +- typify-impl/src/lib.rs | 634 +++++++++++++++++++++++++++- typify-impl/src/merge.rs | 2 +- typify-impl/src/util.rs | 8 +- 13 files changed, 1015 insertions(+), 33 deletions(-) create mode 100644 .idea/.gitignore create mode 100644 .idea/modules.xml create mode 100644 .idea/typify.iml create mode 100644 .idea/vcs.xml create mode 100644 rustfmt.toml diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 00000000..13566b81 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 00000000..a97b4fec --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/typify.iml b/.idea/typify.iml new file mode 100644 index 00000000..c1686e0c --- /dev/null +++ b/.idea/typify.iml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 00000000..35eb1ddf --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 4e267ee6..53d7480f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,25 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "abnf" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "087113bd50d9adce24850eed5d0476c7d199d532fce8fab5173650331e09033a" +dependencies = [ + "abnf-core", + "nom", +] + +[[package]] +name = "abnf-core" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c44e09c43ae1c368fb91a03a566472d0087c26cf7e1b9e8e289c14ede681dd7d" +dependencies = [ + "nom", +] + [[package]] name = "addr2line" version = "0.21.0" @@ -155,6 +174,15 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bstr" version = "1.8.0" @@ -166,6 +194,30 @@ dependencies = [ "serde", ] +[[package]] +name = "btree-range-map" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1be5c9672446d3800bcbcaabaeba121fe22f1fb25700c4562b22faf76d377c33" +dependencies = [ + "btree-slab", + "cc-traits", + "range-traits", + "serde", + "slab", +] + +[[package]] +name = "btree-slab" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2b56d3029f075c4fa892428a098425b86cef5c89ae54073137ece416aef13c" +dependencies = [ + "cc-traits", + "slab", + "smallvec", +] + [[package]] name = "bumpalo" version = "3.14.0" @@ -199,6 +251,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cc-traits" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "060303ef31ef4a522737e1b1ab68c67916f2a787bb2f4f54f383279adba962b5" +dependencies = [ + "slab", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -220,6 +281,33 @@ dependencies = [ "windows-targets 0.52.0", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "clap" version = "4.5.8" @@ -251,7 +339,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.68", ] [[package]] @@ -312,12 +400,47 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +[[package]] +name = "cpufeatures" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +dependencies = [ + "libc", +] + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "difflib" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "doc-comment" version = "0.3.3" @@ -379,7 +502,7 @@ dependencies = [ "schemars", "serde", "serde_json", - "syn", + "syn 2.0.68", "typify", ] @@ -425,6 +548,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "gimli" version = "0.28.1" @@ -437,6 +570,16 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +[[package]] +name = "half" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +dependencies = [ + "cfg-if", + "crunchy", +] + [[package]] name = "hashbrown" version = "0.14.3" @@ -459,6 +602,12 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +[[package]] +name = "hex_fmt" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b07f60793ff0a4d9cef0f18e63b5357e06209987153a64648c972c1e5aff336f" + [[package]] name = "home" version = "0.5.5" @@ -513,6 +662,12 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "indoc" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" + [[package]] name = "ipnetwork" version = "0.20.0" @@ -523,6 +678,18 @@ dependencies = [ "serde", ] +[[package]] +name = "iref" +version = "3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cfb4452af4310b464be640c873d17a1ae845709cdf2f024257deebabf1b84b4" +dependencies = [ + "pct-str", + "smallvec", + "static-regular-grammar", + "thiserror", +] + [[package]] name = "is-terminal" version = "0.4.9" @@ -588,6 +755,12 @@ version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.7.1" @@ -606,6 +779,16 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "num-traits" version = "0.2.17" @@ -642,6 +825,22 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pathdiff" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" + +[[package]] +name = "pct-str" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf1bdcc492c285a50bed60860dfa00b50baf1f60c73c7d6b435b01a2a11fd6ff" +dependencies = [ + "thiserror", + "utf8-decode", +] + [[package]] name = "pin-project-lite" version = "0.2.13" @@ -683,7 +882,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.68", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", ] [[package]] @@ -732,6 +955,12 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" +[[package]] +name = "range-traits" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d20581732dd76fa913c7dff1a2412b714afe3573e94d41c34719de73337cc8ab" + [[package]] name = "rdrand" version = "0.4.0" @@ -854,7 +1083,7 @@ dependencies = [ "proc-macro2", "quote", "schema-derive", - "syn", + "syn 2.0.68", ] [[package]] @@ -866,7 +1095,7 @@ dependencies = [ "paste", "proc-macro2", "quote", - "syn", + "syn 2.0.68", ] [[package]] @@ -891,7 +1120,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn", + "syn 2.0.68", ] [[package]] @@ -920,7 +1149,7 @@ checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.68", ] [[package]] @@ -931,7 +1160,7 @@ checksum = "330f01ce65a3a5fe59a60c82f3c9a024b573b8a6e875bd233fe5f934e71d54e3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.68", ] [[package]] @@ -963,7 +1192,18 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn", + "syn 2.0.68", +] + +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", ] [[package]] @@ -981,12 +1221,57 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2aeaf503862c419d66959f5d7ca015337d864e9c49485d771b732e2a20453597" +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "static-regular-grammar" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f74f34069d6f26a5d63fcbc3378a20798cf83572fb87a71b805bca7e6a790cc" +dependencies = [ + "abnf", + "btree-range-map", + "ciborium", + "hex_fmt", + "indoc", + "proc-macro-error", + "proc-macro2", + "quote", + "serde", + "sha2", + "syn 2.0.68", + "thiserror", +] + [[package]] name = "strsim" version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ee073c9e4cd00e28217186dbe12796d692868f432bf2e97ee73bed0c56dfa01" +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.68" @@ -1053,7 +1338,7 @@ checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.68", ] [[package]] @@ -1168,6 +1453,12 @@ dependencies = [ "toml", ] +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + [[package]] name = "typify" version = "0.1.0" @@ -1195,8 +1486,10 @@ dependencies = [ "env_logger", "expectorate", "heck", + "iref", "log", "paste", + "pathdiff", "proc-macro2", "quote", "regress", @@ -1206,7 +1499,7 @@ dependencies = [ "semver", "serde", "serde_json", - "syn", + "syn 2.0.68", "thiserror", "unicode-ident", "uuid", @@ -1223,7 +1516,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn", + "syn 2.0.68", "typify-impl", ] @@ -1237,7 +1530,7 @@ dependencies = [ "schemars", "serde", "serde_json", - "syn", + "syn 2.0.68", "typify", ] @@ -1259,6 +1552,12 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" +[[package]] +name = "utf8-decode" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca61eb27fa339aa08826a29f03e87b99b4d8f0fc2255306fd266bb1b6a9de498" + [[package]] name = "utf8parse" version = "0.2.1" @@ -1326,7 +1625,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn", + "syn 2.0.68", "wasm-bindgen-shared", ] @@ -1348,7 +1647,7 @@ checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.68", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -1623,5 +1922,5 @@ checksum = "b3c129550b3e6de3fd0ba67ba5c81818f9805e58b8d7fee80a3a59d2c9fc601a" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.68", ] diff --git a/cargo-typify/src/lib.rs b/cargo-typify/src/lib.rs index 328b57b2..ef365417 100644 --- a/cargo-typify/src/lib.rs +++ b/cargo-typify/src/lib.rs @@ -53,6 +53,9 @@ pub struct CliArgs { value_parser = ["generate", "allow", "deny"] )] unknown_crates: Option, + + #[arg(short = 'D', long, default_value = "false")] + distinct_definitions: bool, } impl CliArgs { @@ -162,6 +165,8 @@ pub fn convert(args: &CliArgs) -> Result { } let mut type_space = TypeSpace::new(&settings); + type_space.with_path(&args.input); + type_space.distinct_defs(args.distinct_definitions); type_space .add_root_schema(schema) .wrap_err("Schema conversion failed")?; @@ -195,6 +200,7 @@ mod tests { no_builder: false, crates: vec![], unknown_crates: Default::default(), + distinct_definitions: false, }; assert_eq!(args.output_path(), None); @@ -210,6 +216,7 @@ mod tests { no_builder: false, crates: vec![], unknown_crates: Default::default(), + distinct_definitions: false, }; assert_eq!(args.output_path(), Some(PathBuf::from("some_file.rs"))); @@ -225,6 +232,7 @@ mod tests { no_builder: false, crates: vec![], unknown_crates: Default::default(), + distinct_definitions: false, }; assert_eq!(args.output_path(), Some(PathBuf::from("input.rs"))); @@ -240,6 +248,7 @@ mod tests { no_builder: false, crates: vec![], unknown_crates: Default::default(), + distinct_definitions: false, }; assert!(args.use_builder()); @@ -255,6 +264,7 @@ mod tests { no_builder: true, crates: vec![], unknown_crates: Default::default(), + distinct_definitions: false, }; assert!(!args.use_builder()); @@ -270,6 +280,7 @@ mod tests { no_builder: false, crates: vec![], unknown_crates: Default::default(), + distinct_definitions: false, }; assert!(args.use_builder()); diff --git a/cargo-typify/tests/outputs/help.txt b/cargo-typify/tests/outputs/help.txt index 74172166..2c61057f 100644 --- a/cargo-typify/tests/outputs/help.txt +++ b/cargo-typify/tests/outputs/help.txt @@ -29,6 +29,9 @@ Options: [possible values: generate, allow, deny] + -D, --distinct-definitions + + -h, --help Print help (see a summary with '-h') diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 00000000..248fb36c --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1 @@ +ignore = ["/"] diff --git a/typify-impl/Cargo.toml b/typify-impl/Cargo.toml index a135fa6a..bee7e47c 100644 --- a/typify-impl/Cargo.toml +++ b/typify-impl/Cargo.toml @@ -20,7 +20,8 @@ serde_json = "1.0.119" syn = { version = "2.0.68", features = ["full"] } thiserror = "1.0.61" unicode-ident = "1.0.12" - +pathdiff = "0.2.1" +iref = "3.1.4" [dev-dependencies] env_logger = "0.10.2" diff --git a/typify-impl/src/convert.rs b/typify-impl/src/convert.rs index ac8a7a02..203b15e5 100644 --- a/typify-impl/src/convert.rs +++ b/typify-impl/src/convert.rs @@ -366,7 +366,7 @@ impl TypeSpace { object: None, reference: Some(reference), extensions: _, - } => self.convert_reference(metadata, reference), + } => self.convert_reference(metadata, &reference), // Accept references that... for some reason... include the type. // TODO this could be generalized to validate any redundant @@ -884,6 +884,7 @@ impl TypeSpace { // this case and strip out the null in both enum values and instance // type. Nevertheless, we do our best to interpret even incorrect // JSON schema. + let mut has_null = false; let validator = StringValidator::new(&type_name, validation)?; @@ -1030,7 +1031,12 @@ impl TypeSpace { // f64 here, but we're already constrained by the schemars // representation so ... it's probably the best we can do at // the moment. - match (default.as_f64(), min, max) { + let d = match default { + serde_json::Value::Number(a) => a.as_f64(), + serde_json::Value::String(a) => a.parse().ok(), + _ => None, + }; + match (d, min, max) { (Some(_), None, None) => Some(()), (Some(value), None, Some(fmax)) if value <= fmax => Some(()), (Some(value), Some(fmin), None) if value >= fmin => Some(()), @@ -1173,9 +1179,6 @@ impl TypeSpace { metadata: &'a Option>, ref_name: &str, ) -> Result<(TypeEntry, &'a Option>)> { - if !ref_name.starts_with('#') { - panic!("external references are not supported: {}", ref_name); - } let key = ref_key(ref_name); let type_id = self .ref_to_id diff --git a/typify-impl/src/lib.rs b/typify-impl/src/lib.rs index b66f0b7a..4ee3ad08 100644 --- a/typify-impl/src/lib.rs +++ b/typify-impl/src/lib.rs @@ -5,13 +5,19 @@ #![deny(missing_docs)] use std::collections::{BTreeMap, BTreeSet}; +use std::path::PathBuf; use conversions::SchemaCache; +use iref::iri::FragmentBuf; +use iref::Iri; use log::info; use output::OutputSpace; +use pathdiff::diff_paths; use proc_macro2::TokenStream; use quote::{format_ident, quote, ToTokens}; -use schemars::schema::{Metadata, RootSchema, Schema}; +use schemars::schema::{ + Metadata, RootSchema, Schema, SchemaObject, SingleOrVec, SubschemaValidation, +}; use thiserror::Error; use type_entry::{ StructPropertyState, TypeEntry, TypeEntryDetails, TypeEntryNative, TypeEntryNewtype, @@ -207,6 +213,23 @@ pub struct TypeSpace { // Shared functions for generating default values defaults: BTreeSet, + + file_path: PathBuf, + + distinct_definitions: bool, +} + +impl TypeSpace { + + /// Sets the file path for the `TypeSpace` instance. + pub fn with_path>(&mut self, path: T) { + self.file_path = path.into().canonicalize().unwrap(); + } + + /// Configures whether the `TypeSpace` instance should use distinct definitions. + pub fn distinct_defs(&mut self, value: bool) { + self.distinct_definitions = value; + } } impl Default for TypeSpace { @@ -225,6 +248,8 @@ impl Default for TypeSpace { settings: Default::default(), cache: Default::default(), defaults: Default::default(), + file_path: Default::default(), + distinct_definitions: false, } } } @@ -470,6 +495,20 @@ impl TypeSpacePatch { } } +/// Retrieves id of the schema from possible places +fn get_schema_id(schema: &SchemaObject) -> Option { + schema + .metadata + .as_ref() + .and_then(|m| m.id.clone()) + .or_else(|| { + schema + .extensions + .get("id") + .map(|id| id.as_str().unwrap().to_string()) + }) +} + impl TypeSpace { /// Create a new TypeSpace with custom settings pub fn new(settings: &TypeSpaceSettings) -> Self { @@ -531,16 +570,16 @@ impl TypeSpace { .insert(ref_name.clone(), TypeId(base_id + index as u64)); self.definitions.insert(ref_name.clone(), schema.clone()); } - // Convert all types; note that we use the type id assigned from the // previous step because each type may create additional types. This // effectively is doing the work of `add_type_with_name` but for a // batch of types. for (index, (ref_name, schema)) in definitions.into_iter().enumerate() { info!( - "converting type: {:?} with schema {}", + "converting type: {:?} with schema {} {}", ref_name, - serde_json::to_string(&schema).unwrap() + serde_json::to_string(&schema).unwrap(), + line!() ); // Check for manually replaced types. Proceed with type conversion @@ -691,24 +730,117 @@ impl TypeSpace { pub fn add_root_schema(&mut self, schema: RootSchema) -> Result> { let RootSchema { meta_schema: _, - schema, + schema: schema_object, definitions, - } = schema; + } = schema.clone(); + + let s_id = get_schema_id(&schema_object); + + let untracked = schema_object + .extensions + .clone() + .into_iter() + .filter_map(|(_, value)| { + if !value.is_object() { + None + } else { + let object = value.as_object().unwrap(); + Some( + object + .iter() + .filter_map(|(key, value)| { + if let Ok(schema) = serde_json::from_value::(value.clone()) + { + Some((RefKey::Def(key.clone()), schema)) + } else { + None + } + }) + .collect::>(), + ) + } + }) + .flatten() + .collect::>(); let mut defs = definitions .into_iter() .map(|(key, schema)| (RefKey::Def(key), schema)) + .chain(untracked) .collect::>(); // Does the root type have a name (otherwise... ignore it) - let root_type = schema + let root_type = schema_object .metadata .as_ref() .and_then(|m| m.title.as_ref()) .is_some(); if root_type { - defs.push((RefKey::Root, schema.into())); + defs.push((RefKey::Root, schema_object.into())); + } + + let mut external_references = BTreeMap::new(); + + for (_, def) in &defs { + fetch_external_definitions( + &schema, + def, + &self.file_path, + &s_id, + &mut external_references, + true, + ); + } + + let mut ext_refs = vec![]; + for (_, schema) in defs.iter_mut() { + format_reference(schema, &s_id, &s_id); + } + + for (reference, (mut schema, path, id)) in external_references { + let path = path.canonicalize().unwrap(); + if let RefKey::Def(reference) = reference { + let path = path.canonicalize().unwrap(); + let relpath = diff_paths(&path, self.file_path.parent().unwrap()) + .unwrap_or_default() + .to_string_lossy() + .replace(format!("..{LINE_SEPARATOR}").as_str(), "Parent"); + let ref_name = if relpath.ends_with(LINE_SEPARATOR) { + format!( + "{}{}", + relpath, + reference.split("/").last().unwrap_or_default() + ) + } else { + format!( + "{}{}{}", + relpath, + LINE_SEPARATOR, + reference.split("/").last().unwrap_or_default() + ) + } + .replace(".json", LINE_SEPARATOR.to_string().as_str()) + .trim_matches(LINE_SEPARATOR_CHAR) + .replace( + format!("{LINE_SEPARATOR}{LINE_SEPARATOR}").as_str(), + LINE_SEPARATOR, + ) + .to_string(); + format_reference(&mut schema, &id, &s_id); + ext_refs.push((RefKey::Def(ref_name), schema)); + } + } + + defs.extend(ext_refs.into_iter()); + if self.distinct_definitions { + let mut old = defs.len(); + + distinct_definitions(&mut defs); + while (old - defs.len()) != 0 { + old = defs.len(); + distinct_definitions(&mut defs); + } } self.add_ref_types_impl(defs)?; @@ -1098,6 +1230,492 @@ impl<'a> TypeNewtype<'a> { } } +fn fetch_external_definitions( + base_schema: &RootSchema, + definition: &Schema, + base_path: &PathBuf, + base_id: &Option, + external_references: &mut BTreeMap)>, + first_run: bool, +) { + for mut reference in get_references(&definition) { + if reference.is_empty() { + continue; + } + if reference.starts_with("#") { + if first_run { + continue; + } + + reference.remove(0); + let fragment = reference + .split("/") + .into_iter() + .map(|s| s.to_string()) + .filter(|s| !s.is_empty()) + .collect(); + let definition_schema = fetch_defenition(base_schema, &reference, &fragment); + let k = format!("{}{}", base_id.as_ref().unwrap(), reference); + let key = RefKey::Def(k); + if external_references.contains_key(&key) { + continue; + } else { + external_references.insert( + key, + ( + definition_schema.clone(), + base_path.clone(), + base_id.clone(), + ), + ); + fetch_external_definitions( + base_schema, + &definition_schema, + base_path, + base_id, + external_references, + false, + ); + } + } else { + let base_id = base_id + .as_ref() + .expect("missing 'id' attribute in schema definition"); + let id = Iri::new(base_id).unwrap(); // path + last ref + let reff = Iri::new(&reference).unwrap(); + let fragment = reff + .fragment() + .as_ref() + .unwrap_or(&FragmentBuf::new("".to_string()).unwrap().as_fragment()) + .to_string() + .split("/") + .filter_map(|s| (!s.is_empty()).then_some(s.to_string())) + .collect::>(); + let relpath = + diff_paths(reff.path().as_str(), id.path().parent_or_empty().as_str()).unwrap(); + let file_path = base_path.parent().unwrap().join(&relpath); + let content = std::fs::read_to_string(&file_path).expect(&format!( + "Failed to open input file: {}", + &file_path.display() + )); + + let root_schema = serde_json::from_str::(&content) + .expect("Failed to parse input file as JSON Schema"); + let definition_schema = fetch_defenition(&root_schema, &reference, &fragment); + let key = RefKey::Def(reference.clone()); + if external_references.contains_key(&key) { + continue; + } else { + let s_id = get_schema_id(&root_schema.schema); + + external_references.insert( + key, + (definition_schema.clone(), file_path.clone(), s_id.clone()), + ); + fetch_external_definitions( + &root_schema, + &definition_schema, + &file_path, + &s_id, + external_references, + false, + ) + } + } + } +} + +fn fetch_defenition( + base_schema: &RootSchema, + reference: &String, + fragment: &Vec, +) -> Schema { + if fragment.is_empty() { + return Schema::Object(base_schema.schema.clone()); + } + let definition_schema = if fragment[0] == "definitions" { + base_schema + .definitions + .get( + reference + .split('/') + .last() + .expect("unexpected end of reference"), + ) + .unwrap() + .clone() + } else { + let mut value = base_schema.schema.extensions.get(&fragment[0]).unwrap(); + for x in fragment.iter().skip(1) { + value = value.as_object().unwrap().get(x).unwrap(); + } + serde_json::from_value(value.clone()).unwrap() + }; + definition_schema +} + +// fn get_references(schema: &Schema, base_id: &Option) -> Vec { +fn get_references(schema: &Schema) -> Vec { + match schema { + Schema::Object(obj) => { + let mut result = vec![]; + obj.clone() + .reference + .map(|reference| result.push(reference)); + if let Some(o) = &obj.object { + let prop_refs = o + .properties + .values() + .into_iter() + .flat_map(|p| get_references(p)) + .collect::>(); + result.extend(prop_refs); + if let Some(additional_props) = &o.additional_properties { + result.extend(get_references(&additional_props)); + } + let pattern_refs = o + .pattern_properties + .values() + .into_iter() + .flat_map(|p| get_references(p)) + .collect::>(); + if let Some(property_names) = &o.property_names { + result.extend(get_references(&property_names)) + } + result.extend(pattern_refs); + } + if let Some(o) = &obj.array { + result.extend( + o.contains + .as_ref() + .map(|s| get_references(s.as_ref())) + .unwrap_or_default(), + ); + result.extend( + o.additional_items + .as_ref() + .map(|s| get_references(s.as_ref())) + .unwrap_or_default(), + ); + result.extend( + o.items + .as_ref() + .map(|s| match s { + SingleOrVec::Single(v) => get_references(v.as_ref()), + SingleOrVec::Vec(v) => v + .iter() + .flat_map(|element| get_references(element)) + .collect::>(), + }) + .unwrap_or_default(), + ); + } + if let Some(SubschemaValidation { + all_of, + any_of, + one_of, + not, + if_schema, + then_schema, + else_schema, + }) = obj.subschemas.as_ref().map(AsRef::as_ref) + { + result.extend( + all_of + .as_ref() + .map(|s| { + s.iter() + .flat_map(|element| get_references(element)) + .collect::>() + .into_iter() + }) + .unwrap_or_default(), + ); + result.extend( + any_of + .as_ref() + .map(|s| { + s.iter() + .flat_map(|element| get_references(element)) + .collect::>() + .into_iter() + }) + .unwrap_or_default(), + ); + result.extend( + one_of + .as_ref() + .map(|s| { + s.iter() + .flat_map(|element| get_references(element)) + .collect::>() + .into_iter() + }) + .unwrap_or_default(), + ); + result.extend( + not.as_ref() + .map(|s| get_references(s.as_ref())) + .unwrap_or_default(), + ); + result.extend( + if_schema + .as_ref() + .map(|s| get_references(s.as_ref())) + .unwrap_or_default(), + ); + result.extend( + then_schema + .as_ref() + .map(|s| get_references(s.as_ref())) + .unwrap_or_default(), + ); + result.extend( + else_schema + .as_ref() + .map(|s| get_references(s.as_ref())) + .unwrap_or_default(), + ); + } + result + } + _ => vec![], + } +} + +#[cfg(target_os = "windows")] +const LINE_SEPARATOR: &str = "\\"; + +#[cfg(target_os = "windows")] +const LINE_SEPARATOR_CHAR: char = '\\'; + +#[cfg(not(target_os = "windows"))] +const LINE_SEPARATOR: &str = "/"; + +#[cfg(not(target_os = "windows"))] +const LINE_SEPARATOR_CHAR: char = '/'; + +fn format_reference(schema: &mut Schema, id: &Option, base_id: &Option) { + match schema { + Schema::Bool(_) => {} + Schema::Object(obj) => { + obj.reference.as_mut().map(|reference| { + let mut r = reference.clone(); + if r.starts_with("#") { + if id == base_id { + // dbg!(&reference); + // *reference = reference.split("/").last().unwrap_or_default().to_string(); + return; + } + r = id.clone().unwrap(); + } + let b_id = base_id.clone().unwrap(); + let id = Iri::new(&b_id).unwrap(); // path + last ref + let reff = Iri::new(&r).unwrap(); + let dif = diff_paths(reff.path().as_str(), id.path().parent_or_empty().as_str()) + .unwrap_or_default() + .to_string_lossy() + .replace(format!("..{LINE_SEPARATOR}").as_str(), "Parent"); + + let mut r = format!("{}{}", dif, reference.split("/").last().unwrap_or_default()) + .replace(".json", LINE_SEPARATOR.to_string().as_str()); + if r.ends_with(LINE_SEPARATOR) { + r.pop(); + } + *reference = r; + }); + if let Some(o) = obj.object.as_mut() { + for (_, s) in o.properties.iter_mut() { + format_reference(s, id, base_id); + } + if let Some(additional_props) = o.additional_properties.as_mut() { + format_reference(additional_props, id, base_id); + } + + for (_, s) in o.pattern_properties.iter_mut() { + format_reference(s, id, base_id); + } + if let Some(property_names) = o.property_names.as_mut() { + format_reference(property_names, id, base_id); + } + } + if let Some(o) = obj.array.as_mut() { + if let Some(s) = o.contains.as_mut() { + format_reference(s, id, base_id); + } + if let Some(s) = o.additional_items.as_mut() { + format_reference(s, id, base_id); + } + if let Some(s) = o.items.as_mut() { + match s { + SingleOrVec::Single(s) => format_reference(s, id, base_id), + SingleOrVec::Vec(v) => { + for schema in v.iter_mut() { + format_reference(schema, id, base_id); + } + } + } + } + } + + if let Some(SubschemaValidation { + all_of, + any_of, + one_of, + not, + if_schema, + then_schema, + else_schema, + }) = obj.subschemas.as_mut().map(AsMut::as_mut) + { + if let Some(s) = all_of.as_mut() { + for s in s.iter_mut() { + format_reference(s, id, base_id); + } + } + if let Some(s) = any_of.as_mut() { + for s in s.iter_mut() { + format_reference(s, id, base_id); + } + } + if let Some(s) = one_of.as_mut() { + for s in s.iter_mut() { + format_reference(s, id, base_id); + } + } + if let Some(s) = not.as_mut() { + format_reference(s, id, base_id); + } + if let Some(s) = if_schema.as_mut() { + format_reference(s, id, base_id); + } + if let Some(s) = then_schema.as_mut() { + format_reference(s, id, base_id); + } + if let Some(s) = else_schema.as_mut() { + format_reference(s, id, base_id); + } + } + } + } +} + +fn distinct_definitions(definitions: &mut Vec<(RefKey, Schema)>) -> &mut Vec<(RefKey, Schema)> { + let mut delete_id = std::collections::HashSet::new(); + let mut replace_from_to = BTreeMap::new(); + for i in 0..definitions.len() { + if delete_id.contains(&i) { + continue; + } + for j in (i + 1)..definitions.len() { + if &definitions[i].1 == &definitions[j].1 { + delete_id.insert(j); + if let (RefKey::Def(k), RefKey::Def(key)) = (&definitions[j].0, &definitions[i].0) { + replace_from_to.insert(k.clone(), key.clone()); + } + } + } + } + let mut d = std::mem::take(definitions); + d = d + .into_iter() + .enumerate() + .filter(|(index, _)| !delete_id.contains(index)) + .map(|(_, v)| v) + .collect::>(); + + d.iter_mut() + .for_each(|(_, schema)| replace_reference(schema, &replace_from_to)); + + *definitions = d; + definitions +} + +fn replace_reference(schema: &mut Schema, dictionary: &BTreeMap) { + match schema { + Schema::Bool(_) => {} + Schema::Object(obj) => { + obj.reference.as_mut().map(|reference| { + if let Some(r) = dictionary.get(reference) { + *reference = r.to_string(); + } + }); + if let Some(o) = obj.object.as_mut() { + for (_, s) in o.properties.iter_mut() { + replace_reference(s, dictionary); + } + if let Some(additional_props) = o.additional_properties.as_mut() { + replace_reference(additional_props, dictionary); + } + + for (_, s) in o.pattern_properties.iter_mut() { + replace_reference(s, dictionary); + } + if let Some(property_names) = o.property_names.as_mut() { + replace_reference(property_names, dictionary); + } + } + if let Some(o) = obj.array.as_mut() { + if let Some(s) = o.contains.as_mut() { + replace_reference(s, dictionary); + } + if let Some(s) = o.additional_items.as_mut() { + replace_reference(s, dictionary); + } + if let Some(s) = o.items.as_mut() { + match s { + SingleOrVec::Single(s) => replace_reference(s, dictionary), + SingleOrVec::Vec(v) => { + for schema in v.iter_mut() { + replace_reference(schema, dictionary); + } + } + } + } + } + + if let Some(SubschemaValidation { + all_of, + any_of, + one_of, + not, + if_schema, + then_schema, + else_schema, + }) = obj.subschemas.as_mut().map(AsMut::as_mut) + { + if let Some(s) = all_of.as_mut() { + for s in s.iter_mut() { + replace_reference(s, dictionary); + } + } + if let Some(s) = any_of.as_mut() { + for s in s.iter_mut() { + replace_reference(s, dictionary); + } + } + if let Some(s) = one_of.as_mut() { + for s in s.iter_mut() { + replace_reference(s, dictionary); + } + } + if let Some(s) = not.as_mut() { + replace_reference(s, dictionary); + } + if let Some(s) = if_schema.as_mut() { + replace_reference(s, dictionary); + } + if let Some(s) = then_schema.as_mut() { + replace_reference(s, dictionary); + } + if let Some(s) = else_schema.as_mut() { + replace_reference(s, dictionary); + } + } + } + } +} + #[cfg(test)] mod tests { use schema::Schema; diff --git a/typify-impl/src/merge.rs b/typify-impl/src/merge.rs index 6ad751da..89c2ab88 100644 --- a/typify-impl/src/merge.rs +++ b/typify-impl/src/merge.rs @@ -81,6 +81,7 @@ fn merge_schema(a: &Schema, b: &Schema, defs: &BTreeMap) -> Sche /// incompatible (i.e. if there is no data that can satisfy them both /// simultaneously) then this returns Err. fn try_merge_schema(a: &Schema, b: &Schema, defs: &BTreeMap) -> Result { + // dbg!((a,b)); match (a, b) { (Schema::Bool(false), _) | (_, Schema::Bool(false)) => Err(()), (Schema::Bool(true), other) | (other, Schema::Bool(true)) => Ok(other.clone()), @@ -126,7 +127,6 @@ fn try_merge_schema(a: &Schema, b: &Schema, defs: &BTreeMap) -> .get(&key) .unwrap_or_else(|| panic!("unresolved reference: {}", ref_name)); let merged_schema = try_merge_schema(resolved, other, defs)?; - // If we merge a referenced schema with another schema **and** // the resulting schema is equivalent to the referenced schema // (i.e. the other schema is identical or less permissive) then we diff --git a/typify-impl/src/util.rs b/typify-impl/src/util.rs index 4b92d685..49795261 100644 --- a/typify-impl/src/util.rs +++ b/typify-impl/src/util.rs @@ -531,9 +531,13 @@ pub(crate) fn ref_key(ref_name: &str) -> RefKey { if ref_name == "#" { RefKey::Root } else if let Some(idx) = ref_name.rfind('/') { - RefKey::Def(ref_name[idx + 1..].to_string()) + if ref_name.starts_with("#") { + RefKey::Def(ref_name[idx + 1..].to_string()) + } else { + RefKey::Def(ref_name.to_string()) + } } else { - panic!("expected a '/' in $ref: {}", ref_name) + RefKey::Def(ref_name.to_string()) } } From 0e1916125c162d814027602f1147b85b4f418dec Mon Sep 17 00:00:00 2001 From: Petro Shvayko Date: Mon, 1 Jul 2024 16:50:59 +0300 Subject: [PATCH 2/8] add comments --- typify-impl/src/lib.rs | 109 +++++++++++++++++++++++++++-------------- 1 file changed, 71 insertions(+), 38 deletions(-) diff --git a/typify-impl/src/lib.rs b/typify-impl/src/lib.rs index 4ee3ad08..964ecb66 100644 --- a/typify-impl/src/lib.rs +++ b/typify-impl/src/lib.rs @@ -736,6 +736,7 @@ impl TypeSpace { let s_id = get_schema_id(&schema_object); + // handle definitions from extensions. let untracked = schema_object .extensions .clone() @@ -780,6 +781,7 @@ impl TypeSpace { defs.push((RefKey::Root, schema_object.into())); } + // recursively fetch external references from definitions let mut external_references = BTreeMap::new(); for (_, def) in &defs { @@ -794,10 +796,12 @@ impl TypeSpace { } let mut ext_refs = vec![]; + // format references in internal schemas to prevent collisions in schemas for (_, schema) in defs.iter_mut() { format_reference(schema, &s_id, &s_id); } + // format references in external schemas to prevent collisions in schemas for (reference, (mut schema, path, id)) in external_references { let path = path.canonicalize().unwrap(); if let RefKey::Def(reference) = reference { @@ -832,8 +836,35 @@ impl TypeSpace { } } + // merge internal and external schemas defs.extend(ext_refs.into_iter()); + if self.distinct_definitions { + // recursevely distinct definition to strip count of definitions + // for example: + // ... + // "foo":{ + // "$ref": "#/definitions/a", + // }, + // "bar":{ + // "$ref": "#/definitions/b", + // }, + // "a": { + // "type": "string" + // }, + // "b": { + // "type": "string" + // }, + // ⬇️ + // "foo":{ + // "$ref": "#/definitions/a", + // }, + // "bar":{ + // "$ref": "#/definitions/a", + // }, + // "a": { + // "type": "string" + // } let mut old = defs.len(); distinct_definitions(&mut defs); @@ -1231,35 +1262,38 @@ impl<'a> TypeNewtype<'a> { } fn fetch_external_definitions( - base_schema: &RootSchema, - definition: &Schema, - base_path: &PathBuf, - base_id: &Option, - external_references: &mut BTreeMap)>, - first_run: bool, + base_schema: &RootSchema, // Reference to the base schema + definition: &Schema, // The schema definition to process + base_path: &PathBuf, // Base path for file operations + base_id: &Option, // Optional base ID for schema + external_references: &mut BTreeMap)>, // Map to store external references + first_run: bool, // Flag to indicate if this is the first run of the function ) { + // Iterate through each reference found in the given schema definition for mut reference in get_references(&definition) { if reference.is_empty() { - continue; + continue; // Skip empty references } if reference.starts_with("#") { + // Handle internal references if first_run { - continue; + continue; // Skip processing internal references on the first run } - reference.remove(0); + reference.remove(0); // Remove the '#' character from the reference let fragment = reference - .split("/") - .into_iter() - .map(|s| s.to_string()) - .filter(|s| !s.is_empty()) - .collect(); - let definition_schema = fetch_defenition(base_schema, &reference, &fragment); - let k = format!("{}{}", base_id.as_ref().unwrap(), reference); + .split("/") + .into_iter() + .map(|s| s.to_string()) + .filter(|s| !s.is_empty()) + .collect(); // Split and collect the reference into a vector of strings + let definition_schema = fetch_defenition(base_schema, &reference, &fragment); // Fetch the internal schema definition + let k = format!("{}{}", base_id.as_ref().unwrap(), reference); // Create a key for the reference let key = RefKey::Def(k); if external_references.contains_key(&key) { - continue; + continue; // Skip if the reference already exists in the map } else { + // Insert the reference into the map and recursively fetch external definitions external_references.insert( key, ( @@ -1278,36 +1312,38 @@ fn fetch_external_definitions( ); } } else { + // Handle external references let base_id = base_id - .as_ref() - .expect("missing 'id' attribute in schema definition"); - let id = Iri::new(base_id).unwrap(); // path + last ref - let reff = Iri::new(&reference).unwrap(); + .as_ref() + .expect("missing 'id' attribute in schema definition"); // Ensure base_id is present + let id = Iri::new(base_id).unwrap(); // Create an IRI from the base ID + let reff = Iri::new(&reference).unwrap(); // Create an IRI from the reference let fragment = reff - .fragment() - .as_ref() - .unwrap_or(&FragmentBuf::new("".to_string()).unwrap().as_fragment()) - .to_string() - .split("/") - .filter_map(|s| (!s.is_empty()).then_some(s.to_string())) - .collect::>(); + .fragment() + .as_ref() + .unwrap_or(&FragmentBuf::new("".to_string()).unwrap().as_fragment()) + .to_string() + .split("/") + .filter_map(|s| (!s.is_empty()).then_some(s.to_string())) + .collect::>(); // Process the fragment part of the reference let relpath = - diff_paths(reff.path().as_str(), id.path().parent_or_empty().as_str()).unwrap(); - let file_path = base_path.parent().unwrap().join(&relpath); + diff_paths(reff.path().as_str(), id.path().parent_or_empty().as_str()).unwrap(); // Determine the relative path + let file_path = base_path.parent().unwrap().join(&relpath); // Construct the file path let content = std::fs::read_to_string(&file_path).expect(&format!( "Failed to open input file: {}", &file_path.display() - )); + )); // Read the file content let root_schema = serde_json::from_str::(&content) - .expect("Failed to parse input file as JSON Schema"); - let definition_schema = fetch_defenition(&root_schema, &reference, &fragment); + .expect("Failed to parse input file as JSON Schema"); // Parse the file content as JSON Schema + let definition_schema = fetch_defenition(&root_schema, &reference, &fragment); // Fetch the external schema definition let key = RefKey::Def(reference.clone()); if external_references.contains_key(&key) { - continue; + continue; // Skip if the reference already exists in the map } else { - let s_id = get_schema_id(&root_schema.schema); + let s_id = get_schema_id(&root_schema.schema); // Get the schema ID + // Insert the reference into the map and recursively fetch external definitions external_references.insert( key, (definition_schema.clone(), file_path.clone(), s_id.clone()), @@ -1354,7 +1390,6 @@ fn fetch_defenition( definition_schema } -// fn get_references(schema: &Schema, base_id: &Option) -> Vec { fn get_references(schema: &Schema) -> Vec { match schema { Schema::Object(obj) => { @@ -1503,8 +1538,6 @@ fn format_reference(schema: &mut Schema, id: &Option, base_id: &Option Date: Mon, 1 Jul 2024 16:22:15 +0300 Subject: [PATCH 3/8] Delete .idea directory --- .idea/.gitignore | 8 -------- .idea/modules.xml | 8 -------- .idea/typify.iml | 20 -------------------- .idea/vcs.xml | 6 ------ 4 files changed, 42 deletions(-) delete mode 100644 .idea/.gitignore delete mode 100644 .idea/modules.xml delete mode 100644 .idea/typify.iml delete mode 100644 .idea/vcs.xml diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index 13566b81..00000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml -# Editor-based HTTP Client requests -/httpRequests/ -# Datasource local storage ignored files -/dataSources/ -/dataSources.local.xml diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index a97b4fec..00000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/typify.iml b/.idea/typify.iml deleted file mode 100644 index c1686e0c..00000000 --- a/.idea/typify.iml +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 35eb1ddf..00000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file From 0be83f2f02e6bf0c9bb004e53086e50027cac134 Mon Sep 17 00:00:00 2001 From: Petro Shvayko Date: Mon, 1 Jul 2024 17:03:00 +0300 Subject: [PATCH 4/8] add test & comments --- typify-impl/src/convert.rs | 2 + typify-impl/src/merge.rs | 1 - typify-impl/tests/external_references.json | 13 ++++ typify-impl/tests/external_references.out | 77 +++++++++++++++++++ typify-impl/tests/test_external_references.rs | 26 +++++++ typify-impl/tests/v1/external_reference.json | 17 ++++ 6 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 typify-impl/tests/external_references.json create mode 100644 typify-impl/tests/external_references.out create mode 100644 typify-impl/tests/test_external_references.rs create mode 100644 typify-impl/tests/v1/external_reference.json diff --git a/typify-impl/src/convert.rs b/typify-impl/src/convert.rs index 203b15e5..7b0ebbb8 100644 --- a/typify-impl/src/convert.rs +++ b/typify-impl/src/convert.rs @@ -1031,6 +1031,8 @@ impl TypeSpace { // f64 here, but we're already constrained by the schemars // representation so ... it's probably the best we can do at // the moment. + // + // I added this because numbers are sometimes specified in double quotes let d = match default { serde_json::Value::Number(a) => a.as_f64(), serde_json::Value::String(a) => a.parse().ok(), diff --git a/typify-impl/src/merge.rs b/typify-impl/src/merge.rs index 89c2ab88..608be4e7 100644 --- a/typify-impl/src/merge.rs +++ b/typify-impl/src/merge.rs @@ -81,7 +81,6 @@ fn merge_schema(a: &Schema, b: &Schema, defs: &BTreeMap) -> Sche /// incompatible (i.e. if there is no data that can satisfy them both /// simultaneously) then this returns Err. fn try_merge_schema(a: &Schema, b: &Schema, defs: &BTreeMap) -> Result { - // dbg!((a,b)); match (a, b) { (Schema::Bool(false), _) | (_, Schema::Bool(false)) => Err(()), (Schema::Bool(true), other) | (other, Schema::Bool(true)) => Ok(other.clone()), diff --git a/typify-impl/tests/external_references.json b/typify-impl/tests/external_references.json new file mode 100644 index 00000000..466bcc7a --- /dev/null +++ b/typify-impl/tests/external_references.json @@ -0,0 +1,13 @@ +{ + "id": "https://schemas.mysite.com/schemas/external_references.json#", + "$schema": "http://json-schema.org/draft-04/schema#", + "definitions": { + "somename": { + "properties": { + "someproperty": { + "$ref": "https://schemas.mysite.com/schemas/v1/external_reference.json#/definitions/someothername" + } + } + } + } +} \ No newline at end of file diff --git a/typify-impl/tests/external_references.out b/typify-impl/tests/external_references.out new file mode 100644 index 00000000..00baa3de --- /dev/null +++ b/typify-impl/tests/external_references.out @@ -0,0 +1,77 @@ +#[doc = r" Error types."] +pub mod error { + #[doc = r" Error from a TryFrom or FromStr implementation."] + pub struct ConversionError(std::borrow::Cow<'static, str>); + impl std::error::Error for ConversionError {} + impl std::fmt::Display for ConversionError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { + std::fmt::Display::fmt(&self.0, f) + } + } + impl std::fmt::Debug for ConversionError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { + std::fmt::Debug::fmt(&self.0, f) + } + } + impl From<&'static str> for ConversionError { + fn from(value: &'static str) -> Self { + Self(value.into()) + } + } + impl From for ConversionError { + fn from(value: String) -> Self { + Self(value.into()) + } + } +} +#[doc = "Somename"] +#[doc = r""] +#[doc = r"
JSON schema"] +#[doc = r""] +#[doc = r" ```json"] +#[doc = "{"] +#[doc = " \"properties\": {"] +#[doc = " \"someproperty\": {"] +#[doc = " \"$ref\": \"v1/external_reference/someothername\""] +#[doc = " }"] +#[doc = " }"] +#[doc = "}"] +#[doc = r" ```"] +#[doc = r"
"] +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct Somename { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub someproperty: Option, +} +impl From<&Somename> for Somename { + fn from(value: &Somename) -> Self { + value.clone() + } +} +#[doc = "V1ExternalReferenceSomeothername"] +#[doc = r""] +#[doc = r"
JSON schema"] +#[doc = r""] +#[doc = r" ```json"] +#[doc = "{"] +#[doc = " \"type\": \"object\","] +#[doc = " \"properties\": {"] +#[doc = " \"someproperty\": {"] +#[doc = " \"a\": {"] +#[doc = " \"type\": \"string\""] +#[doc = " }"] +#[doc = " }"] +#[doc = " }"] +#[doc = "}"] +#[doc = r" ```"] +#[doc = r"
"] +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct V1ExternalReferenceSomeothername { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub someproperty: Option, +} +impl From<&V1ExternalReferenceSomeothername> for V1ExternalReferenceSomeothername { + fn from(value: &V1ExternalReferenceSomeothername) -> Self { + value.clone() + } +} diff --git a/typify-impl/tests/test_external_references.rs b/typify-impl/tests/test_external_references.rs new file mode 100644 index 00000000..17b13dab --- /dev/null +++ b/typify-impl/tests/test_external_references.rs @@ -0,0 +1,26 @@ +use std::fs::File; +use std::io::BufReader; +use std::path::Path; +use schemars::schema::RootSchema; +use typify_impl::TypeSpace; + +#[test] +fn test_external_references() { + let mut type_space = TypeSpace::default(); + + let path = Path::new("tests/external_references.json"); + let file = File::open(path).unwrap(); + let reader = BufReader::new(file); + + let schema: RootSchema = serde_json::from_reader(reader).unwrap(); + // schema.schema.metadata().title = Some("Everything".to_string()); + type_space.with_path("tests/external_references.json"); + type_space.add_root_schema(schema).unwrap(); + + let file = type_space.to_stream(); + + let fmt = rustfmt_wrapper::rustfmt(file.to_string()).unwrap(); + println!("{fmt}"); + + expectorate::assert_contents("tests/external_references.out", fmt.as_str()); +} diff --git a/typify-impl/tests/v1/external_reference.json b/typify-impl/tests/v1/external_reference.json new file mode 100644 index 00000000..510e2f87 --- /dev/null +++ b/typify-impl/tests/v1/external_reference.json @@ -0,0 +1,17 @@ +{ + "id": "https://schemas.mysite.com/schemas/v1/external_reference.json#", + "$schema": "http://json-schema.org/draft-04/schema#", + "definitions": { + "someothername": { + "type": "object", + "properties": { + "someproperty": { + "a":{ + "type": "string" + } + } + + } + } + } +} From 074d39734ecfe30d154bf1cb8be9b9cb2fa45294 Mon Sep 17 00:00:00 2001 From: Petro Shvayko Date: Mon, 1 Jul 2024 17:56:43 +0300 Subject: [PATCH 5/8] fmt --- typify-impl/src/lib.rs | 71 +++++++++---------- typify-impl/tests/test_external_references.rs | 26 +++---- 2 files changed, 48 insertions(+), 49 deletions(-) diff --git a/typify-impl/src/lib.rs b/typify-impl/src/lib.rs index 964ecb66..7f9b9569 100644 --- a/typify-impl/src/lib.rs +++ b/typify-impl/src/lib.rs @@ -220,7 +220,6 @@ pub struct TypeSpace { } impl TypeSpace { - /// Sets the file path for the `TypeSpace` instance. pub fn with_path>(&mut self, path: T) { self.file_path = path.into().canonicalize().unwrap(); @@ -1262,36 +1261,36 @@ impl<'a> TypeNewtype<'a> { } fn fetch_external_definitions( - base_schema: &RootSchema, // Reference to the base schema - definition: &Schema, // The schema definition to process - base_path: &PathBuf, // Base path for file operations - base_id: &Option, // Optional base ID for schema - external_references: &mut BTreeMap)>, // Map to store external references - first_run: bool, // Flag to indicate if this is the first run of the function + base_schema: &RootSchema, // Reference to the base schema + definition: &Schema, // The schema definition to process + base_path: &PathBuf, // Base path for file operations + base_id: &Option, // Optional base ID for schema + external_references: &mut BTreeMap)>, // Map to store external references + first_run: bool, // Flag to indicate if this is the first run of the function ) { // Iterate through each reference found in the given schema definition for mut reference in get_references(&definition) { if reference.is_empty() { - continue; // Skip empty references + continue; // Skip empty references } if reference.starts_with("#") { // Handle internal references if first_run { - continue; // Skip processing internal references on the first run + continue; // Skip processing internal references on the first run } - reference.remove(0); // Remove the '#' character from the reference + reference.remove(0); // Remove the '#' character from the reference let fragment = reference - .split("/") - .into_iter() - .map(|s| s.to_string()) - .filter(|s| !s.is_empty()) - .collect(); // Split and collect the reference into a vector of strings - let definition_schema = fetch_defenition(base_schema, &reference, &fragment); // Fetch the internal schema definition - let k = format!("{}{}", base_id.as_ref().unwrap(), reference); // Create a key for the reference + .split("/") + .into_iter() + .map(|s| s.to_string()) + .filter(|s| !s.is_empty()) + .collect(); // Split and collect the reference into a vector of strings + let definition_schema = fetch_defenition(base_schema, &reference, &fragment); // Fetch the internal schema definition + let k = format!("{}{}", base_id.as_ref().unwrap(), reference); // Create a key for the reference let key = RefKey::Def(k); if external_references.contains_key(&key) { - continue; // Skip if the reference already exists in the map + continue; // Skip if the reference already exists in the map } else { // Insert the reference into the map and recursively fetch external definitions external_references.insert( @@ -1314,34 +1313,34 @@ fn fetch_external_definitions( } else { // Handle external references let base_id = base_id - .as_ref() - .expect("missing 'id' attribute in schema definition"); // Ensure base_id is present - let id = Iri::new(base_id).unwrap(); // Create an IRI from the base ID - let reff = Iri::new(&reference).unwrap(); // Create an IRI from the reference + .as_ref() + .expect("missing 'id' attribute in schema definition"); // Ensure base_id is present + let id = Iri::new(base_id).unwrap(); // Create an IRI from the base ID + let reff = Iri::new(&reference).unwrap(); // Create an IRI from the reference let fragment = reff - .fragment() - .as_ref() - .unwrap_or(&FragmentBuf::new("".to_string()).unwrap().as_fragment()) - .to_string() - .split("/") - .filter_map(|s| (!s.is_empty()).then_some(s.to_string())) - .collect::>(); // Process the fragment part of the reference + .fragment() + .as_ref() + .unwrap_or(&FragmentBuf::new("".to_string()).unwrap().as_fragment()) + .to_string() + .split("/") + .filter_map(|s| (!s.is_empty()).then_some(s.to_string())) + .collect::>(); // Process the fragment part of the reference let relpath = - diff_paths(reff.path().as_str(), id.path().parent_or_empty().as_str()).unwrap(); // Determine the relative path - let file_path = base_path.parent().unwrap().join(&relpath); // Construct the file path + diff_paths(reff.path().as_str(), id.path().parent_or_empty().as_str()).unwrap(); // Determine the relative path + let file_path = base_path.parent().unwrap().join(&relpath); // Construct the file path let content = std::fs::read_to_string(&file_path).expect(&format!( "Failed to open input file: {}", &file_path.display() - )); // Read the file content + )); // Read the file content let root_schema = serde_json::from_str::(&content) - .expect("Failed to parse input file as JSON Schema"); // Parse the file content as JSON Schema - let definition_schema = fetch_defenition(&root_schema, &reference, &fragment); // Fetch the external schema definition + .expect("Failed to parse input file as JSON Schema"); // Parse the file content as JSON Schema + let definition_schema = fetch_defenition(&root_schema, &reference, &fragment); // Fetch the external schema definition let key = RefKey::Def(reference.clone()); if external_references.contains_key(&key) { - continue; // Skip if the reference already exists in the map + continue; // Skip if the reference already exists in the map } else { - let s_id = get_schema_id(&root_schema.schema); // Get the schema ID + let s_id = get_schema_id(&root_schema.schema); // Get the schema ID // Insert the reference into the map and recursively fetch external definitions external_references.insert( diff --git a/typify-impl/tests/test_external_references.rs b/typify-impl/tests/test_external_references.rs index 17b13dab..576019cd 100644 --- a/typify-impl/tests/test_external_references.rs +++ b/typify-impl/tests/test_external_references.rs @@ -1,26 +1,26 @@ +use schemars::schema::RootSchema; use std::fs::File; use std::io::BufReader; use std::path::Path; -use schemars::schema::RootSchema; use typify_impl::TypeSpace; #[test] fn test_external_references() { - let mut type_space = TypeSpace::default(); + let mut type_space = TypeSpace::default(); - let path = Path::new("tests/external_references.json"); - let file = File::open(path).unwrap(); - let reader = BufReader::new(file); + let path = Path::new("tests/external_references.json"); + let file = File::open(path).unwrap(); + let reader = BufReader::new(file); - let schema: RootSchema = serde_json::from_reader(reader).unwrap(); - // schema.schema.metadata().title = Some("Everything".to_string()); - type_space.with_path("tests/external_references.json"); - type_space.add_root_schema(schema).unwrap(); + let schema: RootSchema = serde_json::from_reader(reader).unwrap(); + // schema.schema.metadata().title = Some("Everything".to_string()); + type_space.with_path("tests/external_references.json"); + type_space.add_root_schema(schema).unwrap(); - let file = type_space.to_stream(); + let file = type_space.to_stream(); - let fmt = rustfmt_wrapper::rustfmt(file.to_string()).unwrap(); - println!("{fmt}"); + let fmt = rustfmt_wrapper::rustfmt(file.to_string()).unwrap(); + println!("{fmt}"); - expectorate::assert_contents("tests/external_references.out", fmt.as_str()); + expectorate::assert_contents("tests/external_references.out", fmt.as_str()); } From ded5cae9f0638412474f9bbc821046265b9e5e16 Mon Sep 17 00:00:00 2001 From: SRetip Date: Wed, 3 Jul 2024 09:37:04 +0300 Subject: [PATCH 6/8] fix tests for windows --- typify-impl/src/lib.rs | 4 ++++ typify-impl/tests/test_external_references.rs | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/typify-impl/src/lib.rs b/typify-impl/src/lib.rs index 7f9b9569..ecc25e13 100644 --- a/typify-impl/src/lib.rs +++ b/typify-impl/src/lib.rs @@ -831,6 +831,8 @@ impl TypeSpace { ) .to_string(); format_reference(&mut schema, &id, &s_id); + #[cfg(target_os= "windows")] + let ref_name = ref_name.replace("\\", "/"); ext_refs.push((RefKey::Def(ref_name), schema)); } } @@ -1554,6 +1556,8 @@ fn format_reference(schema: &mut Schema, id: &Option, base_id: &Option Date: Tue, 6 Aug 2024 09:04:35 +0300 Subject: [PATCH 7/8] sort dependencies --- typify-impl/Cargo.toml | 4 ++-- typify-impl/tests/test_external_references.rs | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/typify-impl/Cargo.toml b/typify-impl/Cargo.toml index bee7e47c..a8ce7884 100644 --- a/typify-impl/Cargo.toml +++ b/typify-impl/Cargo.toml @@ -9,7 +9,9 @@ readme = "../README.md" [dependencies] heck = "0.5.0" +iref = "3.1.4" log = "0.4.22" +pathdiff = "0.2.1" proc-macro2 = "1.0.86" quote = "1.0.36" regress = "0.10.0" @@ -20,8 +22,6 @@ serde_json = "1.0.119" syn = { version = "2.0.68", features = ["full"] } thiserror = "1.0.61" unicode-ident = "1.0.12" -pathdiff = "0.2.1" -iref = "3.1.4" [dev-dependencies] env_logger = "0.10.2" diff --git a/typify-impl/tests/test_external_references.rs b/typify-impl/tests/test_external_references.rs index 576019cd..2a5db55d 100644 --- a/typify-impl/tests/test_external_references.rs +++ b/typify-impl/tests/test_external_references.rs @@ -13,7 +13,6 @@ fn test_external_references() { let reader = BufReader::new(file); let schema: RootSchema = serde_json::from_reader(reader).unwrap(); - // schema.schema.metadata().title = Some("Everything".to_string()); type_space.with_path("tests/external_references.json"); type_space.add_root_schema(schema).unwrap(); From 996106de17b3883f73f9a3b296b6a75b37244836 Mon Sep 17 00:00:00 2001 From: Petro Shvayko Date: Tue, 6 Aug 2024 11:07:17 +0300 Subject: [PATCH 8/8] fmt --- typify-impl/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/typify-impl/src/lib.rs b/typify-impl/src/lib.rs index ecc25e13..5f5d9e33 100644 --- a/typify-impl/src/lib.rs +++ b/typify-impl/src/lib.rs @@ -831,7 +831,7 @@ impl TypeSpace { ) .to_string(); format_reference(&mut schema, &id, &s_id); - #[cfg(target_os= "windows")] + #[cfg(target_os = "windows")] let ref_name = ref_name.replace("\\", "/"); ext_refs.push((RefKey::Def(ref_name), schema)); } @@ -1556,7 +1556,7 @@ fn format_reference(schema: &mut Schema, id: &Option, base_id: &Option