From 0a5b4b15a4bf34e8bbe4a38d25d085d76325d2cd Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Thu, 6 Jun 2024 20:07:10 +0100 Subject: [PATCH 1/9] [red-knot] Include vendored typeshed stubs as a zipfile in the Ruff binary --- .github/renovate.json5 | 7 + .gitignore | 3 + Cargo.lock | 289 ++++++++++++++++++++++++++++++++++ Cargo.toml | 1 + crates/red_knot/Cargo.toml | 5 + crates/red_knot/build.rs | 90 +++++++++++ crates/red_knot/src/module.rs | 29 +++- 7 files changed, 423 insertions(+), 1 deletion(-) create mode 100644 crates/red_knot/build.rs diff --git a/.github/renovate.json5 b/.github/renovate.json5 index 259ab5c1d9509..33398b0b7e83c 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -41,6 +41,13 @@ description: "Disable PRs updating GitHub runners (e.g. 'runs-on: macos-14')", enabled: false, }, + { + // Disable updates of `zip-rs`; intentionally pinned for now due to ownership change + // See: https://github.com/astral-sh/uv/issues/3642 + matchPackagePatterns: ["zip"], + matchManagers: ["cargo"], + enabled: false, + }, { groupName: "pre-commit dependencies", matchManagers: ["pre-commit"], diff --git a/.gitignore b/.gitignore index 4302ff30a762a..9443c92091188 100644 --- a/.gitignore +++ b/.gitignore @@ -215,3 +215,6 @@ cython_debug/ !crates/ruff_python_resolver/resources/test/airflow/venv/lib !crates/ruff_python_resolver/resources/test/airflow/venv/lib/python3.11/site-packages/_watchdog_fsevents.cpython-311-darwin.so !crates/ruff_python_resolver/resources/test/airflow/venv/lib/python3.11/site-packages/orjson/orjson.cpython-311-darwin.so + +# Ignore the zipped version of typeshed that we create at build time +crates/red_knot/vendor/zipped_typeshed.zip diff --git a/Cargo.lock b/Cargo.lock index 8a974b12ed39d..f657a65b34c36 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,6 +14,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + [[package]] name = "ahash" version = "0.8.11" @@ -161,6 +172,12 @@ version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51" +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" + [[package]] name = "bincode" version = "1.3.3" @@ -182,6 +199,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bstr" version = "1.9.1" @@ -199,6 +225,33 @@ version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "cachedir" version = "0.3.1" @@ -219,6 +272,11 @@ name = "cc" version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d32a725bc159af97c3e629873bb9f88fb8cf8a4867175f76dc987815ea07c83b" +dependencies = [ + "jobserver", + "libc", + "once_cell", +] [[package]] name = "cfg-if" @@ -280,6 +338,16 @@ dependencies = [ "half", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "clap" version = "4.5.4" @@ -446,6 +514,12 @@ dependencies = [ "web-sys", ] +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + [[package]] name = "core-foundation-sys" version = "0.8.6" @@ -458,6 +532,15 @@ version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7704b5fdd17b18ae31c4c1da5a2e0305a2bf17b5249300a9ee9ed7b72114c636" +[[package]] +name = "cpufeatures" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.4.0" @@ -563,6 +646,16 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "ctrlc" version = "3.4.4" @@ -621,12 +714,32 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", +] + [[package]] name = "diff" version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + [[package]] name = "dirs" version = "4.0.0" @@ -801,6 +914,16 @@ dependencies = [ "libc", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getopts" version = "0.2.21" @@ -880,6 +1003,15 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "home" version = "0.5.9" @@ -1021,6 +1153,15 @@ dependencies = [ "libc", ] +[[package]] +name = "inout" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" +dependencies = [ + "generic-array", +] + [[package]] name = "insta" version = "1.39.0" @@ -1123,6 +1264,15 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +[[package]] +name = "jobserver" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" +dependencies = [ + "libc", +] + [[package]] name = "jod-thread" version = "0.1.2" @@ -1405,6 +1555,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-traits" version = "0.2.18" @@ -1476,6 +1632,17 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "password-hash" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" +dependencies = [ + "base64ct", + "rand_core", + "subtle", +] + [[package]] name = "paste" version = "1.0.14" @@ -1512,6 +1679,18 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" +[[package]] +name = "pbkdf2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" +dependencies = [ + "digest", + "hmac", + "password-hash", + "sha2", +] + [[package]] name = "peg" version = "0.8.2" @@ -1629,12 +1808,24 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" +[[package]] +name = "pkg-config" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" + [[package]] name = "portable-atomic" version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -1782,6 +1973,8 @@ dependencies = [ "tracing", "tracing-subscriber", "tracing-tree", + "walkdir", + "zip", ] [[package]] @@ -2620,6 +2813,28 @@ dependencies = [ "syn", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -2855,6 +3070,25 @@ dependencies = [ "tikv-jemalloc-sys", ] +[[package]] +name = "time" +version = "0.3.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde", + "time-core", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + [[package]] name = "tinytemplate" version = "1.2.1" @@ -3006,6 +3240,12 @@ version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + [[package]] name = "unic-char-property" version = "0.9.0" @@ -3593,3 +3833,52 @@ name = "zeroize" version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" + +[[package]] +name = "zip" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" +dependencies = [ + "aes", + "byteorder", + "bzip2", + "constant_time_eq", + "crc32fast", + "crossbeam-utils", + "flate2", + "hmac", + "pbkdf2", + "sha1", + "time", + "zstd", +] + +[[package]] +name = "zstd" +version = "0.11.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "5.0.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.10+zstd.1.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index 841d2b17e39ef..5d72b5420a511 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -136,6 +136,7 @@ walkdir = { version = "2.3.2" } wasm-bindgen = { version = "0.2.92" } wasm-bindgen-test = { version = "0.3.42" } wild = { version = "2" } +zip = { version = "0.6.6" } [workspace.lints.rust] unsafe_code = "warn" diff --git a/crates/red_knot/Cargo.toml b/crates/red_knot/Cargo.toml index 3099b755b08e7..b6ccc856c6e5f 100644 --- a/crates/red_knot/Cargo.toml +++ b/crates/red_knot/Cargo.toml @@ -34,6 +34,11 @@ smol_str = { version = "0.2.1" } tracing = { workspace = true } tracing-subscriber = { workspace = true } tracing-tree = { workspace = true } +zip = { workspace = true } + +[build-dependencies] +zip = { workspace = true } +walkdir = { workspace = true } [dev-dependencies] tempfile = { workspace = true } diff --git a/crates/red_knot/build.rs b/crates/red_knot/build.rs new file mode 100644 index 0000000000000..1ddf0aa67a912 --- /dev/null +++ b/crates/red_knot/build.rs @@ -0,0 +1,90 @@ +//! Build script to package our vendored typeshed files +//! into a zip archive that can be included in the Ruff binary. +//! +//! This script should be automatically run at build time +//! whenever the script itself changes, or whenever any files +//! in `crates/red_knot/vendor/typeshed` change. + +use std::error::Error; +use std::fs::File; +use std::io::{Read, Write}; +use std::path::Path; + +use walkdir::{DirEntry, WalkDir}; +use zip::result::ZipResult; +use zip::write::{FileOptions, ZipWriter}; +use zip::CompressionMethod; + +const TYPESHED_SOURCE_DIR: &str = "vendor/typeshed"; + +// NB: This is .gitignored; make sure to change the .gitignore entry +// if you change this location +const TYPESHED_ZIP_LOCATION: &str = "vendor/zipped_typeshed.zip"; + +/// Recursively zip the contents of an entire directory. +/// +/// This routine is adapted from a recipe at +/// +fn zip_dir( + directory_iterator: &mut impl Iterator, + prefix: &str, + writer: File, +) -> ZipResult<()> { + let mut zip = ZipWriter::new(writer); + + let options = FileOptions::default() + .compression_method(CompressionMethod::Zstd) + .unix_permissions(0o644); + + let mut buffer = Vec::new(); + for entry in directory_iterator { + let path = entry.path(); + let name = path + .strip_prefix(Path::new(prefix)) + .unwrap() + .to_str() + .expect("Unexpected non-utf8 typeshed path!"); + + // Write file or directory explicitly + // Some unzip tools unzip files with directory paths correctly, some do not! + if path.is_file() { + println!("adding file {path:?} as {name:?} ..."); + zip.start_file(name, options)?; + let mut f = File::open(path)?; + + f.read_to_end(&mut buffer)?; + zip.write_all(&buffer)?; + buffer.clear(); + } else if !name.is_empty() { + // Only if not root! Avoids path spec / warning + // and mapname conversion failed error on unzip + println!("adding dir {path:?} as {name:?} ..."); + + zip.add_directory(name, options)?; + } + } + zip.finish()?; + Result::Ok(()) +} + +fn main() -> Result<(), Box> { + println!("cargo:rerun-if-changed={TYPESHED_SOURCE_DIR}"); + + assert!( + Path::new(TYPESHED_SOURCE_DIR).is_dir(), + "Where is typeshed?" + ); + + let zipped_typeshed = File::create(Path::new(TYPESHED_ZIP_LOCATION)).unwrap(); + + let mut typeshed_traverser = WalkDir::new(TYPESHED_SOURCE_DIR) + .into_iter() + .filter_map(std::result::Result::ok); + + zip_dir( + &mut typeshed_traverser, + TYPESHED_SOURCE_DIR, + zipped_typeshed, + )?; + Ok(()) +} diff --git a/crates/red_knot/src/module.rs b/crates/red_knot/src/module.rs index f3386cc02744b..a1cf312f32ca9 100644 --- a/crates/red_knot/src/module.rs +++ b/crates/red_knot/src/module.rs @@ -12,6 +12,13 @@ use crate::files::FileId; use crate::semantic::Dependency; use crate::FxDashMap; +// The file path here is hardcoded in this crate's `build.rs` script. +// Luckily this crate will fail to build if this file isn't available at build time. +// +// Allow dead code for now; actually resolving modules to vendored typeshed stubs will come soon! +#[allow(dead_code)] +const TYPESHED_ZIP_BYTES: &[u8] = include_bytes!("../vendor/zipped_typeshed.zip"); + /// Representation of a Python module. /// /// The inner type wrapped by this struct is a unique identifier for the module @@ -770,14 +777,17 @@ impl PackageKind { #[cfg(test)] mod tests { + use std::io::{Cursor, Read}; use std::num::NonZeroU32; use std::path::PathBuf; + use zip::ZipArchive; + use crate::db::tests::TestDb; use crate::db::SourceDb; use crate::module::{ path_to_module, resolve_module, set_module_search_paths, ModuleKind, ModuleName, - ModuleResolutionInputs, TYPESHED_STDLIB_DIRECTORY, + ModuleResolutionInputs, TYPESHED_STDLIB_DIRECTORY, TYPESHED_ZIP_BYTES, }; use crate::semantic::Dependency; @@ -923,6 +933,23 @@ mod tests { Ok(()) } + #[test] + fn typeshed_zip_created_at_build_time() -> anyhow::Result<()> { + assert!(!TYPESHED_ZIP_BYTES.is_empty()); + let mut typeshed_zip_archive = ZipArchive::new(Cursor::new(TYPESHED_ZIP_BYTES))?; + + let mut functools_module_stub = typeshed_zip_archive + .by_name("stdlib/functools.pyi") + .unwrap(); + assert!(functools_module_stub.is_file()); + + let mut functools_module_stub_source = String::new(); + functools_module_stub.read_to_string(&mut functools_module_stub_source)?; + + assert!(functools_module_stub_source.contains("def update_wrapper(")); + Ok(()) + } + #[test] fn resolve_package() -> anyhow::Result<()> { let TestCase { From 6283dd26de99ddc8163e553b32bb3b915992a602 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Thu, 6 Jun 2024 20:53:56 +0100 Subject: [PATCH 2/9] hopefully this fixes Windows? --- crates/red_knot/src/module.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/crates/red_knot/src/module.rs b/crates/red_knot/src/module.rs index a1cf312f32ca9..7ba7ea2753f2e 100644 --- a/crates/red_knot/src/module.rs +++ b/crates/red_knot/src/module.rs @@ -779,7 +779,7 @@ impl PackageKind { mod tests { use std::io::{Cursor, Read}; use std::num::NonZeroU32; - use std::path::PathBuf; + use std::path::{Path, PathBuf}; use zip::ZipArchive; @@ -938,9 +938,8 @@ mod tests { assert!(!TYPESHED_ZIP_BYTES.is_empty()); let mut typeshed_zip_archive = ZipArchive::new(Cursor::new(TYPESHED_ZIP_BYTES))?; - let mut functools_module_stub = typeshed_zip_archive - .by_name("stdlib/functools.pyi") - .unwrap(); + let path_to_functools = Path::new("stdlib/functools.pyi").to_str().unwrap(); + let mut functools_module_stub = typeshed_zip_archive.by_name(path_to_functools).unwrap(); assert!(functools_module_stub.is_file()); let mut functools_module_stub_source = String::new(); From 049e137b027c2d8cdecc2f8e214d1e429ca4f1e8 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Thu, 6 Jun 2024 21:07:40 +0100 Subject: [PATCH 3/9] this? --- crates/red_knot/src/module.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/red_knot/src/module.rs b/crates/red_knot/src/module.rs index 7ba7ea2753f2e..25862f6f3a808 100644 --- a/crates/red_knot/src/module.rs +++ b/crates/red_knot/src/module.rs @@ -938,8 +938,10 @@ mod tests { assert!(!TYPESHED_ZIP_BYTES.is_empty()); let mut typeshed_zip_archive = ZipArchive::new(Cursor::new(TYPESHED_ZIP_BYTES))?; - let path_to_functools = Path::new("stdlib/functools.pyi").to_str().unwrap(); - let mut functools_module_stub = typeshed_zip_archive.by_name(path_to_functools).unwrap(); + let path_to_functools = Path::new("stdlib").join("functools.pyi"); + let mut functools_module_stub = typeshed_zip_archive + .by_name(path_to_functools.to_str().unwrap()) + .unwrap(); assert!(functools_module_stub.is_file()); let mut functools_module_stub_source = String::new(); From 4eb13e164dfd183e56897f307586a5c7e128848a Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 7 Jun 2024 08:01:05 +0100 Subject: [PATCH 4/9] Update crates/red_knot/build.rs Co-authored-by: Micha Reiser --- crates/red_knot/build.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/red_knot/build.rs b/crates/red_knot/build.rs index 1ddf0aa67a912..423a85e5606fb 100644 --- a/crates/red_knot/build.rs +++ b/crates/red_knot/build.rs @@ -63,8 +63,7 @@ fn zip_dir( zip.add_directory(name, options)?; } } - zip.finish()?; - Result::Ok(()) + zip.finish() } fn main() -> Result<(), Box> { From f6c451ca97ba07208887d674bab57e091e847b3c Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 7 Jun 2024 10:42:07 +0100 Subject: [PATCH 5/9] Misc review comments --- Cargo.lock | 220 ---------------------------------- Cargo.toml | 17 ++- crates/red_knot/build.rs | 49 ++------ crates/red_knot/src/module.rs | 12 +- 4 files changed, 30 insertions(+), 268 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f657a65b34c36..888aee89d5844 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,17 +14,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" -[[package]] -name = "aes" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" -dependencies = [ - "cfg-if", - "cipher", - "cpufeatures", -] - [[package]] name = "ahash" version = "0.8.11" @@ -172,12 +161,6 @@ version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51" -[[package]] -name = "base64ct" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" - [[package]] name = "bincode" version = "1.3.3" @@ -199,15 +182,6 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", -] - [[package]] name = "bstr" version = "1.9.1" @@ -231,27 +205,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" -[[package]] -name = "bzip2" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" -dependencies = [ - "bzip2-sys", - "libc", -] - -[[package]] -name = "bzip2-sys" -version = "0.1.11+1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - [[package]] name = "cachedir" version = "0.3.1" @@ -338,16 +291,6 @@ dependencies = [ "half", ] -[[package]] -name = "cipher" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" -dependencies = [ - "crypto-common", - "inout", -] - [[package]] name = "clap" version = "4.5.4" @@ -514,12 +457,6 @@ dependencies = [ "web-sys", ] -[[package]] -name = "constant_time_eq" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" - [[package]] name = "core-foundation-sys" version = "0.8.6" @@ -532,15 +469,6 @@ version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7704b5fdd17b18ae31c4c1da5a2e0305a2bf17b5249300a9ee9ed7b72114c636" -[[package]] -name = "cpufeatures" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" -dependencies = [ - "libc", -] - [[package]] name = "crc32fast" version = "1.4.0" @@ -646,16 +574,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" -[[package]] -name = "crypto-common" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" -dependencies = [ - "generic-array", - "typenum", -] - [[package]] name = "ctrlc" version = "3.4.4" @@ -714,32 +632,12 @@ dependencies = [ "parking_lot_core", ] -[[package]] -name = "deranged" -version = "0.3.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" -dependencies = [ - "powerfmt", -] - [[package]] name = "diff" version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer", - "crypto-common", - "subtle", -] - [[package]] name = "dirs" version = "4.0.0" @@ -914,16 +812,6 @@ dependencies = [ "libc", ] -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - [[package]] name = "getopts" version = "0.2.21" @@ -1003,15 +891,6 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest", -] - [[package]] name = "home" version = "0.5.9" @@ -1153,15 +1032,6 @@ dependencies = [ "libc", ] -[[package]] -name = "inout" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" -dependencies = [ - "generic-array", -] - [[package]] name = "insta" version = "1.39.0" @@ -1555,12 +1425,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "num-conv" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" - [[package]] name = "num-traits" version = "0.2.18" @@ -1632,17 +1496,6 @@ dependencies = [ "windows-targets 0.48.5", ] -[[package]] -name = "password-hash" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" -dependencies = [ - "base64ct", - "rand_core", - "subtle", -] - [[package]] name = "paste" version = "1.0.14" @@ -1679,18 +1532,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" -[[package]] -name = "pbkdf2" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" -dependencies = [ - "digest", - "hmac", - "password-hash", - "sha2", -] - [[package]] name = "peg" version = "0.8.2" @@ -1820,12 +1661,6 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" -[[package]] -name = "powerfmt" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" - [[package]] name = "ppv-lite86" version = "0.2.17" @@ -2813,28 +2648,6 @@ dependencies = [ "syn", ] -[[package]] -name = "sha1" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - -[[package]] -name = "sha2" -version = "0.10.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - [[package]] name = "sharded-slab" version = "0.1.7" @@ -3070,25 +2883,6 @@ dependencies = [ "tikv-jemalloc-sys", ] -[[package]] -name = "time" -version = "0.3.36" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" -dependencies = [ - "deranged", - "num-conv", - "powerfmt", - "serde", - "time-core", -] - -[[package]] -name = "time-core" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" - [[package]] name = "tinytemplate" version = "1.2.1" @@ -3240,12 +3034,6 @@ version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" -[[package]] -name = "typenum" -version = "1.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" - [[package]] name = "unic-char-property" version = "0.9.0" @@ -3840,17 +3628,9 @@ version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" dependencies = [ - "aes", "byteorder", - "bzip2", - "constant_time_eq", "crc32fast", "crossbeam-utils", - "flate2", - "hmac", - "pbkdf2", - "sha1", - "time", "zstd", ] diff --git a/Cargo.toml b/Cargo.toml index 5d72b5420a511..37550097ba175 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -79,7 +79,9 @@ libc = { version = "0.2.153" } libcst = { version = "1.1.0", default-features = false } log = { version = "0.4.17" } lsp-server = { version = "0.7.6" } -lsp-types = { git = "https://github.com/astral-sh/lsp-types.git", rev = "3512a9f", features = ["proposed"] } +lsp-types = { git = "https://github.com/astral-sh/lsp-types.git", rev = "3512a9f", features = [ + "proposed", +] } matchit = { version = "0.8.1" } memchr = { version = "2.7.1" } mimalloc = { version = "0.1.39" } @@ -106,7 +108,9 @@ serde = { version = "1.0.197", features = ["derive"] } serde-wasm-bindgen = { version = "0.6.4" } serde_json = { version = "1.0.113" } serde_test = { version = "1.0.152" } -serde_with = { version = "3.6.0", default-features = false, features = ["macros"] } +serde_with = { version = "3.6.0", default-features = false, features = [ + "macros", +] } shellexpand = { version = "3.0.0" } similar = { version = "2.4.0", features = ["inline"] } smallvec = { version = "1.13.2" } @@ -131,12 +135,17 @@ unicode_names2 = { version = "1.2.2" } unicode-normalization = { version = "0.1.23" } ureq = { version = "2.9.6" } url = { version = "2.5.0" } -uuid = { version = "1.6.1", features = ["v4", "fast-rng", "macro-diagnostics", "js"] } +uuid = { version = "1.6.1", features = [ + "v4", + "fast-rng", + "macro-diagnostics", + "js", +] } walkdir = { version = "2.3.2" } wasm-bindgen = { version = "0.2.92" } wasm-bindgen-test = { version = "0.3.42" } wild = { version = "2" } -zip = { version = "0.6.6" } +zip = { version = "0.6.6", default-features = false, features = ["zstd"] } [workspace.lints.rust] unsafe_code = "warn" diff --git a/crates/red_knot/build.rs b/crates/red_knot/build.rs index 423a85e5606fb..37fa730d067e8 100644 --- a/crates/red_knot/build.rs +++ b/crates/red_knot/build.rs @@ -5,12 +5,9 @@ //! whenever the script itself changes, or whenever any files //! in `crates/red_knot/vendor/typeshed` change. -use std::error::Error; use std::fs::File; -use std::io::{Read, Write}; use std::path::Path; -use walkdir::{DirEntry, WalkDir}; use zip::result::ZipResult; use zip::write::{FileOptions, ZipWriter}; use zip::CompressionMethod; @@ -25,65 +22,45 @@ const TYPESHED_ZIP_LOCATION: &str = "vendor/zipped_typeshed.zip"; /// /// This routine is adapted from a recipe at /// -fn zip_dir( - directory_iterator: &mut impl Iterator, - prefix: &str, - writer: File, -) -> ZipResult<()> { +fn zip_dir(directory_path: &str, writer: File) -> ZipResult { let mut zip = ZipWriter::new(writer); let options = FileOptions::default() .compression_method(CompressionMethod::Zstd) .unix_permissions(0o644); - let mut buffer = Vec::new(); - for entry in directory_iterator { - let path = entry.path(); - let name = path - .strip_prefix(Path::new(prefix)) + for entry in walkdir::WalkDir::new(directory_path) { + let dir_entry = entry.unwrap(); + let relative_path = dir_entry.path(); + let name = relative_path + .strip_prefix(Path::new(directory_path)) .unwrap() .to_str() .expect("Unexpected non-utf8 typeshed path!"); // Write file or directory explicitly // Some unzip tools unzip files with directory paths correctly, some do not! - if path.is_file() { - println!("adding file {path:?} as {name:?} ..."); + if relative_path.is_file() { + println!("adding file {relative_path:?} as {name:?} ..."); zip.start_file(name, options)?; - let mut f = File::open(path)?; - - f.read_to_end(&mut buffer)?; - zip.write_all(&buffer)?; - buffer.clear(); + let mut f = File::open(relative_path)?; + std::io::copy(&mut f, &mut zip).unwrap(); } else if !name.is_empty() { // Only if not root! Avoids path spec / warning // and mapname conversion failed error on unzip - println!("adding dir {path:?} as {name:?} ..."); - + println!("adding dir {relative_path:?} as {name:?} ..."); zip.add_directory(name, options)?; } } zip.finish() } -fn main() -> Result<(), Box> { +fn main() { println!("cargo:rerun-if-changed={TYPESHED_SOURCE_DIR}"); - assert!( Path::new(TYPESHED_SOURCE_DIR).is_dir(), "Where is typeshed?" ); - let zipped_typeshed = File::create(Path::new(TYPESHED_ZIP_LOCATION)).unwrap(); - - let mut typeshed_traverser = WalkDir::new(TYPESHED_SOURCE_DIR) - .into_iter() - .filter_map(std::result::Result::ok); - - zip_dir( - &mut typeshed_traverser, - TYPESHED_SOURCE_DIR, - zipped_typeshed, - )?; - Ok(()) + zip_dir(TYPESHED_SOURCE_DIR, zipped_typeshed).unwrap(); } diff --git a/crates/red_knot/src/module.rs b/crates/red_knot/src/module.rs index 25862f6f3a808..460967c7cfd18 100644 --- a/crates/red_knot/src/module.rs +++ b/crates/red_knot/src/module.rs @@ -12,13 +12,6 @@ use crate::files::FileId; use crate::semantic::Dependency; use crate::FxDashMap; -// The file path here is hardcoded in this crate's `build.rs` script. -// Luckily this crate will fail to build if this file isn't available at build time. -// -// Allow dead code for now; actually resolving modules to vendored typeshed stubs will come soon! -#[allow(dead_code)] -const TYPESHED_ZIP_BYTES: &[u8] = include_bytes!("../vendor/zipped_typeshed.zip"); - /// Representation of a Python module. /// /// The inner type wrapped by this struct is a unique identifier for the module @@ -787,7 +780,7 @@ mod tests { use crate::db::SourceDb; use crate::module::{ path_to_module, resolve_module, set_module_search_paths, ModuleKind, ModuleName, - ModuleResolutionInputs, TYPESHED_STDLIB_DIRECTORY, TYPESHED_ZIP_BYTES, + ModuleResolutionInputs, TYPESHED_STDLIB_DIRECTORY, }; use crate::semantic::Dependency; @@ -935,6 +928,9 @@ mod tests { #[test] fn typeshed_zip_created_at_build_time() -> anyhow::Result<()> { + // The file path here is hardcoded in this crate's `build.rs` script. + // Luckily this crate will fail to build if this file isn't available at build time. + const TYPESHED_ZIP_BYTES: &[u8] = include_bytes!("../vendor/zipped_typeshed.zip"); assert!(!TYPESHED_ZIP_BYTES.is_empty()); let mut typeshed_zip_archive = ZipArchive::new(Cursor::new(TYPESHED_ZIP_BYTES))?; From 6c7a33c9380676c8c78d75dab6b35e1f3c434cd7 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 7 Jun 2024 12:16:53 +0100 Subject: [PATCH 6/9] Put the zip in OUT_DIR --- .gitignore | 3 --- crates/red_knot/build.rs | 17 ++++++++++++----- crates/red_knot/src/module.rs | 3 ++- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 9443c92091188..4302ff30a762a 100644 --- a/.gitignore +++ b/.gitignore @@ -215,6 +215,3 @@ cython_debug/ !crates/ruff_python_resolver/resources/test/airflow/venv/lib !crates/ruff_python_resolver/resources/test/airflow/venv/lib/python3.11/site-packages/_watchdog_fsevents.cpython-311-darwin.so !crates/ruff_python_resolver/resources/test/airflow/venv/lib/python3.11/site-packages/orjson/orjson.cpython-311-darwin.so - -# Ignore the zipped version of typeshed that we create at build time -crates/red_knot/vendor/zipped_typeshed.zip diff --git a/crates/red_knot/build.rs b/crates/red_knot/build.rs index 37fa730d067e8..8c2449de3ae0e 100644 --- a/crates/red_knot/build.rs +++ b/crates/red_knot/build.rs @@ -13,10 +13,7 @@ use zip::write::{FileOptions, ZipWriter}; use zip::CompressionMethod; const TYPESHED_SOURCE_DIR: &str = "vendor/typeshed"; - -// NB: This is .gitignored; make sure to change the .gitignore entry -// if you change this location -const TYPESHED_ZIP_LOCATION: &str = "vendor/zipped_typeshed.zip"; +const TYPESHED_ZIP_LOCATION: &str = "zipped_typeshed.zip"; /// Recursively zip the contents of an entire directory. /// @@ -61,6 +58,16 @@ fn main() { Path::new(TYPESHED_SOURCE_DIR).is_dir(), "Where is typeshed?" ); - let zipped_typeshed = File::create(Path::new(TYPESHED_ZIP_LOCATION)).unwrap(); + let out_dir = std::env::var("OUT_DIR").unwrap(); + + // N.B. Deliberately using `format!()` instead of `Path::join()` here, + // so that we use `/` as a path separator on all platforms. + // That enables us to load the typeshed zip at compile time in `module.rs` + // (otherwise we'd have to dynamically determine the exact path to the typeshed zip + // based on the default path separator for the specific platform we're on, + // which can't be done at compile time.) + let zipped_typeshed_location = format!("{out_dir}/{TYPESHED_ZIP_LOCATION}"); + + let zipped_typeshed = File::create(zipped_typeshed_location).unwrap(); zip_dir(TYPESHED_SOURCE_DIR, zipped_typeshed).unwrap(); } diff --git a/crates/red_knot/src/module.rs b/crates/red_knot/src/module.rs index 460967c7cfd18..9050aaebbf112 100644 --- a/crates/red_knot/src/module.rs +++ b/crates/red_knot/src/module.rs @@ -930,7 +930,8 @@ mod tests { fn typeshed_zip_created_at_build_time() -> anyhow::Result<()> { // The file path here is hardcoded in this crate's `build.rs` script. // Luckily this crate will fail to build if this file isn't available at build time. - const TYPESHED_ZIP_BYTES: &[u8] = include_bytes!("../vendor/zipped_typeshed.zip"); + const TYPESHED_ZIP_BYTES: &[u8] = + include_bytes!(concat!(env!("OUT_DIR"), "/zipped_typeshed.zip")); assert!(!TYPESHED_ZIP_BYTES.is_empty()); let mut typeshed_zip_archive = ZipArchive::new(Cursor::new(TYPESHED_ZIP_BYTES))?; From c5ad832102cc9cd16da3c8035c0fb9d413da6890 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 7 Jun 2024 15:27:19 +0100 Subject: [PATCH 7/9] Set the location of the zip via environment variable --- .cargo/config.toml | 6 ++++++ crates/red_knot/build.rs | 6 ++++-- crates/red_knot/src/module.rs | 5 ++--- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index f7d8e616f8622..535e2bb0447d0 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -8,3 +8,9 @@ benchmark = "bench -p ruff_benchmark --bench linter --bench formatter --" # See: https://github.com/astral-sh/ruff/issues/11503 [target.'cfg(all(target_env="msvc", target_os = "windows"))'] rustflags = ["-C", "target-feature=+crt-static"] + +[env] +# Location for where to put the zip containing our vendored typeshed stubs +# that the red-knot crate creates at build time. +# The location is relative to the `OUT_DIR` environment variable that cargo sets. +RUFF_VENDORED_TYPESHED_ZIP = "/vendored_typeshed.zip" diff --git a/crates/red_knot/build.rs b/crates/red_knot/build.rs index 8c2449de3ae0e..652882d1d8888 100644 --- a/crates/red_knot/build.rs +++ b/crates/red_knot/build.rs @@ -13,7 +13,6 @@ use zip::write::{FileOptions, ZipWriter}; use zip::CompressionMethod; const TYPESHED_SOURCE_DIR: &str = "vendor/typeshed"; -const TYPESHED_ZIP_LOCATION: &str = "zipped_typeshed.zip"; /// Recursively zip the contents of an entire directory. /// @@ -60,13 +59,16 @@ fn main() { ); let out_dir = std::env::var("OUT_DIR").unwrap(); + // This environment variable is set in `.cargo/config.toml` + let typeshed_destination = std::env::var("RUFF_VENDORED_TYPESHED_ZIP").unwrap(); + // N.B. Deliberately using `format!()` instead of `Path::join()` here, // so that we use `/` as a path separator on all platforms. // That enables us to load the typeshed zip at compile time in `module.rs` // (otherwise we'd have to dynamically determine the exact path to the typeshed zip // based on the default path separator for the specific platform we're on, // which can't be done at compile time.) - let zipped_typeshed_location = format!("{out_dir}/{TYPESHED_ZIP_LOCATION}"); + let zipped_typeshed_location = format!("{out_dir}{typeshed_destination}"); let zipped_typeshed = File::create(zipped_typeshed_location).unwrap(); zip_dir(TYPESHED_SOURCE_DIR, zipped_typeshed).unwrap(); diff --git a/crates/red_knot/src/module.rs b/crates/red_knot/src/module.rs index 9050aaebbf112..35c3b16ab9b3f 100644 --- a/crates/red_knot/src/module.rs +++ b/crates/red_knot/src/module.rs @@ -928,10 +928,9 @@ mod tests { #[test] fn typeshed_zip_created_at_build_time() -> anyhow::Result<()> { - // The file path here is hardcoded in this crate's `build.rs` script. - // Luckily this crate will fail to build if this file isn't available at build time. + // The RUFF_VENDORED_TYPESHED_ZIP environment variable is set in `.cargo/config.toml` const TYPESHED_ZIP_BYTES: &[u8] = - include_bytes!(concat!(env!("OUT_DIR"), "/zipped_typeshed.zip")); + include_bytes!(concat!(env!("OUT_DIR"), env!("RUFF_VENDORED_TYPESHED_ZIP"))); assert!(!TYPESHED_ZIP_BYTES.is_empty()); let mut typeshed_zip_archive = ZipArchive::new(Cursor::new(TYPESHED_ZIP_BYTES))?; From a5f2bcb95ff6113ac3c68466530505e2cb217971 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 7 Jun 2024 15:53:01 +0100 Subject: [PATCH 8/9] Revert "Set the location of the zip via environment variable" This reverts commit c5ad832102cc9cd16da3c8035c0fb9d413da6890. --- .cargo/config.toml | 6 ------ crates/red_knot/build.rs | 6 ++---- crates/red_knot/src/module.rs | 5 +++-- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index 535e2bb0447d0..f7d8e616f8622 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -8,9 +8,3 @@ benchmark = "bench -p ruff_benchmark --bench linter --bench formatter --" # See: https://github.com/astral-sh/ruff/issues/11503 [target.'cfg(all(target_env="msvc", target_os = "windows"))'] rustflags = ["-C", "target-feature=+crt-static"] - -[env] -# Location for where to put the zip containing our vendored typeshed stubs -# that the red-knot crate creates at build time. -# The location is relative to the `OUT_DIR` environment variable that cargo sets. -RUFF_VENDORED_TYPESHED_ZIP = "/vendored_typeshed.zip" diff --git a/crates/red_knot/build.rs b/crates/red_knot/build.rs index 652882d1d8888..8c2449de3ae0e 100644 --- a/crates/red_knot/build.rs +++ b/crates/red_knot/build.rs @@ -13,6 +13,7 @@ use zip::write::{FileOptions, ZipWriter}; use zip::CompressionMethod; const TYPESHED_SOURCE_DIR: &str = "vendor/typeshed"; +const TYPESHED_ZIP_LOCATION: &str = "zipped_typeshed.zip"; /// Recursively zip the contents of an entire directory. /// @@ -59,16 +60,13 @@ fn main() { ); let out_dir = std::env::var("OUT_DIR").unwrap(); - // This environment variable is set in `.cargo/config.toml` - let typeshed_destination = std::env::var("RUFF_VENDORED_TYPESHED_ZIP").unwrap(); - // N.B. Deliberately using `format!()` instead of `Path::join()` here, // so that we use `/` as a path separator on all platforms. // That enables us to load the typeshed zip at compile time in `module.rs` // (otherwise we'd have to dynamically determine the exact path to the typeshed zip // based on the default path separator for the specific platform we're on, // which can't be done at compile time.) - let zipped_typeshed_location = format!("{out_dir}{typeshed_destination}"); + let zipped_typeshed_location = format!("{out_dir}/{TYPESHED_ZIP_LOCATION}"); let zipped_typeshed = File::create(zipped_typeshed_location).unwrap(); zip_dir(TYPESHED_SOURCE_DIR, zipped_typeshed).unwrap(); diff --git a/crates/red_knot/src/module.rs b/crates/red_knot/src/module.rs index 35c3b16ab9b3f..9050aaebbf112 100644 --- a/crates/red_knot/src/module.rs +++ b/crates/red_knot/src/module.rs @@ -928,9 +928,10 @@ mod tests { #[test] fn typeshed_zip_created_at_build_time() -> anyhow::Result<()> { - // The RUFF_VENDORED_TYPESHED_ZIP environment variable is set in `.cargo/config.toml` + // The file path here is hardcoded in this crate's `build.rs` script. + // Luckily this crate will fail to build if this file isn't available at build time. const TYPESHED_ZIP_BYTES: &[u8] = - include_bytes!(concat!(env!("OUT_DIR"), env!("RUFF_VENDORED_TYPESHED_ZIP"))); + include_bytes!(concat!(env!("OUT_DIR"), "/zipped_typeshed.zip")); assert!(!TYPESHED_ZIP_BYTES.is_empty()); let mut typeshed_zip_archive = ZipArchive::new(Cursor::new(TYPESHED_ZIP_BYTES))?; From f5e6966c1a2f0d811a2ee1b4a5ff3213d8de6eb7 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 7 Jun 2024 15:55:07 +0100 Subject: [PATCH 9/9] nit --- Cargo.toml | 15 +++------------ crates/red_knot/build.rs | 4 ++-- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 37550097ba175..2193689607f13 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -79,9 +79,7 @@ libc = { version = "0.2.153" } libcst = { version = "1.1.0", default-features = false } log = { version = "0.4.17" } lsp-server = { version = "0.7.6" } -lsp-types = { git = "https://github.com/astral-sh/lsp-types.git", rev = "3512a9f", features = [ - "proposed", -] } +lsp-types = { git = "https://github.com/astral-sh/lsp-types.git", rev = "3512a9f", features = ["proposed"] } matchit = { version = "0.8.1" } memchr = { version = "2.7.1" } mimalloc = { version = "0.1.39" } @@ -108,9 +106,7 @@ serde = { version = "1.0.197", features = ["derive"] } serde-wasm-bindgen = { version = "0.6.4" } serde_json = { version = "1.0.113" } serde_test = { version = "1.0.152" } -serde_with = { version = "3.6.0", default-features = false, features = [ - "macros", -] } +serde_with = { version = "3.6.0", default-features = false, features = ["macros"] } shellexpand = { version = "3.0.0" } similar = { version = "2.4.0", features = ["inline"] } smallvec = { version = "1.13.2" } @@ -135,12 +131,7 @@ unicode_names2 = { version = "1.2.2" } unicode-normalization = { version = "0.1.23" } ureq = { version = "2.9.6" } url = { version = "2.5.0" } -uuid = { version = "1.6.1", features = [ - "v4", - "fast-rng", - "macro-diagnostics", - "js", -] } +uuid = { version = "1.6.1", features = ["v4", "fast-rng", "macro-diagnostics", "js"] } walkdir = { version = "2.3.2" } wasm-bindgen = { version = "0.2.92" } wasm-bindgen-test = { version = "0.3.42" } diff --git a/crates/red_knot/build.rs b/crates/red_knot/build.rs index 8c2449de3ae0e..c46a354e6cbff 100644 --- a/crates/red_knot/build.rs +++ b/crates/red_knot/build.rs @@ -13,7 +13,7 @@ use zip::write::{FileOptions, ZipWriter}; use zip::CompressionMethod; const TYPESHED_SOURCE_DIR: &str = "vendor/typeshed"; -const TYPESHED_ZIP_LOCATION: &str = "zipped_typeshed.zip"; +const TYPESHED_ZIP_LOCATION: &str = "/zipped_typeshed.zip"; /// Recursively zip the contents of an entire directory. /// @@ -66,7 +66,7 @@ fn main() { // (otherwise we'd have to dynamically determine the exact path to the typeshed zip // based on the default path separator for the specific platform we're on, // which can't be done at compile time.) - let zipped_typeshed_location = format!("{out_dir}/{TYPESHED_ZIP_LOCATION}"); + let zipped_typeshed_location = format!("{out_dir}{TYPESHED_ZIP_LOCATION}"); let zipped_typeshed = File::create(zipped_typeshed_location).unwrap(); zip_dir(TYPESHED_SOURCE_DIR, zipped_typeshed).unwrap();